Traffic reduction and tuned caching strats (#2215)

* Update the feed to only check latest on focus after 30s, but to do a full reset on focus after 1 hour to avoid very stale data

* Remove the isFeedPublic query

* Fix: avoid double next-page fetches

* Reduce some poll intervals to reduce server load

* Guard against double-fires of fetchNextPage

* Reduce polling on blurred screens

src/state/queries/feed.ts

@@ -161,51 +161,6 @@ export function useFeedSourceInfoQuery({uri}: {uri: string}) {
   })
 }
 
-export const isFeedPublicQueryKey = ({uri}: {uri: string}) => [
-  'isFeedPublic',
-  uri,
-]
-
-export function useIsFeedPublicQuery({uri}: {uri: string}) {
-  return useQuery({
-    queryKey: isFeedPublicQueryKey({uri}),
-    queryFn: async ({queryKey}) => {
-      const [, uri] = queryKey
-      try {
-        const res = await getAgent().app.bsky.feed.getFeed({
-          feed: uri,
-          limit: 1,
-        })
-        return {
-          isPublic: Boolean(res.data.feed),
-          error: undefined,
-        }
-      } catch (e: any) {
-        /**
-         * This should be an `XRPCError`, but I can't safely import from
-         * `@atproto/xrpc` due to a depdency on node's `crypto` module.
-         *
-         * @see https://github.com/bluesky-social/atproto/blob/c17971a2d8e424cc7f10c071d97c07c08aa319cf/packages/xrpc/src/client.ts#L126
-         */
-        if (e?.status === 401) {
-          return {
-            isPublic: false,
-            error: e,
-          }
-        }
-
-        /*
-         * Non-401 response means something else went wrong on the server
-         */
-        return {
-          isPublic: true,
-          error: e,
-        }
-      }
-    },
-  })
-}
-
 export const useGetPopularFeedsQueryKey = ['getPopularFeeds']
 
 export function useGetPopularFeedsQuery() {

src/state/queries/notifications/feed.ts

@@ -16,7 +16,7 @@
  * 3. Don't call this query's `refetch()` if you're trying to sync latest; call `checkUnread()` instead.
  */
 
-import {useEffect} from 'react'
+import {useEffect, useRef} from 'react'
 import {AppBskyFeedDefs} from '@atproto/api'
 import {
   useInfiniteQuery,
@@ -49,6 +49,7 @@ export function useNotificationFeedQuery(opts?: {enabled?: boolean}) {
   const threadMutes = useMutedThreads()
   const unreads = useUnreadNotificationsApi()
   const enabled = opts?.enabled !== false
+  const lastPageCountRef = useRef(0)
 
   const query = useInfiniteQuery<
     FeedPage,
@@ -104,24 +105,26 @@ export function useNotificationFeedQuery(opts?: {enabled?: boolean}) {
 
   useEffect(() => {
     const {isFetching, hasNextPage, data} = query
-
+    if (isFetching || !hasNextPage) {
-    let count = 0
+      return
-    let numEmpties = 0
-    for (const page of data?.pages || []) {
-      if (!page.items.length) {
-        numEmpties++
-      }
-      count += page.items.length
     }
 
+    // avoid double-fires of fetchNextPage()
     if (
-      !isFetching &&
+      lastPageCountRef.current !== 0 &&
-      hasNextPage &&
+      lastPageCountRef.current === data?.pages?.length
-      count < PAGE_SIZE &&
-      numEmpties < 3 &&
-      (data?.pages.length || 0) < 6
     ) {
+      return
+    }
+
+    // fetch next page if we haven't gotten a full page of content
+    let count = 0
+    for (const page of data?.pages || []) {
+      count += page.items.length
+    }
+    if (count < PAGE_SIZE && (data?.pages.length || 0) < 6) {
       query.fetchNextPage()
+      lastPageCountRef.current = data?.pages?.length || 0
     }
   }, [query])
 

src/state/queries/post-feed.ts

@@ -1,4 +1,4 @@
-import React, {useCallback, useEffect} from 'react'
+import React, {useCallback, useEffect, useRef} from 'react'
 import {
   AppBskyFeedDefs,
   AppBskyFeedPost,
@@ -78,6 +78,7 @@ export interface FeedPageUnselected {
   api: FeedAPI
   cursor: string | undefined
   feed: AppBskyFeedDefs.FeedViewPost[]
+  fetchedAt: number
 }
 
 export interface FeedPage {
@@ -85,6 +86,7 @@ export interface FeedPage {
   tuner: FeedTuner | NoopFeedTuner
   cursor: string | undefined
   slices: FeedPostSlice[]
+  fetchedAt: number
 }
 
 const PAGE_SIZE = 30
@@ -98,11 +100,12 @@ export function usePostFeedQuery(
   const feedTuners = useFeedTuners(feedDesc)
   const moderationOpts = useModerationOpts()
   const enabled = opts?.enabled !== false && Boolean(moderationOpts)
-  const lastRun = React.useRef<{
+  const lastRun = useRef<{
     data: InfiniteData<FeedPageUnselected>
     args: typeof selectArgs
     result: InfiniteData<FeedPage>
   } | null>(null)
+  const lastPageCountRef = useRef(0)
 
   // Make sure this doesn't invalidate unless really needed.
   const selectArgs = React.useMemo(
@@ -152,6 +155,7 @@ export function usePostFeedQuery(
         api,
         cursor: res.cursor,
         feed: res.feed,
+        fetchedAt: Date.now(),
       }
     },
     initialPageParam: undefined,
@@ -214,6 +218,7 @@ export function usePostFeedQuery(
               api: page.api,
               tuner,
               cursor: page.cursor,
+              fetchedAt: page.fetchedAt,
               slices: tuner
                 .tune(page.feed)
                 .map(slice => {
@@ -279,26 +284,28 @@ export function usePostFeedQuery(
 
   useEffect(() => {
     const {isFetching, hasNextPage, data} = query
+    if (isFetching || !hasNextPage) {
+      return
+    }
 
+    // avoid double-fires of fetchNextPage()
+    if (
+      lastPageCountRef.current !== 0 &&
+      lastPageCountRef.current === data?.pages?.length
+    ) {
+      return
+    }
+
+    // fetch next page if we haven't gotten a full page of content
     let count = 0
-    let numEmpties = 0
     for (const page of data?.pages || []) {
-      if (page.slices.length === 0) {
-        numEmpties++
-      }
       for (const slice of page.slices) {
         count += slice.items.length
       }
     }
-
+    if (count < PAGE_SIZE && (data?.pages.length || 0) < 6) {
-    if (
-      !isFetching &&
-      hasNextPage &&
-      count < PAGE_SIZE &&
-      numEmpties < 3 &&
-      (data?.pages.length || 0) < 6
-    ) {
       query.fetchNextPage()
+      lastPageCountRef.current = data?.pages?.length || 0
     }
   }, [query])
 

src/state/queries/profile.ts

@@ -35,9 +35,7 @@ export function useProfileQuery({did}: {did: string | undefined}) {
     // if you remove it, the UI infinite-loops
     // -prf
     staleTime: isCurrentAccount ? STALE.SECONDS.THIRTY : STALE.MINUTES.FIVE,
-    refetchInterval: isCurrentAccount
+    refetchInterval: STALE.MINUTES.FIVE,
-      ? STALE.SECONDS.THIRTY
-      : STALE.MINUTES.FIVE,
     queryKey: RQKEY(did || ''),
     queryFn: async () => {
       const res = await getAgent().getProfile({actor: did || ''})

src/view/com/feeds/FeedPage.tsx

@@ -29,7 +29,7 @@ import {truncateAndInvalidate} from '#/state/queries/util'
 import {TabState, getTabState, getRootNavigation} from '#/lib/routes/helpers'
 import {isNative} from '#/platform/detection'
 
-const POLL_FREQ = 30e3 // 30sec
+const POLL_FREQ = 60e3 // 60sec
 
 export function FeedPage({
   testID,

src/view/com/posts/Feed.tsx

@@ -29,12 +29,16 @@ import {
 import {isWeb} from '#/platform/detection'
 import {listenPostCreated} from '#/state/events'
 import {useSession} from '#/state/session'
+import {STALE} from '#/state/queries'
 
 const LOADING_ITEM = {_reactKey: '__loading__'}
 const EMPTY_FEED_ITEM = {_reactKey: '__empty__'}
 const ERROR_ITEM = {_reactKey: '__error__'}
 const LOAD_MORE_ERROR_ITEM = {_reactKey: '__load_more_error__'}
 
+const REFRESH_AFTER = STALE.HOURS.ONE
+const CHECK_LATEST_AFTER = STALE.SECONDS.THIRTY
+
 let Feed = ({
   feed,
   feedParams,
@@ -77,6 +81,7 @@ let Feed = ({
   const {currentAccount} = useSession()
   const [isPTRing, setIsPTRing] = React.useState(false)
   const checkForNewRef = React.useRef<(() => void) | null>(null)
+  const lastFetchRef = React.useRef<number>(Date.now())
 
   const opts = React.useMemo(
     () => ({enabled, ignoreFilterFor}),
@@ -94,6 +99,9 @@ let Feed = ({
     fetchNextPage,
   } = usePostFeedQuery(feed, feedParams, opts)
   const isEmpty = !isFetching && !data?.pages[0]?.slices.length
+  if (data?.pages[0]) {
+    lastFetchRef.current = data?.pages[0].fetchedAt
+  }
 
   const checkForNew = React.useCallback(async () => {
     if (!data?.pages[0] || isFetching || !onHasNew || !enabled) {
@@ -133,11 +141,21 @@ let Feed = ({
     checkForNewRef.current = checkForNew
   }, [checkForNew])
   React.useEffect(() => {
-    if (enabled && checkForNewRef.current) {
+    if (enabled) {
-      // check for new on enable (aka on focus)
+      const timeSinceFirstLoad = Date.now() - lastFetchRef.current
-      checkForNewRef.current()
+      if (timeSinceFirstLoad > REFRESH_AFTER) {
+        // do a full refresh
+        scrollElRef?.current?.scrollToOffset({offset: 0, animated: false})
+        queryClient.resetQueries({queryKey: RQKEY(feed)})
+      } else if (
+        timeSinceFirstLoad > CHECK_LATEST_AFTER &&
+        checkForNewRef.current
+      ) {
+        // check for new on enable (aka on focus)
+        checkForNewRef.current()
+      }
     }
-  }, [enabled])
+  }, [enabled, feed, queryClient, scrollElRef])
   React.useEffect(() => {
     let cleanup1: () => void | undefined, cleanup2: () => void | undefined
     const subscription = AppState.addEventListener('change', nextAppState => {

src/view/screens/ProfileFeed.tsx

@@ -1,7 +1,7 @@
 import React, {useMemo, useCallback} from 'react'
 import {Dimensions, StyleSheet, View, ActivityIndicator} from 'react-native'
 import {NativeStackScreenProps} from '@react-navigation/native-stack'
-import {useNavigation} from '@react-navigation/native'
+import {useIsFocused, useNavigation} from '@react-navigation/native'
 import {useQueryClient} from '@tanstack/react-query'
 import {usePalette} from 'lib/hooks/usePalette'
 import {HeartIcon, HeartIconSolid} from 'lib/icons'
@@ -42,11 +42,7 @@ import {logger} from '#/logger'
 import {Trans, msg} from '@lingui/macro'
 import {useLingui} from '@lingui/react'
 import {useModalControls} from '#/state/modals'
-import {
+import {useFeedSourceInfoQuery, FeedSourceFeedInfo} from '#/state/queries/feed'
-  useFeedSourceInfoQuery,
-  FeedSourceFeedInfo,
-  useIsFeedPublicQuery,
-} from '#/state/queries/feed'
 import {useResolveUriQuery} from '#/state/queries/resolve-uri'
 import {
   UsePreferencesQueryResponse,
@@ -132,10 +128,8 @@ export function ProfileFeedScreen(props: Props) {
 function ProfileFeedScreenIntermediate({feedUri}: {feedUri: string}) {
   const {data: preferences} = usePreferencesQuery()
   const {data: info} = useFeedSourceInfoQuery({uri: feedUri})
-  const {isLoading: isPublicStatusLoading, data: isPublicResponse} =
-    useIsFeedPublicQuery({uri: feedUri})
 
-  if (!preferences || !info || isPublicStatusLoading) {
+  if (!preferences || !info) {
     return (
       <CenteredView>
         <View style={s.p20}>
@@ -149,7 +143,6 @@ function ProfileFeedScreenIntermediate({feedUri}: {feedUri: string}) {
     <ProfileFeedScreenInner
       preferences={preferences}
       feedInfo={info as FeedSourceFeedInfo}
-      isPublicResponse={isPublicResponse}
     />
   )
 }
@@ -157,11 +150,9 @@ function ProfileFeedScreenIntermediate({feedUri}: {feedUri: string}) {
 export function ProfileFeedScreenInner({
   preferences,
   feedInfo,
-  isPublicResponse,
 }: {
   preferences: UsePreferencesQueryResponse
   feedInfo: FeedSourceFeedInfo
-  isPublicResponse: ReturnType<typeof useIsFeedPublicQuery>['data']
 }) {
   const {_} = useLingui()
   const pal = usePalette('default')
@@ -170,6 +161,7 @@ export function ProfileFeedScreenInner({
   const {openComposer} = useComposerControls()
   const {track} = useAnalytics()
   const feedSectionRef = React.useRef<SectionRef>(null)
+  const isScreenFocused = useIsFocused()
 
   const {
     mutateAsync: saveFeed,
@@ -205,6 +197,9 @@ export function ProfileFeedScreenInner({
 
   useSetTitle(feedInfo?.displayName)
 
+  // event handlers
+  //
+
   const onToggleSaved = React.useCallback(async () => {
     try {
       Haptics.default()
@@ -398,21 +393,15 @@ export function ProfileFeedScreenInner({
         isHeaderReady={true}
         renderHeader={renderHeader}
         onCurrentPageSelected={onCurrentPageSelected}>
-        {({headerHeight, scrollElRef, isFocused}) =>
+        {({headerHeight, scrollElRef, isFocused}) => (
-          isPublicResponse?.isPublic ? (
+          <FeedSection
-            <FeedSection
+            ref={feedSectionRef}
-              ref={feedSectionRef}
+            feed={`feedgen|${feedInfo.uri}`}
-              feed={`feedgen|${feedInfo.uri}`}
+            headerHeight={headerHeight}
-              headerHeight={headerHeight}
+            scrollElRef={scrollElRef as ListRef}
-              scrollElRef={scrollElRef as ListRef}
+            isFocused={isScreenFocused && isFocused}
-              isFocused={isFocused}
+          />
-            />
+        )}
-          ) : (
-            <CenteredView sideBorders style={[{paddingTop: headerHeight}]}>
-              <NonPublicFeedMessage rawError={isPublicResponse?.error} />
-            </CenteredView>
-          )
-        }
         {({headerHeight, scrollElRef}) => (
           <AboutSection
             feedOwnerDid={feedInfo.creatorDid}
@@ -446,45 +435,6 @@ export function ProfileFeedScreenInner({
   )
 }
 
-function NonPublicFeedMessage({rawError}: {rawError?: Error}) {
-  const pal = usePalette('default')
-
-  return (
-    <View
-      style={[
-        pal.border,
-        {
-          padding: 18,
-          borderTopWidth: 1,
-          minHeight: Dimensions.get('window').height * 1.5,
-        },
-      ]}>
-      <View
-        style={[
-          pal.viewLight,
-          {
-            padding: 12,
-            borderRadius: 8,
-            gap: 12,
-          },
-        ]}>
-        <Text style={[pal.text]}>
-          <Trans>
-            Looks like this feed is only available to users with a Bluesky
-            account. Please sign up or sign in to view this feed!
-          </Trans>
-        </Text>
-
-        {rawError?.message && (
-          <Text style={pal.textLight}>
-            <Trans>Message from server</Trans>: {rawError.message}
-          </Text>
-        )}
-      </View>
-    </View>
-  )
-}
-
 interface FeedSectionProps {
   feed: FeedDescriptor
   headerHeight: number
@@ -519,7 +469,7 @@ const FeedSection = React.forwardRef<SectionRef, FeedSectionProps>(
         <Feed
           enabled={isFocused}
           feed={feed}
-          pollInterval={30e3}
+          pollInterval={60e3}
           scrollElRef={scrollElRef}
           onHasNew={setHasNew}
           onScrolledDownChange={setIsScrolledDown}

src/view/screens/ProfileList.tsx

@@ -1,6 +1,6 @@
 import React, {useCallback, useMemo} from 'react'
 import {ActivityIndicator, Pressable, StyleSheet, View} from 'react-native'
-import {useFocusEffect} from '@react-navigation/native'
+import {useFocusEffect, useIsFocused} from '@react-navigation/native'
 import {NativeStackScreenProps, CommonNavigatorParams} from 'lib/routes/types'
 import {useNavigation} from '@react-navigation/native'
 import {FontAwesomeIcon} from '@fortawesome/react-native-fontawesome'
@@ -115,6 +115,7 @@ function ProfileListScreenLoaded({
   const aboutSectionRef = React.useRef<SectionRef>(null)
   const {openModal} = useModalControls()
   const isCurateList = list.purpose === 'app.bsky.graph.defs#curatelist'
+  const isScreenFocused = useIsFocused()
 
   useSetTitle(list.name)
 
@@ -165,7 +166,7 @@ function ProfileListScreenLoaded({
               feed={`list|${uri}`}
               scrollElRef={scrollElRef as ListRef}
               headerHeight={headerHeight}
-              isFocused={isFocused}
+              isFocused={isScreenFocused && isFocused}
             />
           )}
           {({headerHeight, scrollElRef}) => (
@@ -623,7 +624,7 @@ const FeedSection = React.forwardRef<SectionRef, FeedSectionProps>(
           testID="listFeed"
           enabled={isFocused}
           feed={feed}
-          pollInterval={30e3}
+          pollInterval={60e3}
           scrollElRef={scrollElRef}
           onHasNew={setHasNew}
           onScrolledDownChange={setIsScrolledDown}