Prefilter the mergefeed to ensure a better mix of following and custom feeds (#1498)

* Prefilter the mergefeed to ensure a better mix of following and custom feeds

* Test suite improvements & tests for the mergefeed (#1499)

* Disable invite codes test for now

* Update test sim to latest iphone

* Introduce TestCtrls driver

* Add mergefeed tests

src/App.native.tsx

@@ -18,6 +18,7 @@ import * as Toast from './view/com/util/Toast'
 import {handleLink} from './Navigation'
 import {QueryClientProvider} from '@tanstack/react-query'
 import {queryClient} from 'lib/react-query'
+import {TestCtrls} from 'view/com/testing/TestCtrls'
 
 SplashScreen.preventAutoHideAsync()
 
@@ -59,6 +60,7 @@ const App = observer(function AppImpl() {
           <analytics.Provider>
             <RootStoreProvider value={rootStore}>
               <GestureHandlerRootView style={s.h100pct}>
+                <TestCtrls />
                 <Shell />
               </GestureHandlerRootView>
             </RootStoreProvider>

src/lib/api/feed-manip.ts

@@ -128,23 +128,32 @@ export class FeedTuner {
   tune(
     feed: FeedViewPost[],
     tunerFns: FeedTunerFn[] = [],
-    {dryRun}: {dryRun: boolean} = {dryRun: false},
+    {dryRun, maintainOrder}: {dryRun: boolean; maintainOrder: boolean} = {
+      dryRun: false,
+      maintainOrder: false,
+    },
   ): FeedViewPostsSlice[] {
     let slices: FeedViewPostsSlice[] = []
 
-    // arrange the posts into thread slices
-    for (let i = feed.length - 1; i >= 0; i--) {
-      const item = feed[i]
+    if (maintainOrder) {
+      slices = feed.map(item => new FeedViewPostsSlice([item]))
+    } else {
+      // arrange the posts into thread slices
+      for (let i = feed.length - 1; i >= 0; i--) {
+        const item = feed[i]
 
-      const selfReplyUri = getSelfReplyUri(item)
-      if (selfReplyUri) {
-        const parent = slices.find(item2 => item2.isNextInThread(selfReplyUri))
-        if (parent) {
-          parent.insert(item)
-          continue
+        const selfReplyUri = getSelfReplyUri(item)
+        if (selfReplyUri) {
+          const parent = slices.find(item2 =>
+            item2.isNextInThread(selfReplyUri),
+          )
+          if (parent) {
+            parent.insert(item)
+            continue
+          }
         }
+        slices.unshift(new FeedViewPostsSlice([item]))
       }
-      slices.unshift(new FeedViewPostsSlice([item]))
     }
 
     // run the custom tuners

src/lib/api/feed/merge.ts

@@ -4,6 +4,7 @@ import {RootStoreModel} from 'state/index'
 import {timeout} from 'lib/async/timeout'
 import {bundleAsync} from 'lib/async/bundle'
 import {feedUriToHref} from 'lib/strings/url-helpers'
+import {FeedTuner} from '../feed-manip'
 import {FeedAPI, FeedAPIResponse, FeedSourceInfo} from './types'
 
 const REQUEST_WAIT_MS = 500 // 500ms
@@ -43,7 +44,7 @@ export class MergeFeedAPI implements FeedAPI {
 
     // always keep following topped up
     if (this.following.numReady < limit) {
-      promises.push(this.following.fetchNext(30))
+      promises.push(this.following.fetchNext(60))
     }
 
     // pick the next feeds to sample from
@@ -84,7 +85,8 @@ export class MergeFeedAPI implements FeedAPI {
     const i = this.itemCursor++
     const candidateFeeds = this.customFeeds.filter(f => f.numReady > 0)
     const canSample = candidateFeeds.length > 0
-    const hasFollows = this.following.numReady > 0
+    const hasFollows = this.following.hasMore
+    const hasFollowsReady = this.following.numReady > 0
 
     // this condition establishes the frequency that custom feeds are woven into follows
     const shouldSample =
@@ -98,7 +100,11 @@ export class MergeFeedAPI implements FeedAPI {
       // time to sample, or the user isnt following anybody
       return candidateFeeds[this.sampleCursor++ % candidateFeeds.length].take(1)
     }
-    // not time to sample
+    if (!hasFollowsReady) {
+      // stop here so more follows can be fetched
+      return []
+    }
+    // provide follow
     return this.following.take(1)
   }
 
@@ -174,6 +180,13 @@ class MergeFeedSource {
 }
 
 class MergeFeedSource_Following extends MergeFeedSource {
+  tuner = new FeedTuner()
+
+  reset() {
+    super.reset()
+    this.tuner.reset()
+  }
+
   async fetchNext(n: number) {
     return this._fetchNextInner(n)
   }
@@ -183,10 +196,16 @@ class MergeFeedSource_Following extends MergeFeedSource {
     limit: number,
   ): Promise<AppBskyFeedGetTimeline.Response> {
     const res = await this.rootStore.agent.getTimeline({cursor, limit})
-    // filter out mutes pre-emptively to ensure better mixing
-    res.data.feed = res.data.feed.filter(
-      post => !post.post.author.viewer?.muted,
+    // run the tuner pre-emptively to ensure better mixing
+    const slices = this.tuner.tune(
+      res.data.feed,
+      this.rootStore.preferences.getFeedTuners('home'),
+      {
+        dryRun: false,
+        maintainOrder: true,
+      },
     )
+    res.data.feed = slices.map(slice => slice.rootItem)
     return res
   }
 }

src/lib/constants.ts

@@ -83,8 +83,14 @@ export async function DEFAULT_FEEDS(
     // local dev
     const aliceDid = await resolveHandle('alice.test')
     return {
-      pinned: [`at://${aliceDid}/app.bsky.feed.generator/alice-favs`],
-      saved: [`at://${aliceDid}/app.bsky.feed.generator/alice-favs`],
+      pinned: [
+        `at://${aliceDid}/app.bsky.feed.generator/alice-favs`,
+        `at://${aliceDid}/app.bsky.feed.generator/alice-favs2`,
+      ],
+      saved: [
+        `at://${aliceDid}/app.bsky.feed.generator/alice-favs`,
+        `at://${aliceDid}/app.bsky.feed.generator/alice-favs2`,
+      ],
     }
   } else if (IS_STAGING(serviceUrl)) {
     // staging

src/state/models/feeds/posts.ts

@@ -139,53 +139,6 @@ export class PostsFeedModel {
     this.tuner.reset()
   }
 
-  get feedTuners() {
-    const areRepliesEnabled = this.rootStore.preferences.homeFeedRepliesEnabled
-    const areRepliesByFollowedOnlyEnabled =
-      this.rootStore.preferences.homeFeedRepliesByFollowedOnlyEnabled
-    const repliesThreshold = this.rootStore.preferences.homeFeedRepliesThreshold
-    const areRepostsEnabled = this.rootStore.preferences.homeFeedRepostsEnabled
-    const areQuotePostsEnabled =
-      this.rootStore.preferences.homeFeedQuotePostsEnabled
-
-    if (this.feedType === 'custom') {
-      return [
-        FeedTuner.dedupReposts,
-        FeedTuner.preferredLangOnly(
-          this.rootStore.preferences.contentLanguages,
-        ),
-      ]
-    }
-    if (this.feedType === 'home' || this.feedType === 'following') {
-      const feedTuners = []
-
-      if (areRepostsEnabled) {
-        feedTuners.push(FeedTuner.dedupReposts)
-      } else {
-        feedTuners.push(FeedTuner.removeReposts)
-      }
-
-      if (areRepliesEnabled) {
-        feedTuners.push(
-          FeedTuner.thresholdRepliesOnly({
-            userDid: this.rootStore.session.data?.did || '',
-            minLikes: repliesThreshold,
-            followedOnly: areRepliesByFollowedOnlyEnabled,
-          }),
-        )
-      } else {
-        feedTuners.push(FeedTuner.removeReplies)
-      }
-
-      if (!areQuotePostsEnabled) {
-        feedTuners.push(FeedTuner.removeQuotePosts)
-      }
-
-      return feedTuners
-    }
-    return []
-  }
-
   /**
    * Load for first render
    */
@@ -275,9 +228,14 @@ export class PostsFeedModel {
     }
     const post = await this.api.peekLatest()
     if (post) {
-      const slices = this.tuner.tune([post], this.feedTuners, {
-        dryRun: true,
-      })
+      const slices = this.tuner.tune(
+        [post],
+        this.rootStore.preferences.getFeedTuners(this.feedType),
+        {
+          dryRun: true,
+          maintainOrder: true,
+        },
+      )
       if (slices[0]) {
         const sliceModel = new PostsFeedSliceModel(this.rootStore, slices[0])
         if (sliceModel.moderation.content.filter) {
@@ -363,7 +321,10 @@ export class PostsFeedModel {
 
     const slices = this.options.isSimpleFeed
       ? res.feed.map(item => new FeedViewPostsSlice([item]))
-      : this.tuner.tune(res.feed, this.feedTuners)
+      : this.tuner.tune(
+          res.feed,
+          this.rootStore.preferences.getFeedTuners(this.feedType),
+        )
 
     const toAppend: PostsFeedSliceModel[] = []
     for (const slice of slices) {

src/state/models/ui/preferences.ts

@@ -8,6 +8,7 @@ import {ModerationOpts} from '@atproto/api'
 import {DEFAULT_FEEDS} from 'lib/constants'
 import {deviceLocales} from 'platform/detection'
 import {getAge} from 'lib/strings/time'
+import {FeedTuner} from 'lib/api/feed-manip'
 import {LANGUAGES} from '../../../locale/languages'
 
 // TEMP we need to permanently convert 'show' to 'ignore', for now we manually convert -prf
@@ -540,6 +541,52 @@ export class PreferencesModel {
   toggleRequireAltTextEnabled() {
     this.requireAltTextEnabled = !this.requireAltTextEnabled
   }
+
+  getFeedTuners(
+    feedType: 'home' | 'following' | 'author' | 'custom' | 'likes',
+  ) {
+    const areRepliesEnabled = this.homeFeedRepliesEnabled
+    const areRepliesByFollowedOnlyEnabled =
+      this.homeFeedRepliesByFollowedOnlyEnabled
+    const repliesThreshold = this.homeFeedRepliesThreshold
+    const areRepostsEnabled = this.homeFeedRepostsEnabled
+    const areQuotePostsEnabled = this.homeFeedQuotePostsEnabled
+
+    if (feedType === 'custom') {
+      return [
+        FeedTuner.dedupReposts,
+        FeedTuner.preferredLangOnly(this.contentLanguages),
+      ]
+    }
+    if (feedType === 'home' || feedType === 'following') {
+      const feedTuners = []
+
+      if (areRepostsEnabled) {
+        feedTuners.push(FeedTuner.dedupReposts)
+      } else {
+        feedTuners.push(FeedTuner.removeReposts)
+      }
+
+      if (areRepliesEnabled) {
+        feedTuners.push(
+          FeedTuner.thresholdRepliesOnly({
+            userDid: this.rootStore.session.data?.did || '',
+            minLikes: repliesThreshold,
+            followedOnly: areRepliesByFollowedOnlyEnabled,
+          }),
+        )
+      } else {
+        feedTuners.push(FeedTuner.removeReplies)
+      }
+
+      if (!areQuotePostsEnabled) {
+        feedTuners.push(FeedTuner.removeQuotePosts)
+      }
+
+      return feedTuners
+    }
+    return []
+  }
 }
 
 // TEMP we need to permanently convert 'show' to 'ignore', for now we manually convert -prf

src/view/com/modals/ProfilePreview.tsx

@@ -35,7 +35,7 @@ export const Component = observer(function ProfilePreviewImpl({
   }, [model, screen])
 
   return (
-    <View style={[pal.view, s.flex1]}>
+    <View testID="profilePreview" style={[pal.view, s.flex1]}>
       <View
         style={[
           styles.headerWrapper,

src/view/com/pager/FeedsTabBarMobile.tsx

@@ -67,6 +67,7 @@ export const FeedsTabBar = observer(function FeedsTabBarImpl(
         </Text>
         <View style={[pal.view]}>
           <Link
+            testID="viewHeaderHomeFeedPrefsBtn"
             href="/settings/home-feed"
             hitSlop={HITSLOP_10}
             accessibilityRole="button"

src/view/com/posts/FeedItem.tsx

@@ -299,6 +299,7 @@ export const FeedItem = observer(function FeedItemImpl({
             {item.richText?.text ? (
               <View style={styles.postTextContainer}>
                 <RichText
+                  testID="postText"
                   type="post-text"
                   richText={item.richText}
                   lineHeight={1.3}

src/view/com/profile/ProfileHeader.tsx

@@ -556,6 +556,7 @@ const ProfileHeaderLoaded = observer(function ProfileHeaderLoadedImpl({
 
       {!isDesktop && !hideBackButton && (
         <TouchableWithoutFeedback
+          testID="profileHeaderBackBtn"
           onPress={onPressBack}
           hitSlop={BACK_HITSLOP}
           accessibilityRole="button"

src/view/com/search/HeaderWithInput.tsx

@@ -102,6 +102,7 @@ export function HeaderWithInput({
         />
         {query ? (
           <TouchableOpacity
+            testID="searchTextInputClearBtn"
             onPress={onPressClearQuery}
             accessibilityRole="button"
             accessibilityLabel="Clear search query"

src/view/com/testing/TestCtrls.e2e.tsx

@@ -0,0 +1,76 @@
+import React from 'react'
+import {Pressable, View} from 'react-native'
+import {useStores} from 'state/index'
+import {navigate} from '../../../Navigation'
+
+/**
+ * This utility component is only included in the test simulator
+ * build. It gives some quick triggers which help improve the pace
+ * of the tests dramatically.
+ */
+
+const BTN = {height: 1, width: 1, backgroundColor: 'red'}
+
+export function TestCtrls() {
+  const store = useStores()
+  const onPressSignInAlice = async () => {
+    await store.session.login({
+      service: 'http://localhost:3000',
+      identifier: 'alice.test',
+      password: 'hunter2',
+    })
+  }
+  const onPressSignInBob = async () => {
+    await store.session.login({
+      service: 'http://localhost:3000',
+      identifier: 'bob.test',
+      password: 'hunter2',
+    })
+  }
+  return (
+    <View style={{position: 'absolute', top: 100, right: 0, zIndex: 100}}>
+      <Pressable
+        testID="e2eSignInAlice"
+        onPress={onPressSignInAlice}
+        accessibilityRole="button"
+        style={BTN}
+      />
+      <Pressable
+        testID="e2eSignInBob"
+        onPress={onPressSignInBob}
+        accessibilityRole="button"
+        style={BTN}
+      />
+      <Pressable
+        testID="e2eGotoHome"
+        onPress={() => navigate('Home')}
+        accessibilityRole="button"
+        style={BTN}
+      />
+      <Pressable
+        testID="e2eGotoSettings"
+        onPress={() => navigate('Settings')}
+        accessibilityRole="button"
+        style={BTN}
+      />
+      <Pressable
+        testID="e2eGotoModeration"
+        onPress={() => navigate('Moderation')}
+        accessibilityRole="button"
+        style={BTN}
+      />
+      <Pressable
+        testID="e2eToggleMergefeed"
+        onPress={() => store.preferences.toggleHomeFeedMergeFeedEnabled()}
+        accessibilityRole="button"
+        style={BTN}
+      />
+      <Pressable
+        testID="e2eRefreshHome"
+        onPress={() => store.me.mainFeed.refresh()}
+        accessibilityRole="button"
+        style={BTN}
+      />
+    </View>
+  )
+}

src/view/com/testing/TestCtrls.tsx

@@ -0,0 +1,3 @@
+export function TestCtrls() {
+  return null
+}

src/view/com/util/forms/ToggleButton.tsx

@@ -8,6 +8,7 @@ import {colors} from 'lib/styles'
 import {TypographyVariant} from 'lib/ThemeContext'
 
 export function ToggleButton({
+  testID,
   type = 'default-light',
   label,
   isSelected,
@@ -15,6 +16,7 @@ export function ToggleButton({
   labelType,
   onPress,
 }: {
+  testID?: string
   type?: ButtonType
   label: string
   isSelected: boolean
@@ -134,7 +136,7 @@ export function ToggleButton({
     },
   })
   return (
-    <Button type={type} onPress={onPress} style={style}>
+    <Button testID={testID} type={type} onPress={onPress} style={style}>
       <View style={styles.outer}>
         <View style={[circleStyle, styles.circle]}>
           <View

src/view/screens/PreferencesHomeFeed.tsx

@@ -86,6 +86,7 @@ export const PreferencesHomeFeed = observer(function PreferencesHomeFeedImpl({
               Set this setting to "No" to hide all replies from your feed.
             </Text>
             <ToggleButton
+              testID="toggleRepliesBtn"
               type="default-light"
               label={store.preferences.homeFeedRepliesEnabled ? 'Yes' : 'No'}
               isSelected={store.preferences.homeFeedRepliesEnabled}