[APP-690] better handling of post languages language filtering (#893)

* add SelectLangBtn

* memoized objects that are created to reduce re-creation on re-render

* add langs when uploading post

* only send the top 3 languages otherwise backend will throw error

* mv ContentLanguagesSettings to folder

* add post languages settings modal and state

* fix typos

* modify feed manip to also check langs label on post

* Fix tests

* Remove log

* Update feed-manip.ts

* Fix syntax errors

* UI tuneups

* Show the currently selected languages in the composer

* fix linting

* Use a bcp-47 matching function

* Fix a duplicate language issue

* Fix web

* Dont include lang in prompt

* Make select language btn an observer

* Keep device languages on top of language selection UIs

* Fix android build settings

* Enforce a max of 3 languages in posts

* Fix tests

* Fix types

---------

Co-authored-by: Paul Frazee <pfrazee@gmail.com>

src/lib/api/feed-manip.ts

@@ -4,6 +4,7 @@ import {
   AppBskyEmbedRecordWithMedia,
   AppBskyEmbedRecord,
 } from '@atproto/api'
+import * as bcp47Match from 'bcp-47-match'
 import lande from 'lande'
 import {hasProp} from 'lib/type-guards'
 import {LANGUAGES_MAP_CODE2} from '../../locale/languages'
@@ -236,44 +237,84 @@ export class FeedTuner {
     }
   }
 
-  static preferredLangOnly(langsCode2: string[]) {
-    const langsCode3 = langsCode2.map(l => LANGUAGES_MAP_CODE2[l]?.code3 || l)
+  /**
+   * This function filters a list of FeedViewPostsSlice items based on whether they contain text in a
+   * preferred language.
+   * @param {string[]} preferredLangsCode2 - An array of prefered language codes in ISO 639-1 or ISO 639-2 format.
+   * @returns A function that takes in a `FeedTuner` and an array of `FeedViewPostsSlice` objects and
+   * returns an array of `FeedViewPostsSlice` objects.
+   */
+  static preferredLangOnly(preferredLangsCode2: string[]) {
+    const langsCode3 = preferredLangsCode2.map(
+      l => LANGUAGES_MAP_CODE2[l]?.code3 || l,
+    )
     return (
       tuner: FeedTuner,
       slices: FeedViewPostsSlice[],
     ): FeedViewPostsSlice[] => {
-      if (!langsCode2.length) {
+      // 1. Early return if no languages have been specified
+      if (!preferredLangsCode2.length || preferredLangsCode2.length === 0) {
         return slices
       }
+
       for (let i = slices.length - 1; i >= 0; i--) {
+        // 2. Set a flag to indicate whether the item has text in a preferred language
         let hasPreferredLang = false
         for (const item of slices[i].items) {
+          // 3. check if the post has a `langs` property and if it is in the list of preferred languages
+          // if it is, set the flag to true
+          // if language is declared, regardless of a match, break out of the loop
           if (
+            hasProp(item.post.record, 'langs') &&
+            Array.isArray(item.post.record.langs)
+          ) {
+            if (
+              bcp47Match.basicFilter(
+                item.post.record.langs,
+                preferredLangsCode2,
+              ).length > 0
+            ) {
+              hasPreferredLang = true
+            }
+            break
+          }
+          // 4. FALLBACK if no language declared :
+          // Get the most likely language of the text in the post from the `lande` library and
+          // check if it is in the list of preferred languages
+          // if it is, set the flag to true and break out of the loop
+          else if (
             hasProp(item.post.record, 'text') &&
             typeof item.post.record.text === 'string'
           ) {
-            // Treat empty text the same as no text.
+            // Treat empty text the same as no text
             if (item.post.record.text.length === 0) {
               hasPreferredLang = true
               break
             }
+            const langsProbabilityMap = lande(item.post.record.text)
+            const mostLikelyLang = langsProbabilityMap[0][0]
+            // const secondMostLikelyLang = langsProbabilityMap[1][0]
+            // const thirdMostLikelyLang = langsProbabilityMap[2][0]
 
-            const res = lande(item.post.record.text)
-
-            if (langsCode3.includes(res[0][0])) {
+            // we check for code3 here because that is what the `lande` library returns
+            if (langsCode3.includes(mostLikelyLang)) {
               hasPreferredLang = true
               break
             }
-          } else {
-            // no text? roll with it
+          }
+          // 5. no text? roll with it (eg: image-only posts, reposts, etc.)
+          else {
             hasPreferredLang = true
             break
           }
         }
+
+        // 6. if item does not fit preferred language, remove it
         if (!hasPreferredLang) {
           slices.splice(i, 1)
         }
       }
+      // 7. return the filtered list of items
       return slices
     }
   }

src/lib/api/index.ts

@@ -65,6 +65,7 @@ interface PostOpts {
   images?: ImageModel[]
   knownHandles?: Set<string>
   onStateChange?: (state: string) => void
+  langs?: string[]
 }
 
 export async function post(store: RootStoreModel, opts: PostOpts) {
@@ -96,6 +97,7 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
     return true
   })
 
+  // add quote embed if present
   if (opts.quote) {
     embed = {
       $type: 'app.bsky.embed.record',
@@ -106,6 +108,7 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
     } as AppBskyEmbedRecord.Main
   }
 
+  // add image embed if present
   if (opts.images?.length) {
     const images: AppBskyEmbedImages.Image[] = []
     for (const image of opts.images) {
@@ -136,6 +139,7 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
     }
   }
 
+  // add external embed if present
   if (opts.extLink && !opts.images?.length) {
     if (opts.extLink.embed) {
       embed = opts.extLink.embed
@@ -197,6 +201,7 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
     }
   }
 
+  // add replyTo if post is a reply to another post
   if (opts.replyTo) {
     const replyToUrip = new AtUri(opts.replyTo)
     const parentPost = await store.agent.getPost({
@@ -215,6 +220,12 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
     }
   }
 
+  // add top 3 languages from user preferences if langs is provided
+  let langs = opts.langs
+  if (opts.langs) {
+    langs = opts.langs.slice(0, 3)
+  }
+
   try {
     opts.onStateChange?.('Posting...')
     return await store.agent.post({
@@ -222,6 +233,7 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
       facets: rt.facets,
       reply,
       embed,
+      langs,
     })
   } catch (e: any) {
     console.error(`Failed to create post: ${e.toString()}`)

src/lib/functions.ts

@@ -4,3 +4,8 @@ export function choose<U, T extends Record<string, U>>(
 ): U {
   return choices[value]
 }
+
+export function dedupArray<T>(arr: T[]): T[] {
+  const s = new Set(arr)
+  return [...s]
+}