[APP-718] Improvements and fixes to language handling (#931)

* Add locale helpers for narrowing languages

* Add a translate link to posts in a different language

* Update language filtering to use narrowing when multiple declared

* Fix a few more RTL layout cases

* Fix types

src/lib/api/feed-manip.ts

@@ -4,10 +4,7 @@ import {
   AppBskyEmbedRecordWithMedia,
   AppBskyEmbedRecord,
 } from '@atproto/api'
-import * as bcp47Match from 'bcp-47-match'
+import {isPostInLanguage} from '../../locale/helpers'
-import lande from 'lande'
-import {hasProp} from 'lib/type-guards'
-import {LANGUAGES_MAP_CODE2} from '../../locale/languages'
 type FeedViewPost = AppBskyFeedDefs.FeedViewPost
 
 export type FeedTunerFn = (
@@ -245,76 +242,29 @@ export class FeedTuner {
    * returns an array of `FeedViewPostsSlice` objects.
    */
   static preferredLangOnly(preferredLangsCode2: string[]) {
-    const langsCode3 = preferredLangsCode2.map(
-      l => LANGUAGES_MAP_CODE2[l]?.code3 || l,
-    )
     return (
       tuner: FeedTuner,
       slices: FeedViewPostsSlice[],
     ): FeedViewPostsSlice[] => {
-      // 1. Early return if no languages have been specified
+      // early return if no languages have been specified
       if (!preferredLangsCode2.length || preferredLangsCode2.length === 0) {
         return slices
       }
 
       for (let i = slices.length - 1; i >= 0; i--) {
-        // 2. Set a flag to indicate whether the item has text in a preferred language
         let hasPreferredLang = false
         for (const item of slices[i].items) {
-          // 3. check if the post has a `langs` property and if it is in the list of preferred languages
+          if (isPostInLanguage(item.post, preferredLangsCode2)) {
-          // if it is, set the flag to true
-          // if language is declared, regardless of a match, break out of the loop
-          if (
-            hasProp(item.post.record, 'langs') &&
-            Array.isArray(item.post.record.langs)
-          ) {
-            if (
-              bcp47Match.basicFilter(
-                item.post.record.langs,
-                preferredLangsCode2,
-              ).length > 0
-            ) {
-              hasPreferredLang = true
-            }
-            break
-          }
-          // 4. FALLBACK if no language declared :
-          // Get the most likely language of the text in the post from the `lande` library and
-          // check if it is in the list of preferred languages
-          // if it is, set the flag to true and break out of the loop
-          else if (
-            hasProp(item.post.record, 'text') &&
-            typeof item.post.record.text === 'string'
-          ) {
-            // Treat empty text the same as no text
-            if (item.post.record.text.length === 0) {
-              hasPreferredLang = true
-              break
-            }
-            const langsProbabilityMap = lande(item.post.record.text)
-            const mostLikelyLang = langsProbabilityMap[0][0]
-            // const secondMostLikelyLang = langsProbabilityMap[1][0]
-            // const thirdMostLikelyLang = langsProbabilityMap[2][0]
-
-            // we check for code3 here because that is what the `lande` library returns
-            if (langsCode3.includes(mostLikelyLang)) {
-              hasPreferredLang = true
-              break
-            }
-          }
-          // 5. no text? roll with it (eg: image-only posts, reposts, etc.)
-          else {
             hasPreferredLang = true
             break
           }
         }
 
-        // 6. if item does not fit preferred language, remove it
+        // if item does not fit preferred language, remove it
         if (!hasPreferredLang) {
           slices.splice(i, 1)
         }
       }
-      // 7. return the filtered list of items
       return slices
     }
   }

src/locale/helpers.ts

@@ -0,0 +1,81 @@
+import {AppBskyFeedDefs, AppBskyFeedPost} from '@atproto/api'
+import lande from 'lande'
+import {hasProp} from 'lib/type-guards'
+import * as bcp47Match from 'bcp-47-match'
+import {LANGUAGES_MAP_CODE2, LANGUAGES_MAP_CODE3} from './languages'
+
+export function code2ToCode3(lang: string): string {
+  if (lang.length === 2) {
+    return LANGUAGES_MAP_CODE2[lang]?.code3 || lang
+  }
+  return lang
+}
+
+export function code3ToCode2(lang: string): string {
+  if (lang.length === 3) {
+    return LANGUAGES_MAP_CODE3[lang]?.code2 || lang
+  }
+  return lang
+}
+
+export function getPostLanguage(
+  post: AppBskyFeedDefs.PostView,
+): string | undefined {
+  let candidates: string[] = []
+  let postText: string = ''
+  if (hasProp(post.record, 'text') && typeof post.record.text === 'string') {
+    postText = post.record.text
+  }
+
+  if (
+    AppBskyFeedPost.isRecord(post.record) &&
+    hasProp(post.record, 'langs') &&
+    Array.isArray(post.record.langs)
+  ) {
+    candidates = post.record.langs
+  }
+
+  // if there's only one declared language, use that
+  if (candidates?.length === 1) {
+    return candidates[0]
+  }
+
+  // no text? can't determine
+  if (postText.trim().length === 0) {
+    return undefined
+  }
+
+  // run the language model
+  let langsProbabilityMap = lande(postText)
+
+  // filter down using declared languages
+  if (candidates?.length) {
+    langsProbabilityMap = langsProbabilityMap.filter(
+      ([lang, _probability]: [string, number]) => {
+        return candidates.includes(code3ToCode2(lang))
+      },
+    )
+  }
+
+  if (langsProbabilityMap[0]) {
+    return code3ToCode2(langsProbabilityMap[0][0])
+  }
+}
+
+export function isPostInLanguage(
+  post: AppBskyFeedDefs.PostView,
+  targetLangs: string[],
+): boolean {
+  const lang = getPostLanguage(post)
+  if (!lang) {
+    // the post has no text, so we just say "yes" for now
+    return true
+  }
+  return bcp47Match.basicFilter(lang, targetLangs).length > 0
+}
+
+export function getTranslatorLink(lang: string, text: string): string {
+  return encodeURI(
+    `https://translate.google.com/?sl=auto&tl=${lang}&text=${text}`,
+  )
+}

src/locale/languages.ts

@@ -555,3 +555,7 @@ export const LANGUAGES_MAP_CODE2 = Object.fromEntries(
 export const LANGUAGES_MAP_CODE3 = Object.fromEntries(
   LANGUAGES.map(lang => [lang.code3, lang]),
 )
+// some additional manual mappings (not clear if these should be in the "official" mappings)
+if (LANGUAGES_MAP_CODE2.fa) {
+  LANGUAGES_MAP_CODE3.pes = LANGUAGES_MAP_CODE2.fa
+}

src/view/com/lists/ListCard.tsx

@@ -96,7 +96,7 @@ export const ListCard = ({
       {descriptionRichText ? (
         <View style={styles.details}>
           <RichTextCom
-            style={pal.text}
+            style={[pal.text, s.flex1]}
             numberOfLines={20}
             richText={descriptionRichText}
           />

src/view/com/lists/ListItems.tsx

@@ -347,6 +347,7 @@ const styles = StyleSheet.create({
     borderTopWidth: 1,
   },
   headerDescription: {
+    flex: 1,
     marginTop: 8,
   },
   headerBtns: {

src/view/com/post-thread/PostThreadItem.tsx

@@ -2,7 +2,7 @@ import React, {useCallback, useMemo} from 'react'
 import {observer} from 'mobx-react-lite'
 import {AccessibilityActionEvent, Linking, StyleSheet, View} from 'react-native'
 import Clipboard from '@react-native-clipboard/clipboard'
-import {AtUri} from '@atproto/api'
+import {AtUri, AppBskyFeedDefs} from '@atproto/api'
 import {
   FontAwesomeIcon,
   FontAwesomeIconStyle,
@@ -18,6 +18,7 @@ import {s} from 'lib/styles'
 import {ago, niceDate} from 'lib/strings/time'
 import {sanitizeDisplayName} from 'lib/strings/display-names'
 import {pluralize} from 'lib/strings/helpers'
+import {getTranslatorLink, isPostInLanguage} from '../../../locale/helpers'
 import {useStores} from 'state/index'
 import {PostMeta} from '../util/PostMeta'
 import {PostEmbeds} from '../util/post-embeds'
@@ -65,6 +66,13 @@ export const PostThreadItem = observer(function PostThreadItem({
   }, [item.post.uri, item.post.author.handle])
   const repostsTitle = 'Reposts of this post'
 
+  const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
+  const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '')
+  const needsTranslation = useMemo(
+    () => !isPostInLanguage(item.post, store.preferences.contentLanguages),
+    [item.post, store.preferences.contentLanguages],
+  )
+
   const onPressReply = React.useCallback(() => {
     store.shell.openComposer({
       replyTo: {
@@ -98,17 +106,9 @@ export const PostThreadItem = observer(function PostThreadItem({
     Toast.show('Copied to clipboard')
   }, [record])
 
-  const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
-
   const onOpenTranslate = React.useCallback(() => {
-    Linking.openURL(
+    Linking.openURL(translatorUrl)
-      encodeURI(
+  }, [translatorUrl])
-        `https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${
-          record?.text || ''
-        }`,
-      ),
-    )
-  }, [record, primaryLanguage])
 
   const onToggleThreadMute = React.useCallback(async () => {
     try {
@@ -276,6 +276,7 @@ export const PostThreadItem = observer(function PostThreadItem({
                   type="post-text-lg"
                   richText={item.richText}
                   lineHeight={1.3}
+                  style={s.flex1}
                 />
               </View>
             ) : undefined}
@@ -283,9 +284,11 @@ export const PostThreadItem = observer(function PostThreadItem({
               <PostEmbeds embed={item.post.embed} style={s.mb10} />
             </ImageHider>
           </ContentHider>
-          <View style={[s.mt2, s.mb10]}>
+          <ExpandedPostDetails
-            <Text style={pal.textLight}>{niceDate(item.post.indexedAt)}</Text>
+            post={item.post}
-          </View>
+            translatorUrl={translatorUrl}
+            needsTranslation={needsTranslation}
+          />
           {hasEngagement ? (
             <View style={[styles.expandedInfo, pal.border]}>
               {item.post.repostCount ? (
@@ -411,7 +414,7 @@ export const PostThreadItem = observer(function PostThreadItem({
                     <RichText
                       type="post-text"
                       richText={item.richText}
-                      style={pal.text}
+                      style={[pal.text, s.flex1]}
                       lineHeight={1.3}
                     />
                   </View>
@@ -419,6 +422,15 @@ export const PostThreadItem = observer(function PostThreadItem({
                 <ImageHider style={s.mb10} moderation={item.moderation.thread}>
                   <PostEmbeds embed={item.post.embed} style={s.mb10} />
                 </ImageHider>
+                {needsTranslation && (
+                  <View style={[pal.borderDark, styles.translateLink]}>
+                    <Link href={translatorUrl} title="Translate">
+                      <Text type="sm" style={pal.link}>
+                        Translate this post
+                      </Text>
+                    </Link>
+                  </View>
+                )}
               </ContentHider>
               <PostCtrls
                 itemUri={itemUri}
@@ -473,6 +485,31 @@ export const PostThreadItem = observer(function PostThreadItem({
   }
 })
 
+function ExpandedPostDetails({
+  post,
+  needsTranslation,
+  translatorUrl,
+}: {
+  post: AppBskyFeedDefs.PostView
+  needsTranslation: boolean
+  translatorUrl: string
+}) {
+  const pal = usePalette('default')
+  return (
+    <View style={[s.flexRow, s.mt2, s.mb10]}>
+      <Text style={pal.textLight}>{niceDate(post.indexedAt)}</Text>
+      {needsTranslation && (
+        <>
+          <Text style={pal.textLight}> • </Text>
+          <Link href={translatorUrl} title="Translate">
+            <Text style={pal.link}>Translate</Text>
+          </Link>
+        </>
+      )}
+    </View>
+  )
+}
+
 const styles = StyleSheet.create({
   outer: {
     borderTopWidth: 1,
@@ -540,6 +577,9 @@ const styles = StyleSheet.create({
     paddingHorizontal: 0,
     paddingBottom: 10,
   },
+  translateLink: {
+    marginBottom: 6,
+  },
   contentHider: {
     marginTop: 4,
   },

src/view/com/post/Post.tsx

@@ -30,6 +30,7 @@ import {UserAvatar} from '../util/UserAvatar'
 import {useStores} from 'state/index'
 import {s, colors} from 'lib/styles'
 import {usePalette} from 'lib/hooks/usePalette'
+import {getTranslatorLink} from '../../../locale/helpers'
 
 export const Post = observer(function Post({
   uri,
@@ -167,16 +168,11 @@ const PostLoaded = observer(
     }, [record])
 
     const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
+    const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '')
 
     const onOpenTranslate = React.useCallback(() => {
-      Linking.openURL(
+      Linking.openURL(translatorUrl)
-        encodeURI(
+    }, [translatorUrl])
-          `https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${
-            record?.text || ''
-          }`,
-        ),
-      )
-    }, [record, primaryLanguage])
 
     const onToggleThreadMute = React.useCallback(async () => {
       try {
@@ -299,6 +295,7 @@ const PostLoaded = observer(
                     type="post-text"
                     richText={item.richText}
                     lineHeight={1.3}
+                    style={s.flex1}
                   />
                 </View>
               ) : undefined}

src/view/com/posts/FeedItem.tsx

@@ -27,6 +27,7 @@ import {useStores} from 'state/index'
 import {usePalette} from 'lib/hooks/usePalette'
 import {useAnalytics} from 'lib/analytics/analytics'
 import {sanitizeDisplayName} from 'lib/strings/display-names'
+import {getTranslatorLink, isPostInLanguage} from '../../../locale/helpers'
 
 export const FeedItem = observer(function ({
   item,
@@ -62,6 +63,12 @@ export const FeedItem = observer(function ({
     const urip = new AtUri(record.reply.parent?.uri || record.reply.root.uri)
     return urip.hostname
   }, [record?.reply])
+  const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
+  const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '')
+  const needsTranslation = useMemo(
+    () => !isPostInLanguage(item.post, store.preferences.contentLanguages),
+    [item.post, store.preferences.contentLanguages],
+  )
 
   const onPressReply = React.useCallback(() => {
     track('FeedItem:PostReply')
@@ -98,17 +105,9 @@ export const FeedItem = observer(function ({
     Toast.show('Copied to clipboard')
   }, [record])
 
-  const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
-
   const onOpenTranslate = React.useCallback(() => {
-    Linking.openURL(
+    Linking.openURL(translatorUrl)
-      encodeURI(
+  }, [translatorUrl])
-        `https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${
-          record?.text || ''
-        }`,
-      ),
-    )
-  }, [record, primaryLanguage])
 
   const onToggleThreadMute = React.useCallback(async () => {
     track('FeedItem:ThreadMute')
@@ -301,12 +300,22 @@ export const FeedItem = observer(function ({
                   type="post-text"
                   richText={item.richText}
                   lineHeight={1.3}
+                  style={s.flex1}
                 />
               </View>
             ) : undefined}
             <ImageHider moderation={item.moderation.list} style={styles.embed}>
               <PostEmbeds embed={item.post.embed} style={styles.embed} />
             </ImageHider>
+            {needsTranslation && (
+              <View style={[pal.borderDark, styles.translateLink]}>
+                <Link href={translatorUrl} title="Translate">
+                  <Text type="sm" style={pal.link}>
+                    Translate this post
+                  </Text>
+                </Link>
+              </View>
+            )}
           </ContentHider>
           <PostCtrls
             style={styles.ctrls}
@@ -402,6 +411,9 @@ const styles = StyleSheet.create({
   embed: {
     marginBottom: 6,
   },
+  translateLink: {
+    marginBottom: 6,
+  },
   ctrls: {
     marginTop: 4,
   },

src/view/com/profile/ProfileHeader.tsx

@@ -609,6 +609,7 @@ const styles = StyleSheet.create({
   },
 
   description: {
+    flex: 1,
     marginBottom: 8,
   },