zio
/
bsky-app
2
0
Fork 0
Code Pull Requests Packages Releases Activity

[APP-718] Improvements and fixes to language handling (#931) 

* Add locale helpers for narrowing languages

* Add a translate link to posts in a different language

* Update language filtering to use narrowing when multiple declared

* Fix a few more RTL layout cases

* Fix types
zio/stable
Paul Frazee 2023-06-30 11:35:29 -05:00 committed by GitHub
parent 48844aa4c7
commit ed5a88d9d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 174 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/lib/api/feed-manip.ts
View File

@ -4,10 +4,7 @@ import {
AppBskyEmbedRecordWithMedia,
AppBskyEmbedRecord,
} from '@atproto/api'
import * as bcp47Match from 'bcp-47-match'
import lande from 'lande'
import {hasProp} from 'lib/type-guards'
import {LANGUAGES_MAP_CODE2} from '../../locale/languages'
import {isPostInLanguage} from '../../locale/helpers'
type FeedViewPost = AppBskyFeedDefs.FeedViewPost
export type FeedTunerFn = (
@ -245,76 +242,29 @@ export class FeedTuner {
* returns an array of `FeedViewPostsSlice` objects.
*/
static preferredLangOnly(preferredLangsCode2: string[]) {
const langsCode3 = preferredLangsCode2.map(
l => LANGUAGES_MAP_CODE2[l]?.code3 || l,
)
return (
tuner: FeedTuner,
slices: FeedViewPostsSlice[],
): FeedViewPostsSlice[] => {
// 1. Early return if no languages have been specified
// early return if no languages have been specified
if (!preferredLangsCode2.length || preferredLangsCode2.length === 0) {
return slices
}
for (let i = slices.length - 1; i >= 0; i--) {
// 2. Set a flag to indicate whether the item has text in a preferred language
let hasPreferredLang = false
for (const item of slices[i].items) {
// 3. check if the post has a `langs` property and if it is in the list of preferred languages
// if it is, set the flag to true
// if language is declared, regardless of a match, break out of the loop
if (
hasProp(item.post.record, 'langs') &&
Array.isArray(item.post.record.langs)
) {
if (
bcp47Match.basicFilter(
item.post.record.langs,
preferredLangsCode2,
).length > 0
) {
hasPreferredLang = true
}
break
}
// 4. FALLBACK if no language declared :
// Get the most likely language of the text in the post from the `lande` library and
// check if it is in the list of preferred languages
// if it is, set the flag to true and break out of the loop
else if (
hasProp(item.post.record, 'text') &&
typeof item.post.record.text === 'string'
) {
// Treat empty text the same as no text
if (item.post.record.text.length === 0) {
hasPreferredLang = true
break
}
const langsProbabilityMap = lande(item.post.record.text)
const mostLikelyLang = langsProbabilityMap[0][0]
// const secondMostLikelyLang = langsProbabilityMap[1][0]
// const thirdMostLikelyLang = langsProbabilityMap[2][0]
// we check for code3 here because that is what the `lande` library returns
if (langsCode3.includes(mostLikelyLang)) {
hasPreferredLang = true
break
}
}
// 5. no text? roll with it (eg: image-only posts, reposts, etc.)
else {
if (isPostInLanguage(item.post, preferredLangsCode2)) {
hasPreferredLang = true
break
}
}
// 6. if item does not fit preferred language, remove it
// if item does not fit preferred language, remove it
if (!hasPreferredLang) {
slices.splice(i, 1)
}
}
// 7. return the filtered list of items
return slices
}
}

81
src/locale/helpers.ts 100644
View File

@ -0,0 +1,81 @@
import {AppBskyFeedDefs, AppBskyFeedPost} from '@atproto/api'
import lande from 'lande'
import {hasProp} from 'lib/type-guards'
import * as bcp47Match from 'bcp-47-match'
import {LANGUAGES_MAP_CODE2, LANGUAGES_MAP_CODE3} from './languages'
export function code2ToCode3(lang: string): string {
if (lang.length === 2) {
return LANGUAGES_MAP_CODE2[lang]?.code3 || lang
}
return lang
}
export function code3ToCode2(lang: string): string {
if (lang.length === 3) {
return LANGUAGES_MAP_CODE3[lang]?.code2 || lang
}
return lang
}
export function getPostLanguage(
post: AppBskyFeedDefs.PostView,
): string | undefined {
let candidates: string[] = []
let postText: string = ''
if (hasProp(post.record, 'text') && typeof post.record.text === 'string') {
postText = post.record.text
}
if (
AppBskyFeedPost.isRecord(post.record) &&
hasProp(post.record, 'langs') &&
Array.isArray(post.record.langs)
) {
candidates = post.record.langs
}
// if there's only one declared language, use that
if (candidates?.length === 1) {
return candidates[0]
}
// no text? can't determine
if (postText.trim().length === 0) {
return undefined
}
// run the language model
let langsProbabilityMap = lande(postText)
// filter down using declared languages
if (candidates?.length) {
langsProbabilityMap = langsProbabilityMap.filter(
([lang, _probability]: [string, number]) => {
return candidates.includes(code3ToCode2(lang))
},
)
}
if (langsProbabilityMap[0]) {
return code3ToCode2(langsProbabilityMap[0][0])
}
}
export function isPostInLanguage(
post: AppBskyFeedDefs.PostView,
targetLangs: string[],
): boolean {
const lang = getPostLanguage(post)
if (!lang) {
// the post has no text, so we just say "yes" for now
return true
}
return bcp47Match.basicFilter(lang, targetLangs).length > 0
}
export function getTranslatorLink(lang: string, text: string): string {
return encodeURI(
`https://translate.google.com/?sl=auto&tl=${lang}&text=${text}`,
)
}

4
src/locale/languages.ts
View File

@ -555,3 +555,7 @@ export const LANGUAGES_MAP_CODE2 = Object.fromEntries(
export const LANGUAGES_MAP_CODE3 = Object.fromEntries(
LANGUAGES.map(lang => [lang.code3, lang]),
)
// some additional manual mappings (not clear if these should be in the "official" mappings)
if (LANGUAGES_MAP_CODE2.fa) {
LANGUAGES_MAP_CODE3.pes = LANGUAGES_MAP_CODE2.fa
}

2
src/view/com/lists/ListCard.tsx
View File

@ -96,7 +96,7 @@ export const ListCard = ({
{descriptionRichText ? (
<View style={styles.details}>
<RichTextCom
style={pal.text}
style={[pal.text, s.flex1]}
numberOfLines={20}
richText={descriptionRichText}
/>

1
src/view/com/lists/ListItems.tsx
View File

@ -347,6 +347,7 @@ const styles = StyleSheet.create({
borderTopWidth: 1,
},
headerDescription: {
flex: 1,
marginTop: 8,
},
headerBtns: {

70
src/view/com/post-thread/PostThreadItem.tsx
View File

@ -2,7 +2,7 @@ import React, {useCallback, useMemo} from 'react'
import {observer} from 'mobx-react-lite'
import {AccessibilityActionEvent, Linking, StyleSheet, View} from 'react-native'
import Clipboard from '@react-native-clipboard/clipboard'
import {AtUri} from '@atproto/api'
import {AtUri, AppBskyFeedDefs} from '@atproto/api'
import {
FontAwesomeIcon,
FontAwesomeIconStyle,
@ -18,6 +18,7 @@ import {s} from 'lib/styles'
import {ago, niceDate} from 'lib/strings/time'
import {sanitizeDisplayName} from 'lib/strings/display-names'
import {pluralize} from 'lib/strings/helpers'
import {getTranslatorLink, isPostInLanguage} from '../../../locale/helpers'
import {useStores} from 'state/index'
import {PostMeta} from '../util/PostMeta'
import {PostEmbeds} from '../util/post-embeds'
@ -65,6 +66,13 @@ export const PostThreadItem = observer(function PostThreadItem({
}, [item.post.uri, item.post.author.handle])
const repostsTitle = 'Reposts of this post'
const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '')
const needsTranslation = useMemo(
() => !isPostInLanguage(item.post, store.preferences.contentLanguages),
[item.post, store.preferences.contentLanguages],
)
const onPressReply = React.useCallback(() => {
store.shell.openComposer({
replyTo: {
@ -98,17 +106,9 @@ export const PostThreadItem = observer(function PostThreadItem({
Toast.show('Copied to clipboard')
}, [record])
const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
const onOpenTranslate = React.useCallback(() => {
Linking.openURL(
encodeURI(
`https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${
record?.text || ''
}`,
),
)
}, [record, primaryLanguage])
Linking.openURL(translatorUrl)
}, [translatorUrl])
const onToggleThreadMute = React.useCallback(async () => {
try {
@ -276,6 +276,7 @@ export const PostThreadItem = observer(function PostThreadItem({
type="post-text-lg"
richText={item.richText}
lineHeight={1.3}
style={s.flex1}
/>
</View>
) : undefined}
@ -283,9 +284,11 @@ export const PostThreadItem = observer(function PostThreadItem({
<PostEmbeds embed={item.post.embed} style={s.mb10} />
</ImageHider>
</ContentHider>
<View style={[s.mt2, s.mb10]}>
<Text style={pal.textLight}>{niceDate(item.post.indexedAt)}</Text>
</View>
<ExpandedPostDetails
post={item.post}
translatorUrl={translatorUrl}
needsTranslation={needsTranslation}
/>
{hasEngagement ? (
<View style={[styles.expandedInfo, pal.border]}>
{item.post.repostCount ? (
@ -411,7 +414,7 @@ export const PostThreadItem = observer(function PostThreadItem({
<RichText
type="post-text"
richText={item.richText}
style={pal.text}
style={[pal.text, s.flex1]}
lineHeight={1.3}
/>
</View>
@ -419,6 +422,15 @@ export const PostThreadItem = observer(function PostThreadItem({
<ImageHider style={s.mb10} moderation={item.moderation.thread}>
<PostEmbeds embed={item.post.embed} style={s.mb10} />
</ImageHider>
{needsTranslation && (
<View style={[pal.borderDark, styles.translateLink]}>
<Link href={translatorUrl} title="Translate">
<Text type="sm" style={pal.link}>
Translate this post
</Text>
</Link>
</View>
)}
</ContentHider>
<PostCtrls
itemUri={itemUri}
@ -473,6 +485,31 @@ export const PostThreadItem = observer(function PostThreadItem({
}
})
function ExpandedPostDetails({
post,
needsTranslation,
translatorUrl,
}: {
post: AppBskyFeedDefs.PostView
needsTranslation: boolean
translatorUrl: string
}) {
const pal = usePalette('default')
return (
<View style={[s.flexRow, s.mt2, s.mb10]}>
<Text style={pal.textLight}>{niceDate(post.indexedAt)}</Text>
{needsTranslation && (
<>
<Text style={pal.textLight}> • </Text>
<Link href={translatorUrl} title="Translate">
<Text style={pal.link}>Translate</Text>
</Link>
</>
)}
</View>
)
}
const styles = StyleSheet.create({
outer: {
borderTopWidth: 1,
@ -540,6 +577,9 @@ const styles = StyleSheet.create({
paddingHorizontal: 0,
paddingBottom: 10,
},
translateLink: {
marginBottom: 6,
},
contentHider: {
marginTop: 4,
},

13
src/view/com/post/Post.tsx
View File

@ -30,6 +30,7 @@ import {UserAvatar} from '../util/UserAvatar'
import {useStores} from 'state/index'
import {s, colors} from 'lib/styles'
import {usePalette} from 'lib/hooks/usePalette'
import {getTranslatorLink} from '../../../locale/helpers'
export const Post = observer(function Post({
uri,
@ -167,16 +168,11 @@ const PostLoaded = observer(
}, [record])
const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '')
const onOpenTranslate = React.useCallback(() => {
Linking.openURL(
encodeURI(
`https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${
record?.text || ''
}`,
),
)
}, [record, primaryLanguage])
Linking.openURL(translatorUrl)
}, [translatorUrl])
const onToggleThreadMute = React.useCallback(async () => {
try {
@ -299,6 +295,7 @@ const PostLoaded = observer(
type="post-text"
richText={item.richText}
lineHeight={1.3}
style={s.flex1}
/>
</View>
) : undefined}

32
src/view/com/posts/FeedItem.tsx
View File

@ -27,6 +27,7 @@ import {useStores} from 'state/index'
import {usePalette} from 'lib/hooks/usePalette'
import {useAnalytics} from 'lib/analytics/analytics'
import {sanitizeDisplayName} from 'lib/strings/display-names'
import {getTranslatorLink, isPostInLanguage} from '../../../locale/helpers'
export const FeedItem = observer(function ({
item,
@ -62,6 +63,12 @@ export const FeedItem = observer(function ({
const urip = new AtUri(record.reply.parent?.uri || record.reply.root.uri)
return urip.hostname
}, [record?.reply])
const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
const translatorUrl = getTranslatorLink(primaryLanguage, record?.text || '')
const needsTranslation = useMemo(
() => !isPostInLanguage(item.post, store.preferences.contentLanguages),
[item.post, store.preferences.contentLanguages],
)
const onPressReply = React.useCallback(() => {
track('FeedItem:PostReply')
@ -98,17 +105,9 @@ export const FeedItem = observer(function ({
Toast.show('Copied to clipboard')
}, [record])
const primaryLanguage = store.preferences.contentLanguages[0] || 'en'
const onOpenTranslate = React.useCallback(() => {
Linking.openURL(
encodeURI(
`https://translate.google.com/?sl=auto&tl=${primaryLanguage}&text=${
record?.text || ''
}`,
),
)
}, [record, primaryLanguage])
Linking.openURL(translatorUrl)
}, [translatorUrl])
const onToggleThreadMute = React.useCallback(async () => {
track('FeedItem:ThreadMute')
@ -301,12 +300,22 @@ export const FeedItem = observer(function ({
type="post-text"
richText={item.richText}
lineHeight={1.3}
style={s.flex1}
/>
</View>
) : undefined}
<ImageHider moderation={item.moderation.list} style={styles.embed}>
<PostEmbeds embed={item.post.embed} style={styles.embed} />
</ImageHider>
{needsTranslation && (
<View style={[pal.borderDark, styles.translateLink]}>
<Link href={translatorUrl} title="Translate">
<Text type="sm" style={pal.link}>
Translate this post
</Text>
</Link>
</View>
)}
</ContentHider>
<PostCtrls
style={styles.ctrls}
@ -402,6 +411,9 @@ const styles = StyleSheet.create({
embed: {
marginBottom: 6,
},
translateLink: {
marginBottom: 6,
},
ctrls: {
marginTop: 4,
},

1
src/view/com/profile/ProfileHeader.tsx
View File

@ -609,6 +609,7 @@ const styles = StyleSheet.create({
},
description: {
flex: 1,
marginBottom: 8,
},