zio
/
bsky-app
2
0
Fork 0
Code Pull Requests Packages Releases Activity

[Video] Captions and alt text (#5009) 

* video settings modal in composer

* show done button on web

* rm download options

* fix logic for showing settings button

* add language picker (wip)

* subtitle list with language select

* send captions & alt text with video when posting

* style "ensure you have selected a language" text

* include aspect ratio with video

* filter out captions where the lang is not set

* rm log

* fix label and add hint

* minor scrubber fix
zio/stable
Samuel Newman 2024-08-30 18:45:49 +01:00 committed by GitHub
parent e7954e590b
commit c70ec1ce1a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 503 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/alf/atoms.ts
View File

@ -853,6 +853,7 @@ export const atoms = {
mr_auto: {
marginRight: 'auto',
},
/*
* Pointer events & user select
*/
@ -871,6 +872,7 @@ export const atoms = {
user_select_all: {
userSelect: 'all',
},
/*
* Text decoration
*/
@ -880,4 +882,11 @@ export const atoms = {
strike_through: {
textDecorationLine: 'line-through',
},
/*
* Display
*/
hidden: {
display: 'none',
},
} as const

28
src/lib/api/index.ts
View File

@ -1,4 +1,5 @@
import {
AppBskyEmbedDefs,
AppBskyEmbedExternal,
AppBskyEmbedImages,
AppBskyEmbedRecord,
@ -45,7 +46,12 @@ interface PostOpts {
uri: string
cid: string
}
video?: BlobRef
video?: {
blobRef: BlobRef
altText: string
captions: {lang: string; file: File}[]
aspectRatio?: AppBskyEmbedDefs.AspectRatio
}
extLink?: ExternalEmbedDraft
images?: ImageModel[]
labels?: string[]
@ -128,19 +134,35 @@ export async function post(agent: BskyAgent, opts: PostOpts) {
// add video embed if present
if (opts.video) {
const captions = await Promise.all(
opts.video.captions
.filter(caption => caption.lang !== '')
.map(async caption => {
const {data} = await agent.uploadBlob(caption.file, {
encoding: 'text/vtt',
})
return {lang: caption.lang, file: data.blob}
}),
)
if (opts.quote) {
embed = {
$type: 'app.bsky.embed.recordWithMedia',
record: embed,
media: {
$type: 'app.bsky.embed.video',
video: opts.video,
video: opts.video.blobRef,
alt: opts.video.altText || undefined,
captions: captions.length === 0 ? undefined : captions,
aspectRatio: opts.video.aspectRatio,
} as AppBskyEmbedVideo.Main,
} as AppBskyEmbedRecordWithMedia.Main
} else {
embed = {
$type: 'app.bsky.embed.video',
video: opts.video,
video: opts.video.blobRef,
alt: opts.video.altText || undefined,
captions: captions.length === 0 ? undefined : captions,
aspectRatio: opts.video.aspectRatio,
} as AppBskyEmbedVideo.Main
}
}

6
src/lib/moderation/useLabelInfo.ts
View File

@ -1,9 +1,9 @@
import {
ComAtprotoLabelDefs,
AppBskyLabelerDefs,
LABELS,
interpretLabelValueDefinition,
ComAtprotoLabelDefs,
InterpretedLabelValueDefinition,
interpretLabelValueDefinition,
LABELS,
} from '@atproto/api'
import {useLingui} from '@lingui/react'
import * as bcp47Match from 'bcp-47-match'

18
src/lib/strings/helpers.ts
View File

@ -1,3 +1,6 @@
import {useCallback, useMemo} from 'react'
import Graphemer from 'graphemer'
export function enforceLen(
str: string,
len: number,
@ -23,6 +26,21 @@ export function enforceLen(
return str
}
export function useEnforceMaxGraphemeCount() {
const splitter = useMemo(() => new Graphemer(), [])
return useCallback(
(text: string, maxCount: number) => {
if (splitter.countGraphemes(text) > maxCount) {
return splitter.splitGraphemes(text).slice(0, maxCount).join('')
} else {
return text
}
},
[splitter],
)
}
// https://stackoverflow.com/a/52171480
export function toHashCode(str: string, seed = 0): number {
let h1 = 0xdeadbeef ^ seed,

19
src/state/queries/video/video.ts
View File

@ -1,4 +1,4 @@
import React from 'react'
import React, {useCallback} from 'react'
import {ImagePickerAsset} from 'expo-image-picker'
import {AppBskyVideoDefs, BlobRef} from '@atproto/api'
import {msg} from '@lingui/macro'
@ -20,6 +20,7 @@ type Action =
| {type: 'SetError'; error: string | undefined}
| {type: 'Reset'}
| {type: 'SetAsset'; asset: ImagePickerAsset}
| {type: 'SetDimensions'; width: number; height: number}
| {type: 'SetVideo'; video: CompressedVideo}
| {type: 'SetJobStatus'; jobStatus: AppBskyVideoDefs.JobStatus}
| {type: 'SetBlobRef'; blobRef: BlobRef}
@ -58,6 +59,13 @@ function reducer(queryClient: QueryClient) {
}
} else if (action.type === 'SetAsset') {
updatedState = {...state, asset: action.asset}
} else if (action.type === 'SetDimensions') {
updatedState = {
...state,
asset: state.asset
? {...state.asset, width: action.width, height: action.height}
: undefined,
}
} else if (action.type === 'SetVideo') {
updatedState = {...state, video: action.video}
} else if (action.type === 'SetJobStatus') {
@ -178,11 +186,20 @@ export function useUploadVideo({
dispatch({type: 'Reset'})
}
const updateVideoDimensions = useCallback((width: number, height: number) => {
dispatch({
type: 'SetDimensions',
width,
height,
})
}, [])
return {
state,
dispatch,
selectVideo,
clearVideo,
updateVideoDimensions,
}
}

40
src/view/com/composer/Composer.tsx
View File

@ -108,6 +108,7 @@ import {TextInput, TextInputRef} from './text-input/TextInput'
import {ThreadgateBtn} from './threadgate/ThreadgateBtn'
import {useExternalLinkFetch} from './useExternalLinkFetch'
import {SelectVideoBtn} from './videos/SelectVideoBtn'
import {SubtitleDialogBtn} from './videos/SubtitleDialog'
import {VideoPreview} from './videos/VideoPreview'
import {VideoTranscodeProgress} from './videos/VideoTranscodeProgress'
@ -172,10 +173,14 @@ export const ComposePost = observer(function ComposePost({
initQuote,
)
const [videoAltText, setVideoAltText] = useState('')
const [captions, setCaptions] = useState<{lang: string; file: File}[]>([])
const {
selectVideo,
clearVideo,
state: videoUploadState,
updateVideoDimensions,
} = useUploadVideo({
setStatus: setProcessingState,
onSuccess: () => {
@ -347,7 +352,19 @@ export const ComposePost = observer(function ComposePost({
postgate,
onStateChange: setProcessingState,
langs: toPostLanguages(langPrefs.postLanguage),
video: videoUploadState.blobRef,
video: videoUploadState.blobRef
? {
blobRef: videoUploadState.blobRef,
altText: videoAltText,
captions: captions,
aspectRatio: videoUploadState.asset
? {
width: videoUploadState.asset?.width,
height: videoUploadState.asset?.height,
}
: undefined,
}
: undefined,
})
).uri
try {
@ -694,16 +711,29 @@ export const ComposePost = observer(function ComposePost({
)}
</View>
) : null}
{videoUploadState.status === 'compressing' &&
videoUploadState.asset ? (
{videoUploadState.asset &&
(videoUploadState.status === 'compressing' ? (
<VideoTranscodeProgress
asset={videoUploadState.asset}
progress={videoUploadState.progress}
clear={clearVideo}
/>
) : videoUploadState.video ? (
<VideoPreview video={videoUploadState.video} clear={clearVideo} />
) : null}
<VideoPreview
asset={videoUploadState.asset}
video={videoUploadState.video}
setDimensions={updateVideoDimensions}
clear={clearVideo}
/>
) : null)}
{(videoUploadState.asset || videoUploadState.video) && (
<SubtitleDialogBtn
altText={videoAltText}
setAltText={setVideoAltText}
captions={captions}
setCaptions={setCaptions}
/>
)}
</View>
</Animated.ScrollView>
<SuggestedLanguage text={richtext.text} />

265
src/view/com/composer/videos/SubtitleDialog.tsx 100644
View File

@ -0,0 +1,265 @@
import React, {useCallback} from 'react'
import {StyleProp, View, ViewStyle} from 'react-native'
import RNPickerSelect from 'react-native-picker-select'
import {msg, Trans} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import {MAX_ALT_TEXT} from '#/lib/constants'
import {useEnforceMaxGraphemeCount} from '#/lib/strings/helpers'
import {LANGUAGES} from '#/locale/languages'
import {isWeb} from '#/platform/detection'
import {useLanguagePrefs} from '#/state/preferences'
import {atoms as a, useTheme, web} from '#/alf'
import {Button, ButtonIcon, ButtonText} from '#/components/Button'
import * as Dialog from '#/components/Dialog'
import * as TextField from '#/components/forms/TextField'
import {CC_Stroke2_Corner0_Rounded as CCIcon} from '#/components/icons/CC'
import {PageText_Stroke2_Corner0_Rounded as PageTextIcon} from '#/components/icons/PageText'
import {TimesLarge_Stroke2_Corner0_Rounded as X} from '#/components/icons/Times'
import {Warning_Stroke2_Corner0_Rounded as WarningIcon} from '#/components/icons/Warning'
import {Text} from '#/components/Typography'
import {SubtitleFilePicker} from './SubtitleFilePicker'
interface Props {
altText: string
captions: {lang: string; file: File}[]
setAltText: (altText: string) => void
setCaptions: React.Dispatch<
React.SetStateAction<{lang: string; file: File}[]>
>
}
export function SubtitleDialogBtn(props: Props) {
const control = Dialog.useDialogControl()
const {_} = useLingui()
return (
<View style={[a.flex_row, a.mt_xs]}>
<Button
label={isWeb ? _('Captions & alt text') : _('Alt text')}
accessibilityHint={
isWeb
? _('Opens captions and alt text dialog')
: _('Opens alt text dialog')
}
size="xsmall"
color="secondary"
variant="ghost"
onPress={control.open}>
<ButtonIcon icon={CCIcon} />
<ButtonText>
{isWeb ? <Trans>Captions & alt text</Trans> : <Trans>Alt text</Trans>}
</ButtonText>
</Button>
<Dialog.Outer control={control}>
<Dialog.Handle />
<SubtitleDialogInner {...props} />
</Dialog.Outer>
</View>
)
}
function SubtitleDialogInner({
altText,
setAltText,
captions,
setCaptions,
}: Props) {
const control = Dialog.useDialogContext()
const {_} = useLingui()
const t = useTheme()
const enforceLen = useEnforceMaxGraphemeCount()
const {primaryLanguage} = useLanguagePrefs()
const handleSelectFile = useCallback(
(file: File) => {
setCaptions(subs => [
...subs,
{
lang: subs.some(s => s.lang === primaryLanguage)
? ''
: primaryLanguage,
file,
},
])
},
[setCaptions, primaryLanguage],
)
const subtitleMissingLanguage = captions.some(sub => sub.lang === '')
return (
<Dialog.ScrollableInner label={_(msg`Video settings`)}>
<View style={a.gap_md}>
<Text style={[a.text_xl, a.font_bold, a.leading_tight]}>
<Trans>Alt text</Trans>
</Text>
<TextField.Root>
<Dialog.Input
label={_(msg`Alt text`)}
placeholder={_(msg`Add alt text (optional)`)}
value={altText}
onChangeText={evt => setAltText(enforceLen(evt, MAX_ALT_TEXT))}
maxLength={MAX_ALT_TEXT * 10}
multiline
numberOfLines={3}
onKeyPress={({nativeEvent}) => {
if (nativeEvent.key === 'Escape') {
control.close()
}
}}
/>
</TextField.Root>
{isWeb && (
<>
<View
style={[
a.border_t,
a.w_full,
t.atoms.border_contrast_medium,
a.my_md,
]}
/>
<Text style={[a.text_xl, a.font_bold, a.leading_tight]}>
<Trans>Captions (.vtt)</Trans>
</Text>
<SubtitleFilePicker
onSelectFile={handleSelectFile}
disabled={subtitleMissingLanguage || captions.length >= 4}
/>
<View>
{captions.map((subtitle, i) => (
<SubtitleFileRow
key={subtitle.lang}
language={subtitle.lang}
file={subtitle.file}
setCaptions={setCaptions}
otherLanguages={LANGUAGES.filter(
lang =>
langCode(lang) === subtitle.lang ||
!captions.some(s => s.lang === langCode(lang)),
)}
style={[i % 2 === 0 && t.atoms.bg_contrast_25]}
/>
))}
</View>
</>
)}
{subtitleMissingLanguage && (
<Text style={[a.text_sm, t.atoms.text_contrast_medium]}>
Ensure you have selected a language for each subtitle file.
</Text>
)}
<View style={web([a.flex_row, a.justify_end])}>
<Button
label={_(msg`Done`)}
size={isWeb ? 'small' : 'medium'}
color="primary"
variant="solid"
onPress={() => control.close()}
style={a.mt_lg}>
<ButtonText>
<Trans>Done</Trans>
</ButtonText>
</Button>
</View>
</View>
<Dialog.Close />
</Dialog.ScrollableInner>
)
}
function SubtitleFileRow({
language,
file,
otherLanguages,
setCaptions,
style,
}: {
language: string
file: File
otherLanguages: {code2: string; code3: string; name: string}[]
setCaptions: React.Dispatch<
React.SetStateAction<{lang: string; file: File}[]>
>
style: StyleProp<ViewStyle>
}) {
const {_} = useLingui()
const t = useTheme()
const handleValueChange = useCallback(
(lang: string) => {
if (lang) {
setCaptions(subs =>
subs.map(s => (s.lang === language ? {lang, file: s.file} : s)),
)
}
},
[setCaptions, language],
)
return (
<View
style={[
a.flex_row,
a.justify_between,
a.py_md,
a.px_lg,
a.rounded_md,
a.gap_md,
style,
]}>
<View style={[a.flex_1, a.gap_xs, a.justify_center]}>
<View style={[a.flex_row, a.align_center, a.gap_sm]}>
{language === '' ? (
<WarningIcon
style={a.flex_shrink_0}
fill={t.palette.negative_500}
size="sm"
/>
) : (
<PageTextIcon style={[t.atoms.text, a.flex_shrink_0]} size="sm" />
)}
<Text
style={[a.flex_1, a.leading_snug, a.font_bold, a.mb_2xs]}
numberOfLines={1}>
{file.name}
</Text>
<RNPickerSelect
placeholder={{
label: _(msg`Select language...`),
value: '',
}}
value={language}
onValueChange={handleValueChange}
items={otherLanguages.map(lang => ({
label: `${lang.name} (${langCode(lang)})`,
value: langCode(lang),
}))}
style={{viewContainer: {maxWidth: 200, flex: 1}}}
/>
</View>
</View>
<Button
label={_(msg`Remove subtitle file`)}
size="tiny"
shape="round"
variant="outline"
color="secondary"
onPress={() =>
setCaptions(subs => subs.filter(s => s.lang !== language))
}
style={[a.ml_sm]}>
<ButtonIcon icon={X} />
</Button>
</View>
)
}
function langCode(lang: {code2: string; code3: string}) {
return lang.code2 || lang.code3
}

3
src/view/com/composer/videos/SubtitleFilePicker.native.tsx 100644
View File

@ -0,0 +1,3 @@
export function SubtitleFilePicker() {
throw new Error('SubtitleFilePicker is a web-only component')
}

63
src/view/com/composer/videos/SubtitleFilePicker.tsx 100644
View File

@ -0,0 +1,63 @@
import React, {useRef} from 'react'
import {View} from 'react-native'
import {msg, Trans} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import * as Toast from '#/view/com/util/Toast'
import {atoms as a} from '#/alf'
import {Button, ButtonIcon, ButtonText} from '#/components/Button'
import {CC_Stroke2_Corner0_Rounded as CCIcon} from '#/components/icons/CC'
export function SubtitleFilePicker({
onSelectFile,
disabled,
}: {
onSelectFile: (file: File) => void
disabled?: boolean
}) {
const {_} = useLingui()
const ref = useRef<HTMLInputElement>(null)
const handleClick = () => {
ref.current?.click()
}
const handlePick = (evt: React.ChangeEvent<HTMLInputElement>) => {
const selectedFile = evt.target.files?.[0]
if (selectedFile) {
if (selectedFile.type === 'text/vtt') {
onSelectFile(selectedFile)
} else {
Toast.show(_(msg`Only WebVTT (.vtt) files are supported`))
}
}
}
return (
<View style={a.gap_lg}>
<input
type="file"
accept=".vtt"
ref={ref}
style={a.hidden}
onChange={handlePick}
disabled={disabled}
aria-disabled={disabled}
/>
<View style={a.flex_row}>
<Button
onPress={handleClick}
label={_('Select subtitle file (.vtt)')}
size="medium"
color="primary"
variant="solid"
disabled={disabled}>
<ButtonIcon icon={CCIcon} />
<ButtonText>
<Trans>Select subtitle file (.vtt)</Trans>
</ButtonText>
</Button>
</View>
</View>
)
}

13
src/view/com/composer/videos/VideoPreview.tsx
View File

@ -1,32 +1,41 @@
/* eslint-disable @typescript-eslint/no-shadow */
import React from 'react'
import {View} from 'react-native'
import {ImagePickerAsset} from 'expo-image-picker'
import {useVideoPlayer, VideoView} from 'expo-video'
import {CompressedVideo} from '#/lib/media/video/compress'
import {ExternalEmbedRemoveBtn} from 'view/com/composer/ExternalEmbedRemoveBtn'
import {atoms as a} from '#/alf'
import {atoms as a, useTheme} from '#/alf'
export function VideoPreview({
asset,
video,
clear,
}: {
asset: ImagePickerAsset
video: CompressedVideo
setDimensions: (width: number, height: number) => void
clear: () => void
}) {
const t = useTheme()
const player = useVideoPlayer(video.uri, player => {
player.loop = true
player.muted = true
player.play()
})
const aspectRatio = asset.width / asset.height
return (
<View
style={[
a.w_full,
a.rounded_sm,
{aspectRatio: 16 / 9},
{aspectRatio: isNaN(aspectRatio) ? 16 / 9 : aspectRatio},
a.overflow_hidden,
a.border,
t.atoms.border_contrast_low,
]}>
<VideoView
player={player}

46
src/view/com/composer/videos/VideoPreview.web.tsx
View File

@ -1,27 +1,65 @@
import React from 'react'
import React, {useEffect, useRef} from 'react'
import {View} from 'react-native'
import {ImagePickerAsset} from 'expo-image-picker'
import {CompressedVideo} from '#/lib/media/video/compress'
import {ExternalEmbedRemoveBtn} from 'view/com/composer/ExternalEmbedRemoveBtn'
import {atoms as a} from '#/alf'
import {atoms as a, useTheme} from '#/alf'
export function VideoPreview({
asset,
video,
setDimensions,
clear,
}: {
asset: ImagePickerAsset
video: CompressedVideo
setDimensions: (width: number, height: number) => void
clear: () => void
}) {
const t = useTheme()
const ref = useRef<HTMLVideoElement>(null)
useEffect(() => {
if (!ref.current) return
const abortController = new AbortController()
const {signal} = abortController
ref.current.addEventListener(
'loadedmetadata',
function () {
setDimensions(this.videoWidth, this.videoHeight)
},
{signal},
)
return () => {
abortController.abort()
}
}, [setDimensions])
const aspectRatio = asset.width / asset.height
return (
<View
style={[
a.w_full,
a.rounded_sm,
{aspectRatio: 16 / 9},
{aspectRatio: isNaN(aspectRatio) ? 16 / 9 : aspectRatio},
a.overflow_hidden,
{backgroundColor: t.palette.black},
]}>
<ExternalEmbedRemoveBtn onRemove={clear} />
<video src={video.uri} style={a.flex_1} autoPlay loop muted playsInline />
<video
ref={ref}
src={video.uri}
style={a.flex_1}
autoPlay
loop
muted
playsInline
/>
</View>
)
}

8
src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx
View File

@ -1,7 +1,3 @@
import React from 'react'
export function VideoTranscodeBackdrop({uri}: {uri: string}) {
return (
<video src={uri} style={{flex: 1, filter: 'blur(10px)'}} muted autoPlay />
)
export function VideoTranscodeBackdrop() {
return null
}

3
src/view/com/composer/videos/VideoTranscodeProgress.tsx
View File

@ -4,6 +4,7 @@ import {View} from 'react-native'
import ProgressPie from 'react-native-progress/Pie'
import {ImagePickerAsset} from 'expo-image-picker'
import {isWeb} from '#/platform/detection'
import {atoms as a, useTheme} from '#/alf'
import {ExternalEmbedRemoveBtn} from '../ExternalEmbedRemoveBtn'
import {VideoTranscodeBackdrop} from './VideoTranscodeBackdrop'
@ -21,6 +22,8 @@ export function VideoTranscodeProgress({
const aspectRatio = asset.width / asset.height
if (isWeb) return null
return (
<View
style={[

2
src/view/com/util/post-embeds/VideoEmbedInner/VideoWebControls.tsx
View File

@ -557,7 +557,7 @@ function Scrubber({
{backgroundColor: 'rgba(255, 255, 255, 0.4)'},
{height: hovered || scrubberActive ? 6 : 3},
]}>
{currentTime > 0 && duration > 0 && (
{duration > 0 && (
<View
style={[
a.h_full,