[Video] Captions and alt text (#5009)

* video settings modal in composer

* show done button on web

* rm download options

* fix logic for showing settings button

* add language picker (wip)

* subtitle list with language select

* send captions & alt text with video when posting

* style "ensure you have selected a language" text

* include aspect ratio with video

* filter out captions where the lang is not set

* rm log

* fix label and add hint

* minor scrubber fix

src/alf/atoms.ts

@@ -853,6 +853,7 @@ export const atoms = {
   mr_auto: {
     marginRight: 'auto',
   },
+
   /*
    * Pointer events & user select
    */
@@ -871,6 +872,7 @@ export const atoms = {
   user_select_all: {
     userSelect: 'all',
   },
+
   /*
    * Text decoration
    */
@@ -880,4 +882,11 @@ export const atoms = {
   strike_through: {
     textDecorationLine: 'line-through',
   },
+
+  /*
+   * Display
+   */
+  hidden: {
+    display: 'none',
+  },
 } as const

src/lib/api/index.ts

@@ -1,4 +1,5 @@
 import {
+  AppBskyEmbedDefs,
   AppBskyEmbedExternal,
   AppBskyEmbedImages,
   AppBskyEmbedRecord,
@@ -45,7 +46,12 @@ interface PostOpts {
     uri: string
     cid: string
   }
-  video?: BlobRef
+  video?: {
+    blobRef: BlobRef
+    altText: string
+    captions: {lang: string; file: File}[]
+    aspectRatio?: AppBskyEmbedDefs.AspectRatio
+  }
   extLink?: ExternalEmbedDraft
   images?: ImageModel[]
   labels?: string[]
@@ -128,19 +134,35 @@ export async function post(agent: BskyAgent, opts: PostOpts) {
 
   // add video embed if present
   if (opts.video) {
+    const captions = await Promise.all(
+      opts.video.captions
+        .filter(caption => caption.lang !== '')
+        .map(async caption => {
+          const {data} = await agent.uploadBlob(caption.file, {
+            encoding: 'text/vtt',
+          })
+          return {lang: caption.lang, file: data.blob}
+        }),
+    )
     if (opts.quote) {
       embed = {
         $type: 'app.bsky.embed.recordWithMedia',
         record: embed,
         media: {
           $type: 'app.bsky.embed.video',
-          video: opts.video,
+          video: opts.video.blobRef,
+          alt: opts.video.altText || undefined,
+          captions: captions.length === 0 ? undefined : captions,
+          aspectRatio: opts.video.aspectRatio,
         } as AppBskyEmbedVideo.Main,
       } as AppBskyEmbedRecordWithMedia.Main
     } else {
       embed = {
         $type: 'app.bsky.embed.video',
-        video: opts.video,
+        video: opts.video.blobRef,
+        alt: opts.video.altText || undefined,
+        captions: captions.length === 0 ? undefined : captions,
+        aspectRatio: opts.video.aspectRatio,
       } as AppBskyEmbedVideo.Main
     }
   }

src/lib/moderation/useLabelInfo.ts

@@ -1,9 +1,9 @@
 import {
-  ComAtprotoLabelDefs,
   AppBskyLabelerDefs,
-  LABELS,
+  ComAtprotoLabelDefs,
-  interpretLabelValueDefinition,
   InterpretedLabelValueDefinition,
+  interpretLabelValueDefinition,
+  LABELS,
 } from '@atproto/api'
 import {useLingui} from '@lingui/react'
 import * as bcp47Match from 'bcp-47-match'

src/lib/strings/helpers.ts

@@ -1,3 +1,6 @@
+import {useCallback, useMemo} from 'react'
+import Graphemer from 'graphemer'
+
 export function enforceLen(
   str: string,
   len: number,
@@ -23,6 +26,21 @@ export function enforceLen(
   return str
 }
 
+export function useEnforceMaxGraphemeCount() {
+  const splitter = useMemo(() => new Graphemer(), [])
+
+  return useCallback(
+    (text: string, maxCount: number) => {
+      if (splitter.countGraphemes(text) > maxCount) {
+        return splitter.splitGraphemes(text).slice(0, maxCount).join('')
+      } else {
+        return text
+      }
+    },
+    [splitter],
+  )
+}
+
 // https://stackoverflow.com/a/52171480
 export function toHashCode(str: string, seed = 0): number {
   let h1 = 0xdeadbeef ^ seed,

src/state/queries/video/video.ts

@@ -1,4 +1,4 @@
-import React from 'react'
+import React, {useCallback} from 'react'
 import {ImagePickerAsset} from 'expo-image-picker'
 import {AppBskyVideoDefs, BlobRef} from '@atproto/api'
 import {msg} from '@lingui/macro'
@@ -20,6 +20,7 @@ type Action =
   | {type: 'SetError'; error: string | undefined}
   | {type: 'Reset'}
   | {type: 'SetAsset'; asset: ImagePickerAsset}
+  | {type: 'SetDimensions'; width: number; height: number}
   | {type: 'SetVideo'; video: CompressedVideo}
   | {type: 'SetJobStatus'; jobStatus: AppBskyVideoDefs.JobStatus}
   | {type: 'SetBlobRef'; blobRef: BlobRef}
@@ -58,6 +59,13 @@ function reducer(queryClient: QueryClient) {
       }
     } else if (action.type === 'SetAsset') {
       updatedState = {...state, asset: action.asset}
+    } else if (action.type === 'SetDimensions') {
+      updatedState = {
+        ...state,
+        asset: state.asset
+          ? {...state.asset, width: action.width, height: action.height}
+          : undefined,
+      }
     } else if (action.type === 'SetVideo') {
       updatedState = {...state, video: action.video}
     } else if (action.type === 'SetJobStatus') {
@@ -178,11 +186,20 @@ export function useUploadVideo({
     dispatch({type: 'Reset'})
   }
 
+  const updateVideoDimensions = useCallback((width: number, height: number) => {
+    dispatch({
+      type: 'SetDimensions',
+      width,
+      height,
+    })
+  }, [])
+
   return {
     state,
     dispatch,
     selectVideo,
     clearVideo,
+    updateVideoDimensions,
   }
 }
 

src/view/com/composer/Composer.tsx

@@ -108,6 +108,7 @@ import {TextInput, TextInputRef} from './text-input/TextInput'
 import {ThreadgateBtn} from './threadgate/ThreadgateBtn'
 import {useExternalLinkFetch} from './useExternalLinkFetch'
 import {SelectVideoBtn} from './videos/SelectVideoBtn'
+import {SubtitleDialogBtn} from './videos/SubtitleDialog'
 import {VideoPreview} from './videos/VideoPreview'
 import {VideoTranscodeProgress} from './videos/VideoTranscodeProgress'
 
@@ -172,10 +173,14 @@ export const ComposePost = observer(function ComposePost({
     initQuote,
   )
 
+  const [videoAltText, setVideoAltText] = useState('')
+  const [captions, setCaptions] = useState<{lang: string; file: File}[]>([])
+
   const {
     selectVideo,
     clearVideo,
     state: videoUploadState,
+    updateVideoDimensions,
   } = useUploadVideo({
     setStatus: setProcessingState,
     onSuccess: () => {
@@ -347,7 +352,19 @@ export const ComposePost = observer(function ComposePost({
           postgate,
           onStateChange: setProcessingState,
           langs: toPostLanguages(langPrefs.postLanguage),
-          video: videoUploadState.blobRef,
+          video: videoUploadState.blobRef
+            ? {
+                blobRef: videoUploadState.blobRef,
+                altText: videoAltText,
+                captions: captions,
+                aspectRatio: videoUploadState.asset
+                  ? {
+                      width: videoUploadState.asset?.width,
+                      height: videoUploadState.asset?.height,
+                    }
+                  : undefined,
+              }
+            : undefined,
         })
       ).uri
       try {
@@ -694,16 +711,29 @@ export const ComposePost = observer(function ComposePost({
                 )}
               </View>
             ) : null}
-            {videoUploadState.status === 'compressing' &&
+            {videoUploadState.asset &&
-            videoUploadState.asset ? (
+              (videoUploadState.status === 'compressing' ? (
-              <VideoTranscodeProgress
+                <VideoTranscodeProgress
-                asset={videoUploadState.asset}
+                  asset={videoUploadState.asset}
-                progress={videoUploadState.progress}
+                  progress={videoUploadState.progress}
-                clear={clearVideo}
+                  clear={clearVideo}
+                />
+              ) : videoUploadState.video ? (
+                <VideoPreview
+                  asset={videoUploadState.asset}
+                  video={videoUploadState.video}
+                  setDimensions={updateVideoDimensions}
+                  clear={clearVideo}
+                />
+              ) : null)}
+            {(videoUploadState.asset || videoUploadState.video) && (
+              <SubtitleDialogBtn
+                altText={videoAltText}
+                setAltText={setVideoAltText}
+                captions={captions}
+                setCaptions={setCaptions}
               />
-            ) : videoUploadState.video ? (
+            )}
-              <VideoPreview video={videoUploadState.video} clear={clearVideo} />
-            ) : null}
           </View>
         </Animated.ScrollView>
         <SuggestedLanguage text={richtext.text} />

src/view/com/composer/videos/SubtitleDialog.tsx

@@ -0,0 +1,265 @@
+import React, {useCallback} from 'react'
+import {StyleProp, View, ViewStyle} from 'react-native'
+import RNPickerSelect from 'react-native-picker-select'
+import {msg, Trans} from '@lingui/macro'
+import {useLingui} from '@lingui/react'
+
+import {MAX_ALT_TEXT} from '#/lib/constants'
+import {useEnforceMaxGraphemeCount} from '#/lib/strings/helpers'
+import {LANGUAGES} from '#/locale/languages'
+import {isWeb} from '#/platform/detection'
+import {useLanguagePrefs} from '#/state/preferences'
+import {atoms as a, useTheme, web} from '#/alf'
+import {Button, ButtonIcon, ButtonText} from '#/components/Button'
+import * as Dialog from '#/components/Dialog'
+import * as TextField from '#/components/forms/TextField'
+import {CC_Stroke2_Corner0_Rounded as CCIcon} from '#/components/icons/CC'
+import {PageText_Stroke2_Corner0_Rounded as PageTextIcon} from '#/components/icons/PageText'
+import {TimesLarge_Stroke2_Corner0_Rounded as X} from '#/components/icons/Times'
+import {Warning_Stroke2_Corner0_Rounded as WarningIcon} from '#/components/icons/Warning'
+import {Text} from '#/components/Typography'
+import {SubtitleFilePicker} from './SubtitleFilePicker'
+
+interface Props {
+  altText: string
+  captions: {lang: string; file: File}[]
+  setAltText: (altText: string) => void
+  setCaptions: React.Dispatch<
+    React.SetStateAction<{lang: string; file: File}[]>
+  >
+}
+
+export function SubtitleDialogBtn(props: Props) {
+  const control = Dialog.useDialogControl()
+  const {_} = useLingui()
+
+  return (
+    <View style={[a.flex_row, a.mt_xs]}>
+      <Button
+        label={isWeb ? _('Captions & alt text') : _('Alt text')}
+        accessibilityHint={
+          isWeb
+            ? _('Opens captions and alt text dialog')
+            : _('Opens alt text dialog')
+        }
+        size="xsmall"
+        color="secondary"
+        variant="ghost"
+        onPress={control.open}>
+        <ButtonIcon icon={CCIcon} />
+        <ButtonText>
+          {isWeb ? <Trans>Captions & alt text</Trans> : <Trans>Alt text</Trans>}
+        </ButtonText>
+      </Button>
+      <Dialog.Outer control={control}>
+        <Dialog.Handle />
+        <SubtitleDialogInner {...props} />
+      </Dialog.Outer>
+    </View>
+  )
+}
+
+function SubtitleDialogInner({
+  altText,
+  setAltText,
+  captions,
+  setCaptions,
+}: Props) {
+  const control = Dialog.useDialogContext()
+  const {_} = useLingui()
+  const t = useTheme()
+  const enforceLen = useEnforceMaxGraphemeCount()
+  const {primaryLanguage} = useLanguagePrefs()
+
+  const handleSelectFile = useCallback(
+    (file: File) => {
+      setCaptions(subs => [
+        ...subs,
+        {
+          lang: subs.some(s => s.lang === primaryLanguage)
+            ? ''
+            : primaryLanguage,
+          file,
+        },
+      ])
+    },
+    [setCaptions, primaryLanguage],
+  )
+
+  const subtitleMissingLanguage = captions.some(sub => sub.lang === '')
+
+  return (
+    <Dialog.ScrollableInner label={_(msg`Video settings`)}>
+      <View style={a.gap_md}>
+        <Text style={[a.text_xl, a.font_bold, a.leading_tight]}>
+          <Trans>Alt text</Trans>
+        </Text>
+        <TextField.Root>
+          <Dialog.Input
+            label={_(msg`Alt text`)}
+            placeholder={_(msg`Add alt text (optional)`)}
+            value={altText}
+            onChangeText={evt => setAltText(enforceLen(evt, MAX_ALT_TEXT))}
+            maxLength={MAX_ALT_TEXT * 10}
+            multiline
+            numberOfLines={3}
+            onKeyPress={({nativeEvent}) => {
+              if (nativeEvent.key === 'Escape') {
+                control.close()
+              }
+            }}
+          />
+        </TextField.Root>
+
+        {isWeb && (
+          <>
+            <View
+              style={[
+                a.border_t,
+                a.w_full,
+                t.atoms.border_contrast_medium,
+                a.my_md,
+              ]}
+            />
+            <Text style={[a.text_xl, a.font_bold, a.leading_tight]}>
+              <Trans>Captions (.vtt)</Trans>
+            </Text>
+            <SubtitleFilePicker
+              onSelectFile={handleSelectFile}
+              disabled={subtitleMissingLanguage || captions.length >= 4}
+            />
+            <View>
+              {captions.map((subtitle, i) => (
+                <SubtitleFileRow
+                  key={subtitle.lang}
+                  language={subtitle.lang}
+                  file={subtitle.file}
+                  setCaptions={setCaptions}
+                  otherLanguages={LANGUAGES.filter(
+                    lang =>
+                      langCode(lang) === subtitle.lang ||
+                      !captions.some(s => s.lang === langCode(lang)),
+                  )}
+                  style={[i % 2 === 0 && t.atoms.bg_contrast_25]}
+                />
+              ))}
+            </View>
+          </>
+        )}
+
+        {subtitleMissingLanguage && (
+          <Text style={[a.text_sm, t.atoms.text_contrast_medium]}>
+            Ensure you have selected a language for each subtitle file.
+          </Text>
+        )}
+
+        <View style={web([a.flex_row, a.justify_end])}>
+          <Button
+            label={_(msg`Done`)}
+            size={isWeb ? 'small' : 'medium'}
+            color="primary"
+            variant="solid"
+            onPress={() => control.close()}
+            style={a.mt_lg}>
+            <ButtonText>
+              <Trans>Done</Trans>
+            </ButtonText>
+          </Button>
+        </View>
+      </View>
+      <Dialog.Close />
+    </Dialog.ScrollableInner>
+  )
+}
+
+function SubtitleFileRow({
+  language,
+  file,
+  otherLanguages,
+  setCaptions,
+  style,
+}: {
+  language: string
+  file: File
+  otherLanguages: {code2: string; code3: string; name: string}[]
+  setCaptions: React.Dispatch<
+    React.SetStateAction<{lang: string; file: File}[]>
+  >
+  style: StyleProp<ViewStyle>
+}) {
+  const {_} = useLingui()
+  const t = useTheme()
+
+  const handleValueChange = useCallback(
+    (lang: string) => {
+      if (lang) {
+        setCaptions(subs =>
+          subs.map(s => (s.lang === language ? {lang, file: s.file} : s)),
+        )
+      }
+    },
+    [setCaptions, language],
+  )
+
+  return (
+    <View
+      style={[
+        a.flex_row,
+        a.justify_between,
+        a.py_md,
+        a.px_lg,
+        a.rounded_md,
+        a.gap_md,
+        style,
+      ]}>
+      <View style={[a.flex_1, a.gap_xs, a.justify_center]}>
+        <View style={[a.flex_row, a.align_center, a.gap_sm]}>
+          {language === '' ? (
+            <WarningIcon
+              style={a.flex_shrink_0}
+              fill={t.palette.negative_500}
+              size="sm"
+            />
+          ) : (
+            <PageTextIcon style={[t.atoms.text, a.flex_shrink_0]} size="sm" />
+          )}
+          <Text
+            style={[a.flex_1, a.leading_snug, a.font_bold, a.mb_2xs]}
+            numberOfLines={1}>
+            {file.name}
+          </Text>
+          <RNPickerSelect
+            placeholder={{
+              label: _(msg`Select language...`),
+              value: '',
+            }}
+            value={language}
+            onValueChange={handleValueChange}
+            items={otherLanguages.map(lang => ({
+              label: `${lang.name} (${langCode(lang)})`,
+              value: langCode(lang),
+            }))}
+            style={{viewContainer: {maxWidth: 200, flex: 1}}}
+          />
+        </View>
+      </View>
+
+      <Button
+        label={_(msg`Remove subtitle file`)}
+        size="tiny"
+        shape="round"
+        variant="outline"
+        color="secondary"
+        onPress={() =>
+          setCaptions(subs => subs.filter(s => s.lang !== language))
+        }
+        style={[a.ml_sm]}>
+        <ButtonIcon icon={X} />
+      </Button>
+    </View>
+  )
+}
+
+function langCode(lang: {code2: string; code3: string}) {
+  return lang.code2 || lang.code3
+}

src/view/com/composer/videos/SubtitleFilePicker.native.tsx

@@ -0,0 +1,3 @@
+export function SubtitleFilePicker() {
+  throw new Error('SubtitleFilePicker is a web-only component')
+}

src/view/com/composer/videos/SubtitleFilePicker.tsx

@@ -0,0 +1,63 @@
+import React, {useRef} from 'react'
+import {View} from 'react-native'
+import {msg, Trans} from '@lingui/macro'
+import {useLingui} from '@lingui/react'
+
+import * as Toast from '#/view/com/util/Toast'
+import {atoms as a} from '#/alf'
+import {Button, ButtonIcon, ButtonText} from '#/components/Button'
+import {CC_Stroke2_Corner0_Rounded as CCIcon} from '#/components/icons/CC'
+
+export function SubtitleFilePicker({
+  onSelectFile,
+  disabled,
+}: {
+  onSelectFile: (file: File) => void
+  disabled?: boolean
+}) {
+  const {_} = useLingui()
+  const ref = useRef<HTMLInputElement>(null)
+
+  const handleClick = () => {
+    ref.current?.click()
+  }
+
+  const handlePick = (evt: React.ChangeEvent<HTMLInputElement>) => {
+    const selectedFile = evt.target.files?.[0]
+    if (selectedFile) {
+      if (selectedFile.type === 'text/vtt') {
+        onSelectFile(selectedFile)
+      } else {
+        Toast.show(_(msg`Only WebVTT (.vtt) files are supported`))
+      }
+    }
+  }
+
+  return (
+    <View style={a.gap_lg}>
+      <input
+        type="file"
+        accept=".vtt"
+        ref={ref}
+        style={a.hidden}
+        onChange={handlePick}
+        disabled={disabled}
+        aria-disabled={disabled}
+      />
+      <View style={a.flex_row}>
+        <Button
+          onPress={handleClick}
+          label={_('Select subtitle file (.vtt)')}
+          size="medium"
+          color="primary"
+          variant="solid"
+          disabled={disabled}>
+          <ButtonIcon icon={CCIcon} />
+          <ButtonText>
+            <Trans>Select subtitle file (.vtt)</Trans>
+          </ButtonText>
+        </Button>
+      </View>
+    </View>
+  )
+}

src/view/com/composer/videos/VideoPreview.tsx

@@ -1,32 +1,41 @@
 /* eslint-disable @typescript-eslint/no-shadow */
 import React from 'react'
 import {View} from 'react-native'
+import {ImagePickerAsset} from 'expo-image-picker'
 import {useVideoPlayer, VideoView} from 'expo-video'
 
 import {CompressedVideo} from '#/lib/media/video/compress'
 import {ExternalEmbedRemoveBtn} from 'view/com/composer/ExternalEmbedRemoveBtn'
-import {atoms as a} from '#/alf'
+import {atoms as a, useTheme} from '#/alf'
 
 export function VideoPreview({
+  asset,
   video,
   clear,
 }: {
+  asset: ImagePickerAsset
   video: CompressedVideo
+  setDimensions: (width: number, height: number) => void
   clear: () => void
 }) {
+  const t = useTheme()
   const player = useVideoPlayer(video.uri, player => {
     player.loop = true
     player.muted = true
     player.play()
   })
 
+  const aspectRatio = asset.width / asset.height
+
   return (
     <View
       style={[
         a.w_full,
         a.rounded_sm,
-        {aspectRatio: 16 / 9},
+        {aspectRatio: isNaN(aspectRatio) ? 16 / 9 : aspectRatio},
         a.overflow_hidden,
+        a.border,
+        t.atoms.border_contrast_low,
       ]}>
       <VideoView
         player={player}

src/view/com/composer/videos/VideoPreview.web.tsx

@@ -1,27 +1,65 @@
-import React from 'react'
+import React, {useEffect, useRef} from 'react'
 import {View} from 'react-native'
+import {ImagePickerAsset} from 'expo-image-picker'
 
 import {CompressedVideo} from '#/lib/media/video/compress'
 import {ExternalEmbedRemoveBtn} from 'view/com/composer/ExternalEmbedRemoveBtn'
-import {atoms as a} from '#/alf'
+import {atoms as a, useTheme} from '#/alf'
 
 export function VideoPreview({
+  asset,
   video,
+  setDimensions,
   clear,
 }: {
+  asset: ImagePickerAsset
   video: CompressedVideo
+  setDimensions: (width: number, height: number) => void
   clear: () => void
 }) {
+  const t = useTheme()
+  const ref = useRef<HTMLVideoElement>(null)
+
+  useEffect(() => {
+    if (!ref.current) return
+
+    const abortController = new AbortController()
+    const {signal} = abortController
+    ref.current.addEventListener(
+      'loadedmetadata',
+      function () {
+        setDimensions(this.videoWidth, this.videoHeight)
+      },
+      {signal},
+    )
+
+    return () => {
+      abortController.abort()
+    }
+  }, [setDimensions])
+
+  const aspectRatio = asset.width / asset.height
+
   return (
     <View
       style={[
         a.w_full,
         a.rounded_sm,
-        {aspectRatio: 16 / 9},
+
+        {aspectRatio: isNaN(aspectRatio) ? 16 / 9 : aspectRatio},
         a.overflow_hidden,
+        {backgroundColor: t.palette.black},
       ]}>
       <ExternalEmbedRemoveBtn onRemove={clear} />
-      <video src={video.uri} style={a.flex_1} autoPlay loop muted playsInline />
+      <video
+        ref={ref}
+        src={video.uri}
+        style={a.flex_1}
+        autoPlay
+        loop
+        muted
+        playsInline
+      />
     </View>
   )
 }

src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx

@@ -1,7 +1,3 @@
-import React from 'react'
+export function VideoTranscodeBackdrop() {
-
+  return null
-export function VideoTranscodeBackdrop({uri}: {uri: string}) {
-  return (
-    <video src={uri} style={{flex: 1, filter: 'blur(10px)'}} muted autoPlay />
-  )
 }

src/view/com/composer/videos/VideoTranscodeProgress.tsx

@@ -4,6 +4,7 @@ import {View} from 'react-native'
 import ProgressPie from 'react-native-progress/Pie'
 import {ImagePickerAsset} from 'expo-image-picker'
 
+import {isWeb} from '#/platform/detection'
 import {atoms as a, useTheme} from '#/alf'
 import {ExternalEmbedRemoveBtn} from '../ExternalEmbedRemoveBtn'
 import {VideoTranscodeBackdrop} from './VideoTranscodeBackdrop'
@@ -21,6 +22,8 @@ export function VideoTranscodeProgress({
 
   const aspectRatio = asset.width / asset.height
 
+  if (isWeb) return null
+
   return (
     <View
       style={[

src/view/com/util/post-embeds/VideoEmbedInner/VideoWebControls.tsx

@@ -557,7 +557,7 @@ function Scrubber({
             {backgroundColor: 'rgba(255, 255, 255, 0.4)'},
             {height: hovered || scrubberActive ? 6 : 3},
           ]}>
-          {currentTime > 0 && duration > 0 && (
+          {duration > 0 && (
             <View
               style={[
                 a.h_full,