[Video] Lexicon implementation (#4881)

* implement AppBskyEmbedVideo lexicon in player

* add alt to native player

* add prerelease package

* update prerelease

* add video embed view manually from record

* fix type error on example video

* black bg + use aspect ratio on web

* add video to feeds

* fix video overflowing aspect ratio

* remove prerelease package

---------

Co-authored-by: Samuel Newman <10959775+mozzius@users.noreply.github.com>

src/view/com/post-thread/PostThread.tsx

@@ -428,6 +428,7 @@ export function PostThread({uri}: {uri: string | undefined}) {
         (item.ctx.depth < 0 && !!item.parent) || item.ctx.depth > 1
       const hasUnrevealedParents =
         index === 0 && skeleton?.parents && maxParents < skeleton.parents.length
+
       return (
         <View
           ref={item.ctx.isHighlightedPost ? highlightedPostRef : undefined}

src/view/com/posts/FeedItem.tsx

@@ -17,37 +17,37 @@ import {msg, Trans} from '@lingui/macro'
 import {useLingui} from '@lingui/react'
 import {useQueryClient} from '@tanstack/react-query'
 
+import {isReasonFeedSource, ReasonFeedSource} from '#/lib/api/feed/types'
+import {MAX_POST_LINES} from '#/lib/constants'
+import {usePalette} from '#/lib/hooks/usePalette'
+import {makeProfileLink} from '#/lib/routes/links'
 import {useGate} from '#/lib/statsig/statsig'
+import {sanitizeDisplayName} from '#/lib/strings/display-names'
+import {sanitizeHandle} from '#/lib/strings/handles'
+import {countLines} from '#/lib/strings/helpers'
+import {s} from '#/lib/styles'
 import {POST_TOMBSTONE, Shadow, usePostShadow} from '#/state/cache/post-shadow'
 import {useFeedFeedbackContext} from '#/state/feed-feedback'
+import {precacheProfile} from '#/state/queries/profile'
 import {useSession} from '#/state/session'
 import {useComposerControls} from '#/state/shell/composer'
 import {useMergedThreadgateHiddenReplies} from '#/state/threadgate-hidden-replies'
-import {isReasonFeedSource, ReasonFeedSource} from 'lib/api/feed/types'
-import {MAX_POST_LINES} from 'lib/constants'
-import {usePalette} from 'lib/hooks/usePalette'
-import {makeProfileLink} from 'lib/routes/links'
-import {sanitizeDisplayName} from 'lib/strings/display-names'
-import {sanitizeHandle} from 'lib/strings/handles'
-import {countLines} from 'lib/strings/helpers'
-import {s} from 'lib/styles'
-import {precacheProfile} from 'state/queries/profile'
+import {FeedNameText} from '#/view/com/util/FeedInfoText'
+import {PostCtrls} from '#/view/com/util/post-ctrls/PostCtrls'
+import {PostEmbeds} from '#/view/com/util/post-embeds'
+import {PostMeta} from '#/view/com/util/PostMeta'
+import {Text} from '#/view/com/util/text/Text'
+import {PreviewableUserAvatar} from '#/view/com/util/UserAvatar'
 import {atoms as a} from '#/alf'
 import {Repost_Stroke2_Corner2_Rounded as Repost} from '#/components/icons/Repost'
 import {ContentHider} from '#/components/moderation/ContentHider'
+import {LabelsOnMyPost} from '#/components/moderation/LabelsOnMe'
+import {PostAlerts} from '#/components/moderation/PostAlerts'
 import {AppModerationCause} from '#/components/Pills'
 import {ProfileHoverCard} from '#/components/ProfileHoverCard'
 import {RichText} from '#/components/RichText'
-import {LabelsOnMyPost} from '../../../components/moderation/LabelsOnMe'
-import {PostAlerts} from '../../../components/moderation/PostAlerts'
-import {FeedNameText} from '../util/FeedInfoText'
 import {Link, TextLink, TextLinkOnWebOnly} from '../util/Link'
-import {PostCtrls} from '../util/post-ctrls/PostCtrls'
-import {PostEmbeds} from '../util/post-embeds'
 import {VideoEmbed} from '../util/post-embeds/VideoEmbed'
-import {PostMeta} from '../util/PostMeta'
-import {Text} from '../util/text/Text'
-import {PreviewableUserAvatar} from '../util/UserAvatar'
 import {AviFollowButton} from './AviFollowButton'
 
 interface FeedItemProps {
@@ -571,7 +571,11 @@ function VideoDebug() {
 
   return (
     <VideoEmbed
-      source={`https://lumi.jazco.dev/watch/did:plc:q6gjnaw2blty4crticxkmujt/Qmc8w93UpTa2adJHg4ZhnDPrBs1EsbzrekzPcqF5SwusuZ/playlist.m3u8?ignore_me_just_testing_frontend_stuff=${id}`}
+      embed={{
+        playlist: `https://lumi.jazco.dev/watch/did:plc:q6gjnaw2blty4crticxkmujt/Qmc8w93UpTa2adJHg4ZhnDPrBs1EsbzrekzPcqF5SwusuZ/playlist.m3u8?ignore_me_just_testing_frontend_stuff=${id}`,
+        cid: 'Qmc8w93UpTa2adJHg4ZhnDPrBs1EsbzrekzPcqF5SwusuZ',
+        aspectRatio: {height: 9, width: 16},
+      }}
     />
   )
 }

src/view/com/util/post-embeds/VideoEmbed.tsx

@@ -1,21 +1,25 @@
 import React, {useCallback, useState} from 'react'
 import {View} from 'react-native'
+import {Image} from 'expo-image'
+import {AppBskyEmbedVideo} from '@atproto/api'
 import {msg, Trans} from '@lingui/macro'
 import {useLingui} from '@lingui/react'
 
-import {VideoEmbedInnerNative} from 'view/com/util/post-embeds/VideoEmbedInner/VideoEmbedInnerNative'
+import {clamp} from '#/lib/numbers'
+import {useGate} from '#/lib/statsig/statsig'
+import {VideoEmbedInnerNative} from '#/view/com/util/post-embeds/VideoEmbedInner/VideoEmbedInnerNative'
 import {atoms as a, useTheme} from '#/alf'
-import {Button, ButtonIcon} from '#/components/Button'
+import {Button} from '#/components/Button'
 import {Play_Filled_Corner2_Rounded as PlayIcon} from '#/components/icons/Play'
 import {VisibilityView} from '../../../../../modules/expo-bluesky-swiss-army'
 import {ErrorBoundary} from '../ErrorBoundary'
 import {useActiveVideoNative} from './ActiveVideoNativeContext'
 import * as VideoFallback from './VideoEmbedInner/VideoFallback'
 
-export function VideoEmbed({source}: {source: string}) {
+export function VideoEmbed({embed}: {embed: AppBskyEmbedVideo.View}) {
   const t = useTheme()
   const {activeSource, setActiveSource} = useActiveVideoNative()
-  const isActive = source === activeSource
+  const isActive = embed.playlist === activeSource
   const {_} = useLingui()
 
   const [key, setKey] = useState(0)
@@ -25,39 +29,61 @@ export function VideoEmbed({source}: {source: string}) {
     ),
     [key],
   )
+  const gate = useGate()
+
+  if (!gate('videos')) {
+    return null
+  }
+
+  let aspectRatio = 16 / 9
+
+  if (embed.aspectRatio) {
+    const {width, height} = embed.aspectRatio
+    aspectRatio = width / height
+    aspectRatio = clamp(aspectRatio, 1 / 1, 3 / 1)
+  }
 
   return (
     <View
       style={[
         a.w_full,
         a.rounded_sm,
-        {aspectRatio: 16 / 9},
         a.overflow_hidden,
-        t.atoms.bg_contrast_25,
+        {aspectRatio},
+        {backgroundColor: t.palette.black},
         a.my_xs,
       ]}>
       <ErrorBoundary renderError={renderError} key={key}>
         <VisibilityView
           enabled={true}
-          onChangeStatus={isActive => {
-            if (isActive) {
-              setActiveSource(source)
+          onChangeStatus={isVisible => {
+            if (isVisible) {
+              setActiveSource(embed.playlist)
             }
           }}>
           {isActive ? (
-            <VideoEmbedInnerNative />
+            <VideoEmbedInnerNative embed={embed} />
           ) : (
-            <Button
-              style={[a.flex_1, t.atoms.bg_contrast_25]}
-              onPress={() => {
-                setActiveSource(source)
-              }}
-              label={_(msg`Play video`)}
-              variant="ghost"
-              color="secondary"
-              size="large">
-              <ButtonIcon icon={PlayIcon} />
-            </Button>
+            <>
+              <Image
+                source={{uri: embed.thumbnail}}
+                alt={embed.alt}
+                style={a.flex_1}
+                contentFit="contain"
+                accessibilityIgnoresInvertColors
+              />
+              <Button
+                style={[a.absolute, a.inset_0]}
+                onPress={() => {
+                  setActiveSource(embed.playlist)
+                }}
+                label={_(msg`Play video`)}
+                variant="ghost"
+                color="secondary"
+                size="large">
+                <PlayIcon width={48} fill={t.palette.white} />
+              </Button>
+            </>
           )}
         </VisibilityView>
       </ErrorBoundary>

src/view/com/util/post-embeds/VideoEmbed.web.tsx

@@ -1,19 +1,23 @@
 import React, {useCallback, useEffect, useRef, useState} from 'react'
 import {View} from 'react-native'
+import {AppBskyEmbedVideo} from '@atproto/api'
 import {Trans} from '@lingui/macro'
 
+import {clamp} from '#/lib/numbers'
+import {useGate} from '#/lib/statsig/statsig'
 import {
   HLSUnsupportedError,
   VideoEmbedInnerWeb,
-} from 'view/com/util/post-embeds/VideoEmbedInner/VideoEmbedInnerWeb'
+} from '#/view/com/util/post-embeds/VideoEmbedInner/VideoEmbedInnerWeb'
 import {atoms as a, useTheme} from '#/alf'
 import {ErrorBoundary} from '../ErrorBoundary'
 import {useActiveVideoWeb} from './ActiveVideoWebContext'
 import * as VideoFallback from './VideoEmbedInner/VideoFallback'
 
-export function VideoEmbed({source}: {source: string}) {
+export function VideoEmbed({embed}: {embed: AppBskyEmbedVideo.View}) {
   const t = useTheme()
   const ref = useRef<HTMLDivElement>(null)
+  const gate = useGate()
   const {active, setActive, sendPosition, currentActiveView} =
     useActiveVideoWeb()
   const [onScreen, setOnScreen] = useState(false)
@@ -43,12 +47,25 @@ export function VideoEmbed({source}: {source: string}) {
     [key],
   )
 
+  if (!gate('videos')) {
+    return null
+  }
+
+  let aspectRatio = 16 / 9
+
+  if (embed.aspectRatio) {
+    const {width, height} = embed.aspectRatio
+    // min: 3/1, max: square
+    aspectRatio = clamp(width / height, 1 / 1, 3 / 1)
+  }
+
   return (
     <View
       style={[
         a.w_full,
-        {aspectRatio: 16 / 9},
-        t.atoms.bg_contrast_25,
+        {aspectRatio},
+        {backgroundColor: t.palette.black},
+        a.relative,
         a.rounded_sm,
         a.my_xs,
       ]}>
@@ -61,7 +78,7 @@ export function VideoEmbed({source}: {source: string}) {
             sendPosition={sendPosition}
             isAnyViewActive={currentActiveView !== null}>
             <VideoEmbedInnerWeb
-              source={source}
+              embed={embed}
               active={active}
               setActive={setActive}
               onScreen={onScreen}

src/view/com/util/post-embeds/VideoEmbedInner/VideoEmbedInnerNative.tsx

@@ -2,12 +2,14 @@ import React, {useCallback, useEffect, useRef, useState} from 'react'
 import {Pressable, View} from 'react-native'
 import Animated, {FadeInDown} from 'react-native-reanimated'
 import {VideoPlayer, VideoView} from 'expo-video'
+import {AppBskyEmbedVideo} from '@atproto/api'
 import {msg} from '@lingui/macro'
 import {useLingui} from '@lingui/react'
 import {useIsFocused} from '@react-navigation/native'
 
 import {HITSLOP_30} from '#/lib/constants'
 import {useAppState} from '#/lib/hooks/useAppState'
+import {clamp} from '#/lib/numbers'
 import {logger} from '#/logger'
 import {useActiveVideoNative} from 'view/com/util/post-embeds/ActiveVideoNativeContext'
 import {atoms as a, useTheme} from '#/alf'
@@ -19,7 +21,12 @@ import {
 } from '../../../../../../modules/expo-bluesky-swiss-army'
 import {TimeIndicator} from './TimeIndicator'
 
-export function VideoEmbedInnerNative() {
+export function VideoEmbedInnerNative({
+  embed,
+}: {
+  embed: AppBskyEmbedVideo.View
+}) {
+  const {_} = useLingui()
   const {player} = useActiveVideoNative()
   const ref = useRef<VideoView>(null)
   const isScreenFocused = useIsFocused()
@@ -47,13 +54,23 @@ export function VideoEmbedInnerNative() {
     ref.current?.enterFullscreen()
   }, [])
 
+  let aspectRatio = 16 / 9
+
+  if (embed.aspectRatio) {
+    const {width, height} = embed.aspectRatio
+    aspectRatio = width / height
+    aspectRatio = clamp(aspectRatio, 1 / 1, 3 / 1)
+  }
+
   return (
-    <View style={[a.flex_1, a.relative]}>
+    <View style={[a.flex_1, a.relative, {aspectRatio}]}>
       <VideoView
         ref={ref}
         player={player}
         style={[a.flex_1, a.rounded_sm]}
+        contentFit="contain"
         nativeControls={true}
+        accessibilityIgnoresInvertColors
         onEnterFullscreen={() => {
           PlatformInfo.setAudioCategory(AudioCategory.Playback)
           PlatformInfo.setAudioActive(true)
@@ -65,13 +82,17 @@ export function VideoEmbedInnerNative() {
           player.muted = true
           if (!player.playing) player.play()
         }}
+        accessibilityLabel={
+          embed.alt ? _(msg`Video: ${embed.alt}`) : _(msg`Video`)
+        }
+        accessibilityHint=""
       />
-      <Controls player={player} enterFullscreen={enterFullscreen} />
+      <VideoControls player={player} enterFullscreen={enterFullscreen} />
     </View>
   )
 }
 
-function Controls({
+function VideoControls({
   player,
   enterFullscreen,
 }: {

src/view/com/util/post-embeds/VideoEmbedInner/VideoEmbedInnerWeb.tsx

@@ -1,31 +1,27 @@
-import React, {useEffect, useRef, useState} from 'react'
+import React, {useEffect, useId, useRef, useState} from 'react'
 import {View} from 'react-native'
+import {AppBskyEmbedVideo} from '@atproto/api'
 import Hls from 'hls.js'
 
 import {atoms as a} from '#/alf'
 import {Controls} from './VideoWebControls'
 
 export function VideoEmbedInnerWeb({
-  source,
+  embed,
   active,
   setActive,
   onScreen,
 }: {
-  source: string
-  active?: boolean
-  setActive?: () => void
-  onScreen?: boolean
+  embed: AppBskyEmbedVideo.View
+  active: boolean
+  setActive: () => void
+  onScreen: boolean
 }) {
-  if (active == null || setActive == null || onScreen == null) {
-    throw new Error(
-      'active, setActive, and onScreen are required VideoEmbedInner props on web.',
-    )
-  }
-
   const containerRef = useRef<HTMLDivElement>(null)
   const ref = useRef<HTMLVideoElement>(null)
   const [focused, setFocused] = useState(false)
   const [hasSubtitleTrack, setHasSubtitleTrack] = useState(false)
+  const figId = useId()
 
   const hlsRef = useRef<Hls | undefined>(undefined)
 
@@ -37,7 +33,7 @@ export function VideoEmbedInnerWeb({
     hlsRef.current = hls
 
     hls.attachMedia(ref.current)
-    hls.loadSource(source)
+    hls.loadSource(embed.playlist)
 
     // initial value, later on it's managed by Controls
     hls.autoLevelCapping = 0
@@ -53,29 +49,40 @@ export function VideoEmbedInnerWeb({
       hls.detachMedia()
       hls.destroy()
     }
-  }, [source])
+  }, [embed.playlist])
 
   return (
-    <View
-      style={[
-        a.w_full,
-        a.rounded_sm,
-        // TODO: get from embed metadata
-        // max should be 1 / 1
-        {aspectRatio: 16 / 9},
-        a.overflow_hidden,
-      ]}>
-      <div
-        ref={containerRef}
-        style={{width: '100%', height: '100%', display: 'flex'}}>
-        <video
-          ref={ref}
-          style={{width: '100%', height: '100%', objectFit: 'contain'}}
-          playsInline
-          preload="none"
-          loop
-          muted={!focused}
-        />
+    <View style={[a.flex_1, a.rounded_sm, a.overflow_hidden]}>
+      <div ref={containerRef} style={{height: '100%', width: '100%'}}>
+        <figure style={{margin: 0, position: 'absolute', inset: 0}}>
+          <video
+            ref={ref}
+            poster={embed.thumbnail}
+            style={{width: '100%', height: '100%', objectFit: 'contain'}}
+            playsInline
+            preload="none"
+            loop
+            muted={!focused}
+            aria-labelledby={embed.alt ? figId : undefined}
+          />
+          {embed.alt && (
+            <figcaption
+              id={figId}
+              style={{
+                position: 'absolute',
+                width: 1,
+                height: 1,
+                padding: 0,
+                margin: -1,
+                overflow: 'hidden',
+                clip: 'rect(0, 0, 0, 0)',
+                whiteSpace: 'nowrap',
+                borderWidth: 0,
+              }}>
+              {embed.alt}
+            </figcaption>
+          )}
+        </figure>
         <Controls
           videoRef={ref}
           hlsRef={hlsRef}

src/view/com/util/post-embeds/index.tsx

@@ -13,6 +13,7 @@ import {
   AppBskyEmbedImages,
   AppBskyEmbedRecord,
   AppBskyEmbedRecordWithMedia,
+  AppBskyEmbedVideo,
   AppBskyFeedDefs,
   AppBskyGraphDefs,
   moderateFeedGenerator,
@@ -33,10 +34,12 @@ import {AutoSizedImage} from '../images/AutoSizedImage'
 import {ImageLayoutGrid} from '../images/ImageLayoutGrid'
 import {ExternalLinkEmbed} from './ExternalLinkEmbed'
 import {MaybeQuoteEmbed} from './QuoteEmbed'
+import {VideoEmbed} from './VideoEmbed'
 
 type Embed =
   | AppBskyEmbedRecord.View
   | AppBskyEmbedImages.View
+  | AppBskyEmbedVideo.View
   | AppBskyEmbedExternal.View
   | AppBskyEmbedRecordWithMedia.View
   | {$type: string; [k: string]: unknown}
@@ -175,6 +178,14 @@ export function PostEmbeds({
     )
   }
 
+  if (AppBskyEmbedVideo.isView(embed)) {
+    return (
+      <ContentHider modui={moderation?.ui('contentMedia')}>
+        <VideoEmbed embed={embed} />
+      </ContentHider>
+    )
+  }
+
   return <View />
 }