🚧 Video subtitle

app/surprise-me/page.tsx

@@ -1,5 +1,5 @@
 'use client'
-import { useEffect, useState } from 'react'
+import { useState } from 'react'
 import { useInterval, useTitle } from 'react-use'
 import dynamic from 'next/dynamic'
 

components/common/posts/PostsFeed.tsx

@@ -1,21 +1,21 @@
 'use client'
 import { useEffect, useState } from 'react'
+import dynamic from 'next/dynamic'
 import {
   AppBskyFeedDefs,
   AppBskyEmbedImages,
   AppBskyEmbedExternal,
   AppBskyEmbedRecordWithMedia,
+  AppBskyEmbedVideo,
   AppBskyFeedPost,
   AppBskyEmbedRecord,
-  AppBskyGraphList,
   AppBskyGraphDefs,
 } from '@atproto/api'
 import Link from 'next/link'
 import {
   DocumentMagnifyingGlassIcon,
   ExclamationCircleIcon,
   LanguageIcon,
-  InformationCircleIcon,
   FolderMinusIcon,
   FolderPlusIcon,
 } from '@heroicons/react/24/outline'
@@ -37,6 +37,9 @@ import {
   useWorkspaceRemoveItemsMutation,
 } from '@/workspace/hooks'
 import { ImageList } from './ImageList'
+const VideoPlayer = dynamic(() => import('@/common/video/player'), {
+  ssr: false,
+})
 
 export function PostsFeed({
   items,
@@ -229,6 +232,18 @@ export function PostEmbeds({ item }: { item: AppBskyFeedDefs.FeedViewPost }) {
     imageRequiresBlur ? 'blur-sm hover:blur-none' : '',
   )
 
+  if (AppBskyEmbedVideo.isView(embed)) {
+    return (
+      <div className="flex gap-2 pb-2 pl-4" aria-label={embed.alt}>
+        <VideoPlayer
+          source={embed.playlist}
+          thumbnail={embed.thumbnail}
+          alt={embed.alt}
+        />
+      </div>
+    )
+  }
+
   // render image embeds
   if (AppBskyEmbedImages.isView(embed)) {
     const embeddedImageClassName = classNames(

components/common/video/player.tsx

@@ -0,0 +1,105 @@
+import Hls from 'hls.js/dist/hls.light' // Use light build of hls.
+import { useEffect, useId, useRef, useState } from 'react'
+import { ActionButton } from '../buttons'
+import { useSubtitle } from './useSubtitle'
+
+export default function VideoPlayer({
+  source,
+  alt,
+  thumbnail,
+}: {
+  source: string
+  alt?: string
+  thumbnail?: string
+}) {
+  const [hls] = useState(() => new Hls())
+  const [isUnsupported, setIsUnsupported] = useState(false)
+  const { subtitle, loadSubtitle } = useSubtitle(source)
+  const figId = useId()
+  const ref = useRef<HTMLVideoElement>(null)
+
+  useEffect(() => {
+    if (ref.current && Hls.isSupported()) {
+      hls.attachMedia(ref.current)
+
+      return () => {
+        hls.detachMedia()
+      }
+    }
+  }, [hls])
+
+  useEffect(() => {
+    if (ref.current) {
+      if (Hls.isSupported()) {
+        setIsUnsupported(false)
+        hls.loadSource(source)
+      } else {
+        setIsUnsupported(true)
+      }
+    }
+  }, [source, hls])
+
+  return (
+    <figure>
+      <video
+        poster={thumbnail}
+        ref={ref}
+        style={{ flex: 1 }}
+        playsInline
+        preload="none"
+        controls
+        loop
+        muted
+        aria-labelledby={alt ? figId : undefined}
+      >
+        {!!subtitle.url && (
+          <track
+            label="English"
+            kind="subtitles"
+            srcLang="en"
+            src={subtitle.url}
+            default
+          />
+        )}
+        {isUnsupported && (
+          <p>
+            Your browser does not seem to support HLS videos. Please switch to a
+            different browser to view this content.
+          </p>
+        )}
+      </video>
+      <figcaption className="flex flex-row justify-end mt-2">
+        <ActionButton
+          appearance={subtitle.url ? 'primary' : 'outlined'}
+          disabled={subtitle.isLoading}
+          onClick={() => loadSubtitle()}
+          size="sm"
+        >
+          {subtitle.isLoading
+            ? 'Loading Subtitles...'
+            : subtitle.url
+            ? 'Hide Subtitles'
+            : 'Show Subtitles'}
+        </ActionButton>
+      </figcaption>
+      {alt && (
+        <figcaption
+          id={figId}
+          style={{
+            position: 'absolute',
+            width: 1,
+            height: 1,
+            padding: 0,
+            margin: -1,
+            overflow: 'hidden',
+            clip: 'rect(0, 0, 0, 0)',
+            whiteSpace: 'nowrap',
+            borderWidth: 0,
+          }}
+        >
+          {alt}
+        </figcaption>
+      )}
+    </figure>
+  )
+}

components/common/video/useSubtitle.tsx

@@ -0,0 +1,66 @@
+import client from '@/lib/client'
+import { useState } from 'react'
+import { toast } from 'react-toastify'
+
+export const useSubtitle = (source: string) => {
+  const [subtitle, setSubtitle] = useState<{
+    isLoading: boolean
+    error?: string
+    url: string
+  }>({
+    isLoading: false,
+    error: '',
+    url: '',
+  })
+
+  const loadSubtitle = async () => {
+    setSubtitle({
+      isLoading: true,
+      error: '',
+      url: '',
+    })
+    try {
+      const videoUrl = new URL(source)
+      const sourceFragments = source.split('/')
+      const videoCid = sourceFragments[sourceFragments.length - 2]
+      const authorDid = sourceFragments[sourceFragments.length - 3]
+
+      const { data: serviceAuth } =
+        await client.api.com.atproto.server.getServiceAuth({
+          aud: videoUrl.host,
+          lxm: 'app.bsky.video.getTranscript',
+        })
+
+      if (!serviceAuth?.token) {
+        toast.error('Failed to get auth token')
+        setSubtitle({
+          isLoading: false,
+          error: 'Failed to get auth token',
+          url: '',
+        })
+        return
+      }
+
+      setSubtitle({
+        isLoading: false,
+        error: '',
+        // TODO: does the endpoint expect the token only in the header of the req? or does query param work?
+        url: `${videoUrl.origin}/video/${authorDid}/${videoCid}/transcription_eng.vtt?token=${serviceAuth.token}`,
+      })
+    } catch (error) {
+      console.error(error)
+      const err = `Failed to get subtitle: ${error?.['message']}`
+      setSubtitle({
+        isLoading: false,
+        error: err,
+        url: '',
+      })
+      toast.error(err)
+    }
+  }
+
+  return {
+    subtitle,
+    loadSubtitle,
+  }
+}

lib/client.ts

@@ -1,4 +1,4 @@
-import { AtpAgent, AtpServiceClient, AtpSessionData } from '@atproto/api'
+import { AtpAgent, Agent, AtpSessionData } from '@atproto/api'
 import { AuthState } from './types'
 import { OzoneConfig, getConfig } from './client-config'
 import { OZONE_SERVICE_DID } from './constants'
@@ -36,7 +36,7 @@ class ClientManager extends EventTarget {
     return AuthState.LoggedIn
   }
 
-  get api(): AtpServiceClient {
+  get api(): Agent {
     if (this._agent) {
       return this._agent.api
     }

package.json

@@ -17,7 +17,7 @@
     "e2e:run": "$(yarn bin)/cypress run --browser chrome"
   },
   "dependencies": {
-    "@atproto/api": "0.12.22",
+    "@atproto/api": "0.13.5",
     "@headlessui/react": "^1.7.7",
     "@heroicons/react": "^2.0.13",
     "@tanstack/react-query": "^4.22.0",
@@ -29,6 +29,7 @@
     "date-fns": "^2.29.3",
     "eslint": "8.30.0",
     "eslint-config-next": "13.4.8",
+    "hls.js": "^1.5.13",
     "kbar": "^0.1.0-beta.45",
     "lande": "^1.0.10",
     "next": "14.2.5",

yarn.lock

@@ -2,15 +2,16 @@
 # yarn lockfile v1
 
 
-"@atproto/api@0.12.22":
-  version "0.12.22"
-  resolved "https://registry.yarnpkg.com/@atproto/api/-/api-0.12.22.tgz#1880a93a0caa4485cd8463bd1e10bf2424b9826c"
-  integrity sha512-TIXSnf3qqyX40Ei/FkK4H24w+7s5rOc63TPwrGakRBOqIgSNBKOggei8I600fJ/AXB7HO6Vp9tBmDVOt2+021A==
+"@atproto/api@0.13.5":
+  version "0.13.5"
+  resolved "https://registry.yarnpkg.com/@atproto/api/-/api-0.13.5.tgz#04305cdb0a467ba366305c5e95cebb7ce0d39735"
+  integrity sha512-yT/YimcKYkrI0d282Zxo7O30OSYR+KDW89f81C6oYZfDRBcShC1aniVV8kluP5LrEAg8O27yrOSnBgx2v7XPew==
   dependencies:
     "@atproto/common-web" "^0.3.0"
-    "@atproto/lexicon" "^0.4.0"
+    "@atproto/lexicon" "^0.4.1"
     "@atproto/syntax" "^0.3.0"
-    "@atproto/xrpc" "^0.5.0"
+    "@atproto/xrpc" "^0.6.1"
+    await-lock "^2.2.2"
     multiformats "^9.9.0"
     tlds "^1.234.0"
 
@@ -24,29 +25,29 @@
     uint8arrays "3.0.0"
     zod "^3.21.4"
 
-"@atproto/lexicon@^0.4.0":
-  version "0.4.0"
-  resolved "https://registry.yarnpkg.com/@atproto/lexicon/-/lexicon-0.4.0.tgz#63e8829945d80c25524882caa8ed27b1151cc576"
-  integrity sha512-RvCBKdSI4M8qWm5uTNz1z3R2yIvIhmOsMuleOj8YR6BwRD+QbtUBy3l+xQ7iXf4M5fdfJFxaUNa6Ty0iRwdKqQ==
+"@atproto/lexicon@^0.4.1":
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/@atproto/lexicon/-/lexicon-0.4.1.tgz#19155210570a2fafbcc7d4f655d9b813948e72a0"
+  integrity sha512-bzyr+/VHXLQWbumViX5L7h1NKQObfs8Z+XZJl43OUK8nYFUI4e/sW1IZKRNfw7Wvi5YVNK+J+yP3DWIBZhkCYA==
   dependencies:
     "@atproto/common-web" "^0.3.0"
     "@atproto/syntax" "^0.3.0"
     iso-datestring-validator "^2.2.2"
     multiformats "^9.9.0"
-    zod "^3.21.4"
+    zod "^3.23.8"
 
 "@atproto/syntax@^0.3.0":
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/@atproto/syntax/-/syntax-0.3.0.tgz#fafa2dbea9add37253005cb663e7373e05e618b3"
   integrity sha512-Weq0ZBxffGHDXHl9U7BQc2BFJi/e23AL+k+i5+D9hUq/bzT4yjGsrCejkjq0xt82xXDjmhhvQSZ0LqxyZ5woxA==
 
-"@atproto/xrpc@^0.5.0":
-  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/@atproto/xrpc/-/xrpc-0.5.0.tgz#dacbfd8f7b13f0ab5bd56f8fdd4b460e132a6032"
-  integrity sha512-swu+wyOLvYW4l3n+VAuJbHcPcES+tin2Lsrp8Bw5aIXIICiuFn1YMFlwK9JwVUzTH21Py1s1nHEjr4CJeElJog==
+"@atproto/xrpc@^0.6.1":
+  version "0.6.1"
+  resolved "https://registry.yarnpkg.com/@atproto/xrpc/-/xrpc-0.6.1.tgz#dcd1315c8c60eef5af2db7fa4e35a38ebc6d79d5"
+  integrity sha512-Zy5ydXEdk6sY7FDUZcEVfCL1jvbL4tXu5CcdPqbEaW6LQtk9GLds/DK1bCX9kswTGaBC88EMuqQMfkxOhp2t4A==
   dependencies:
-    "@atproto/lexicon" "^0.4.0"
-    zod "^3.21.4"
+    "@atproto/lexicon" "^0.4.1"
+    zod "^3.23.8"
 
 "@babel/runtime-corejs3@^7.10.2":
   version "7.20.6"
@@ -742,6 +743,11 @@ autoprefixer@^10.4.13:
     picocolors "^1.0.0"
     postcss-value-parser "^4.2.0"
 
+await-lock@^2.2.2:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/await-lock/-/await-lock-2.2.2.tgz#a95a9b269bfd2f69d22b17a321686f551152bcef"
+  integrity sha512-aDczADvlvTGajTDjcjpJMqRkOF6Qdz3YbPZm/PyW6tKPkx2hlYBzxMhEywM/tU72HrVZjgl5VCdRuMlA7pZ8Gw==
+
 aws-sign2@~0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
@@ -2196,6 +2202,11 @@ hastscript@^7.0.0:
     property-information "^6.0.0"
     space-separated-tokens "^2.0.0"
 
+hls.js@^1.5.13:
+  version "1.5.13"
+  resolved "https://registry.yarnpkg.com/hls.js/-/hls.js-1.5.13.tgz#27bf1c9b91c433e25e7a84635fe7491bb5988d93"
+  integrity sha512-xRgKo84nsC7clEvSfIdgn/Tc0NOT+d7vdiL/wvkLO+0k0juc26NRBPPG1SfB8pd5bHXIjMW/F5VM8VYYkOYYdw==
+
 html-void-elements@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-2.0.1.tgz#29459b8b05c200b6c5ee98743c41b979d577549f"
@@ -4896,7 +4907,7 @@ yocto-queue@^0.1.0:
   resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
 
-zod@^3.21.4:
+zod@^3.21.4, zod@^3.23.8:
   version "3.23.8"
   resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
   integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==