WebVTT: support styled cues and voice spans

package.json

@@ -87,14 +87,14 @@
     "@stitches/react": "^1.2.8",
     "flexsearch": "^0.7.43",
     "hls.js": "^1.5.3",
-    "node-webvtt": "^1.9.4",
     "openseadragon": "^4.1.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-error-boundary": "^4.0.12",
     "sanitize-html": "^2.11.0",
-    "swiper": "^9.4.1",
-    "uuid": "^9.0.1"
+    "swiper": "^9.0.0",
+    "uuid": "^9.0.1",
+    "vtt.js": "^0.13.0"
   },
   "devDependencies": {
     "@iiif/presentation-3": "^1.1.3",

src/components/Viewer/InformationPanel/Annotation/VTT/Cue.test.tsx

@@ -8,7 +8,7 @@ describe("Information panel cue component", () => {
   it("renders", () => {
     render(
       <Group>
-        <Cue label="Text" start={107} end={150} />
+        <Cue html="<div>Text</div>" text="Text" start={107} end={150} />
       </Group>,
     );
     const cue = screen.getByTestId("information-panel-cue");

src/components/Viewer/InformationPanel/Annotation/VTT/Cue.tsx

@@ -12,7 +12,8 @@ import {
 const AutoScrollDisableTime = 750;
 
 interface Props {
-  label: string;
+  html: string;
+  text: string;
   start: number;
   end: number;
 }
@@ -34,7 +35,7 @@ const findScrollableParent = (
   return null;
 };
 
-const Cue: React.FC<Props> = ({ label, start, end }) => {
+const Cue: React.FC<Props> = ({ html, text, start, end }) => {
   const dispatch: any = useViewerDispatch();
   const {
     configOptions,
@@ -132,9 +133,12 @@ const Cue: React.FC<Props> = ({ label, start, end }) => {
       aria-checked={isActive}
       data-testid="information-panel-cue"
       onClick={handleClick}
-      value={label}
+      value={text}
     >
-      {label}
+      <div
+        className="webvtt-cue"
+        dangerouslySetInnerHTML={{ __html: html }}
+      ></div>
       <strong>{convertTime(start)}</strong>
     </Item>
   );

src/components/Viewer/InformationPanel/Annotation/VTT/VTT.test.tsx

@@ -4,6 +4,10 @@ import AnnotationItemVTT from "./VTT";
 import Menu from "src/components/Viewer/InformationPanel/Menu";
 import React from "react";
 
+// Required to prevent an annoying but harmless error while running this test
+import { VTTCue } from "vtt.js";
+window.VTTCue = VTTCue;
+
 vi.mock("src/components/Viewer/InformationPanel/Menu");
 vi.mocked(Menu).mockReturnValue(<div>Menu Component</div>);
 
@@ -49,6 +53,7 @@ describe("AnnotationItemVTT", () => {
     global.fetch = vitest.fn(() =>
       Promise.reject(new Error("I am the error message")),
     );
+
     render(<AnnotationItemVTT {...props} />);
     expect(await screen.findByTestId("error-message")).toHaveTextContent(
       "Network Error: Error: I am the error message",

src/components/Viewer/InformationPanel/Annotation/VTT/VTT.tsx

@@ -1,14 +1,10 @@
 import React, { useEffect } from "react";
-import useWebVtt, {
-  NodeWebVttCue,
-  NodeWebVttCueNested,
-} from "src/hooks/use-webvtt";
+import useWebVtt, { NodeWebVttCueNested } from "src/hooks/use-webvtt";
 
 import { Group } from "src/components/Viewer/InformationPanel/Annotation/VTT/Cue.styled";
 import { InternationalString } from "@iiif/presentation-3";
 import Menu from "src/components/Viewer/InformationPanel/Menu";
 import { getLabel } from "src/hooks/use-iiif";
-import { parse } from "node-webvtt";
 
 type AnnotationItemVTTProps = {
   label: InternationalString | undefined;
@@ -20,7 +16,7 @@ const AnnotationItemVTT: React.FC<AnnotationItemVTTProps> = ({
   vttUri,
 }) => {
   const [cues, setCues] = React.useState<Array<NodeWebVttCueNested>>([]);
-  const { createNestedCues, orderCuesByTime } = useWebVtt();
+  const { createNestedCues, orderCuesByTime, parseVttData } = useWebVtt();
   const [isNetworkError, setIsNetworkError] = React.useState<Error>();
 
   useEffect(
@@ -34,11 +30,11 @@ const AnnotationItemVTT: React.FC<AnnotationItemVTTProps> = ({
         })
           .then((response) => response.text())
           .then((data) => {
-            const flatCues = parse(data)
-              .cues as unknown as Array<NodeWebVttCue>;
-            const orderedCues = orderCuesByTime(flatCues);
-            const nestedCues = createNestedCues(orderedCues);
-            setCues(nestedCues);
+            parseVttData(data).then((flatCues) => {
+              const orderedCues = orderCuesByTime(flatCues);
+              const nestedCues = createNestedCues(orderedCues);
+              setCues(nestedCues);
+            });
           })
           .catch((error) => {
             console.error(vttUri, error.toString());

src/components/Viewer/InformationPanel/Menu.tsx

@@ -10,10 +10,10 @@ const Menu: React.FC<MenuProps> = ({ items }) => {
   return (
     <MenuStyled>
       {items.map((item) => {
-        const { text, start, end, children, identifier } = item;
+        const { html, text, start, end, children, identifier } = item;
         return (
           <li key={identifier}>
-            <Cue label={text} start={start} end={end} />
+            <Cue html={html} text={text} start={start} end={end} />
             {children && <Menu items={children} />}
           </li>
         );

src/hooks/use-webvtt.ts

@@ -1,14 +1,15 @@
 // @ts-nocheck
 
 import { v4 as uuidv4 } from "uuid";
+import { WebVTT, VTTCue } from "vtt.js";
 
 export interface NodeWebVttCue {
   identifier?: string;
   start: number;
   end: number;
+  html: string;
   text: string;
-  styles?: string;
-  children?: Array<NodeWebVttCue>;
+  align?: "start" | "left" | "center" | "middle" | "end" | "right";
 }
 export interface NodeWebVttCueNested extends NodeWebVttCue {
   children?: Array<NodeWebVttCueNested>;
@@ -26,7 +27,7 @@ const useWebVtt = () => {
 
   /**
    * This function takes an array of NodeWebVttCue items as input, where each item
-   * is an object with properties identifier, start, end, text, and styles. It
+   * is an object with properties identifier, start, end, html, text, and align. It
    * iterates through the array of items and uses a stack to keep track of nested
    * items. It compares the current item's start with the end of the items in the
    * stack. If the current item's start is smaller than the end of the top item
@@ -36,7 +37,9 @@ const useWebVtt = () => {
    * the nestedItems array. The resulting nestedItems array contains the items
    * organized into nested structures based on their start and end values.
    */
-  function createNestedCues(flat: Array<NodeWebVttCue>): Array<NodeWebVttCue> {
+  function createNestedCues(
+    flat: Array<NodeWebVttCue>,
+  ): Array<NodeWebVttCueNested> {
     const nestedItems = [];
     const stack = [];
 
@@ -89,11 +92,40 @@ const useWebVtt = () => {
     return cues.sort((cue1, cue2) => cue1.start - cue2.start);
   }
 
+  function parseVttData(data: string): Promise<Array<NodeWebVttCue>> {
+    return new Promise((resolve, reject) => {
+      const cues: Array<NodeWebVttCue> = [];
+      const parser = new WebVTT.Parser(window, WebVTT.StringDecoder());
+      parser.oncue = (cue: VTTCue) => {
+        const domTree: DocumentFragment = WebVTT.convertCueToDOMTree(
+          window,
+          cue.text,
+        );
+        const html = domTree.firstElementChild?.outerHTML || "&nbsp;";
+        const text = domTree.firstElementChild?.textContent || "";
+
+        cues.push({
+          identifier: uuidv4(),
+          start: cue.startTime,
+          end: cue.endTime,
+          align: cue.align,
+          html,
+          text,
+        });
+      };
+      parser.onflush = () => resolve(cues);
+      parser.onparsingerror = (err) => reject(err);
+      parser.parse(data);
+      parser.flush();
+    });
+  }
+
   return {
     addIdentifiersToParsedCues,
     createNestedCues,
     isChild,
     orderCuesByTime,
+    parseVttData,
   };
 };