From 6cf9bdcf946e1487c8ede392125e41cc93501fe7 Mon Sep 17 00:00:00 2001
From: dwithana <dwithana@iu.edu>
Date: Wed, 13 Nov 2024 16:54:37 -0500
Subject: [PATCH] Fix annotation parsing for Aviary annotations

---
 src/components/Transcript/Transcript.test.js | 71 ++++++++++++++++
 src/services/transcript-parser.js            | 30 ++++---
 src/services/transcript-parser.test.js       | 63 +++++++++++++-
 src/services/utility-helpers.js              | 18 ++--
 src/test_data/multi-source-manifest.js       | 87 ++++++++++++++++++++
 5 files changed, 249 insertions(+), 20 deletions(-)

diff --git a/src/components/Transcript/Transcript.test.js b/src/components/Transcript/Transcript.test.js
index 7e88ca73..cd34ec8d 100644
--- a/src/components/Transcript/Transcript.test.js
+++ b/src/components/Transcript/Transcript.test.js
@@ -4,6 +4,7 @@ import Transcript from './Transcript';
 import * as transcriptParser from '@Services/transcript-parser';
 import { withManifestAndPlayerProvider } from '@Services/testing-helpers';
 import lunchroomManners from '@TestData/lunchroom-manners';
+import multiSourceManifest from '@TestData/multi-source-manifest';
 
 describe('Transcript component', () => {
   let originalError, originalLogger;
@@ -1067,5 +1068,75 @@ describe('Transcript component', () => {
         console.error = originalError;
       });
     });
+
+    test('manifestUrl with TextualBody supplementing annotations (Aviary)', async () => {
+      const props = {
+        playerID: 'player-id',
+        manifestUrl: 'http://example.com/manifest.json'
+      };
+
+      const transcriptsList = [
+        {
+          canvasId: 0,
+          items: []
+        },
+        {
+          canvasId: 1,
+          items: [
+            {
+              title: 'Aviary Annotation Style Canvas',
+              id: 'Aviary Annotation Style Canvas-1',
+              url: 'http://example.com/manifest.json',
+              isMachineGen: false,
+            }
+          ]
+        }
+      ];
+      const readSupplementingAnnotationsMock = jest
+        .spyOn(transcriptParser, 'readSupplementingAnnotations')
+        .mockReturnValue(transcriptsList);
+
+      const parseTranscriptMock = jest
+        .spyOn(transcriptParser, 'parseTranscriptData')
+        .mockReturnValue({
+          tData: [
+            { begin: 22.2, end: 26.6, text: 'Transcript text line 1', tag: 'TIMED_CUE' },
+            { begin: 26.7, end: 31.5, text: 'Transcript text line 2', tag: 'TIMED_CUE' }
+          ],
+          tUrl: 'http://example.com/manifest.json',
+          tType: transcriptParser.TRANSCRIPT_TYPES.timedText,
+          tFileExt: 'json',
+        });
+
+      const TranscriptWithState = withManifestAndPlayerProvider(Transcript, {
+        initialManifestState: { manifest: multiSourceManifest, canvasIndex: 1 },
+        initialPlayerState: {},
+        ...props
+      });
+
+      render(
+        <>
+          <video id="player-id" />
+          <TranscriptWithState />
+        </>
+      );
+
+      await act(() => Promise.resolve());
+
+      await waitFor(() => {
+        expect(readSupplementingAnnotationsMock).toHaveBeenCalled();
+        expect(parseTranscriptMock).toHaveBeenCalled();
+        expect(screen.queryByTestId('transcript-selector')).toBeInTheDocument();
+        expect(screen.queryByTestId('transcript_content_1')).toBeInTheDocument();
+        expect(screen.queryByTestId('no-transcript')).not.toBeInTheDocument();
+        expect(
+          screen.queryAllByTestId('transcript_time')[0]
+        ).toHaveTextContent('00:00:22');
+        expect(screen.queryAllByTestId('transcript_text')[0]).toHaveTextContent(
+          'Transcript text line 1'
+        );
+        expect(screen.queryByTestId('transcript-machinegen-msg')).not.toBeInTheDocument();
+      });
+    });
   });
 });
diff --git a/src/services/transcript-parser.js b/src/services/transcript-parser.js
index d356f5f9..b9e9c14f 100644
--- a/src/services/transcript-parser.js
+++ b/src/services/transcript-parser.js
@@ -79,12 +79,16 @@ export async function readSupplementingAnnotations(manifestURL, title = '') {
           let annotations = getAnnotations(canvas.annotations, 'supplementing');
           let canvasTranscripts = [];
           if (annotations.length > 0) {
-            let annotBody = annotations[0].body;
+            // Check if 'body' property is an array
+            let annotBody = annotations[0].body?.length > 0
+              ? annotations[0].body[0] : annotations[0].body;
+            // Get AnnotationPage label if it is available
+            let annotationLabel = canvas.annotations?.length > 0 && canvas.annotations[0].label
+              ? getLabelValue(canvas.annotations[0].label) : title;
             if (annotBody.type === 'TextualBody') {
               let label = title.length > 0
                 ? title
-                : (annotBody.label ? getLabelValue(annotBody.label) : `Canvas-${index}`
-                );
+                : (annotationLabel ? annotationLabel : `Canvas-${index}`);
               let { isMachineGen, labelText } = identifyMachineGen(label);
               canvasTranscripts.push({
                 url: annotBody.id === undefined ? manifestURL : annotBody.id,
@@ -422,7 +426,9 @@ export function parseManifestTranscript(manifest, manifestURL, canvasIndex) {
   // a list of transcript fragments
   if (annotations.length > 0) {
     let annotation = annotations[0];
-    let tType = annotation.body.type;
+    // 'body' property can be either an array or an object
+    let tType = annotation.body?.length > 0
+      ? annotation.body[0].type : annotation.body.type;
     if (tType == 'TextualBody') {
       isExternalAnnotation = false;
     } else {
@@ -520,14 +526,16 @@ function createTData(annotations) {
   let tData = [];
   annotations.map((a) => {
     if (a.id != null) {
-      const tBody = a.body;
+      const tBody = a.body?.length > 0 ? a.body : [a.body];
       const { start, end } = getMediaFragment(a.target);
-      tData.push({
-        text: tBody.value,
-        format: tBody.format,
-        begin: parseFloat(start),
-        end: parseFloat(end),
-        tag: TRANSCRIPT_CUE_TYPES.timedCue
+      tBody.map((t) => {
+        tData.push({
+          text: t.value,
+          format: t.format,
+          begin: parseFloat(start),
+          end: parseFloat(end),
+          tag: TRANSCRIPT_CUE_TYPES.timedCue
+        });
       });
     }
   });
diff --git a/src/services/transcript-parser.test.js b/src/services/transcript-parser.test.js
index 74ec017f..e003e90c 100644
--- a/src/services/transcript-parser.test.js
+++ b/src/services/transcript-parser.test.js
@@ -2,6 +2,7 @@ import * as transcriptParser from './transcript-parser';
 import manifestTranscript from '@TestData/volleyball-for-boys';
 import multipleCanvas from '@TestData/transcript-multiple-canvas';
 import annotationTranscript from '@TestData/transcript-annotation';
+import multiSourceManifest from '@TestData/multi-source-manifest';
 import mammoth from 'mammoth';
 import { cleanup } from '@testing-library/react';
 const utils = require('./utility-helpers');
@@ -109,6 +110,33 @@ describe('transcript-parser', () => {
           }
         ]);
       });
+
+      test('for transcript as an AnnotationPage (Aviary)', async () => {
+        const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValueOnce({
+          status: 200,
+          headers: { get: jest.fn(() => 'application/json') },
+          json: jest.fn(() => multiSourceManifest),
+        });
+
+        const transcripts = await transcriptParser.readSupplementingAnnotations(
+          'https://example.com/multi-source/manifest.json'
+        );
+
+        expect(fetchSpy).toHaveBeenCalledTimes(1);
+        expect(fetchSpy).toHaveBeenCalledWith('https://example.com/multi-source/manifest.json');
+        expect(transcripts).toHaveLength(2);
+        expect(transcripts[0].items).toHaveLength(0);
+        expect(transcripts[1].items).toHaveLength(1);
+        expect(transcripts[1].items).toEqual([
+          {
+            title: 'Aviary Supplementing Annotations',
+            id: 'Aviary Supplementing Annotations-1',
+            url: 'https://example.com/multi-source/manifest.json',
+            isMachineGen: false,
+            format: ''
+          }
+        ]);
+      });
     });
   });
 
@@ -588,20 +616,20 @@ describe('transcript-parser', () => {
           1
           00:00:01.200 --> 00:00:21.000
           [music]
-          
+
           2
           00:00:22.200 --> 00:00:26.600
           Just before lunch one day, a puppet show 
           was put on at school.
-          
+
           3
           00:00:26.700 --> 00:00:31.500
           It was called "Mister Bungle Goes to Lunch".
-          
+
           4
           00:00:31.600 --> 00:00:34.500
           It was fun to watch.
-          
+
           5
           00:00:36.100 --> 00:00:41.300
           In the puppet show, Mr. Bungle came to the 
@@ -674,6 +702,33 @@ describe('transcript-parser', () => {
           expect(tFileExt).toEqual('json');
           expect(tType).toEqual(1);
         });
+
+        test('as an AnnotationPage with a list of annotations (Aviary)', () => {
+          const { tData, tUrl, tType, tFileExt } = transcriptParser.parseManifestTranscript(
+            multiSourceManifest,
+            'https://example.com/multi-source-manifest.json',
+            1
+          );
+
+          expect(tData).toHaveLength(2);
+          expect(tData[0]).toEqual({
+            text: 'Transcript text line 1',
+            format: 'text/plain',
+            begin: 22.2,
+            end: 26.6,
+            tag: 'TIMED_CUE'
+          });
+          expect(tData[1]).toEqual({
+            text: 'Transcript text line 2',
+            format: 'text/plain',
+            begin: 26.7,
+            end: 31.5,
+            tag: 'TIMED_CUE'
+          });
+          expect(tUrl).toEqual('https://example.com/multi-source-manifest.json');
+          expect(tFileExt).toEqual('json');
+          expect(tType).toEqual(1);
+        });
       });
     });
 
diff --git a/src/services/utility-helpers.js b/src/services/utility-helpers.js
index 0e7e2721..b4be4aaa 100644
--- a/src/services/utility-helpers.js
+++ b/src/services/utility-helpers.js
@@ -323,10 +323,18 @@ export function parseResourceAnnotations(annotation, duration, motivation, start
     poster = '',
     error = 'No resources found in Canvas';
 
-  const parseAnnotation = (a) => {
-    const source = getResourceInfo(a, start, duration, motivation);
-    // Check if the parsed sources has a resource URL
-    (source && source.src) && resources.push(source);
+  const parseAnnotation = (annotationItems) => {
+    /**
+     * Convert annotation items to an array, because 'body' property 
+     * can sometimes contain an array instead of an object.
+     * Ex: Aviary annotations: https://weareavp.aviaryplatform.com/iiif/hm52f7jz70/manifest
+     */
+    annotationItems = annotationItems?.length > 0 ? annotationItems : [annotationItems];
+    annotationItems.map((a) => {
+      const source = getResourceInfo(a, start, duration, motivation);
+      // Check if the parsed sources has a resource URL
+      (source && source.src) && resources.push(source);
+    });
   };
 
   if (annotation && annotation != undefined) {
@@ -338,7 +346,7 @@ export function parseResourceAnnotations(annotation, duration, motivation, start
         poster: getPlaceholderCanvas(annotation)
       };
     }
-    // When multiple resources are in a single Canvas
+    // When multiple resources/annotations are in a single Canvas
     else if (items?.length > 1) {
       items.map((p, index) => {
         if (p.motivation === motivation) {
diff --git a/src/test_data/multi-source-manifest.js b/src/test_data/multi-source-manifest.js
index ff899221..f3d4bc2b 100644
--- a/src/test_data/multi-source-manifest.js
+++ b/src/test_data/multi-source-manifest.js
@@ -46,6 +46,93 @@ export default {
           ]
         }
       ],
+    },
+    {
+      id: 'https://example.com/multi-source-manifest/canvas/2',
+      type: 'Canvas',
+      height: 360,
+      width: 480,
+      duration: 572.034,
+      label: { en: ['Aviary Annotation Style Canvas'] },
+      items: [
+        {
+          id: 'https://example.com/multi-source-manifest/canvas/2/page/1',
+          type: 'AnnotationPage',
+          items: [
+            {
+              id: 'https://example.com/multi-source-manifest/canvas/2/page/1/annotation/1',
+              type: 'Annotation',
+              motivation: 'painting',
+              body: {
+                type: 'Choice',
+                choiceHint: 'user',
+                items: [
+                  {
+                    id: 'https://example.com/sample/transcript-annotation/high/media.mp4',
+                    type: 'Video',
+                    format: 'video/mp4',
+                    label: {
+                      en: ['High'],
+                    },
+                  },
+                  {
+                    id: 'https://example.com/sample/transcript-annotation/medium/media.mp4',
+                    type: 'Video',
+                    format: 'video/mp4',
+                    label: {
+                      en: ['Medium'],
+                    },
+                  },
+                  {
+                    id: 'https://example.com/sample/transcript-annotation/low/media.mp4',
+                    type: 'Video',
+                    format: 'video/mp4',
+                    label: {
+                      en: ['Low'],
+                    },
+                  },
+                ],
+              },
+              target: 'https://example.com/sample/transcript-annotation/canvas/1',
+            },
+          ],
+        }
+      ],
+      annotations: [
+        {
+          id: 'https://example.com/multi-source-manifest/canvas/2/page/2',
+          type: 'AnnotationPage',
+          label: { en: ['Aviary Supplementing Annotations'] },
+          items: [
+            {
+              id: 'https://example.com/multi-source-manifest/canvas/2/page/2/annotation/1',
+              type: 'Annotation',
+              motivation: 'supplementing',
+              body: [
+                {
+                  type: 'TextualBody',
+                  value: 'Transcript text line 1',
+                  format: 'text/plain',
+                }
+              ],
+              target: 'https://example.com/multi-source-manifest/canvas/2#t=22.2,26.6',
+            },
+            {
+              id: 'https://example.com/multi-source-manifest/canvas/2/page/2/annotation/2',
+              type: 'Annotation',
+              motivation: 'supplementing',
+              body: [
+                {
+                  type: 'TextualBody',
+                  value: 'Transcript text line 2',
+                  format: 'text/plain',
+                }
+              ],
+              target: 'https://example.com/multi-source-manifest/canvas/2#t=26.7,31.5',
+            },
+          ],
+        },
+      ]
     }
   ],
   structures: [