Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix annotation parsing for Aviary annotations #740

Merged
merged 1 commit into from
Dec 6, 2024
Merged

Fix annotation parsing for Aviary annotations #740

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions src/components/Transcript/Transcript.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import Transcript from './Transcript';
import * as transcriptParser from '@Services/transcript-parser';
import { withManifestAndPlayerProvider } from '@Services/testing-helpers';
import lunchroomManners from '@TestData/lunchroom-manners';
import multiSourceManifest from '@TestData/multi-source-manifest';

describe('Transcript component', () => {
let originalError, originalLogger;
Expand Down Expand Up @@ -1067,5 +1068,75 @@ describe('Transcript component', () => {
console.error = originalError;
});
});

test('manifestUrl with TextualBody supplementing annotations (Aviary)', async () => {
const props = {
playerID: 'player-id',
manifestUrl: 'http://example.com/manifest.json'
};

const transcriptsList = [
{
canvasId: 0,
items: []
},
{
canvasId: 1,
items: [
{
title: 'Aviary Annotation Style Canvas',
id: 'Aviary Annotation Style Canvas-1',
url: 'http://example.com/manifest.json',
isMachineGen: false,
}
]
}
];
const readSupplementingAnnotationsMock = jest
.spyOn(transcriptParser, 'readSupplementingAnnotations')
.mockReturnValue(transcriptsList);

const parseTranscriptMock = jest
.spyOn(transcriptParser, 'parseTranscriptData')
.mockReturnValue({
tData: [
{ begin: 22.2, end: 26.6, text: 'Transcript text line 1', tag: 'TIMED_CUE' },
{ begin: 26.7, end: 31.5, text: 'Transcript text line 2', tag: 'TIMED_CUE' }
],
tUrl: 'http://example.com/manifest.json',
tType: transcriptParser.TRANSCRIPT_TYPES.timedText,
tFileExt: 'json',
});

const TranscriptWithState = withManifestAndPlayerProvider(Transcript, {
initialManifestState: { manifest: multiSourceManifest, canvasIndex: 1 },
initialPlayerState: {},
...props
});

render(
<>
<video id="player-id" />
<TranscriptWithState />
</>
);

await act(() => Promise.resolve());

await waitFor(() => {
expect(readSupplementingAnnotationsMock).toHaveBeenCalled();
expect(parseTranscriptMock).toHaveBeenCalled();
expect(screen.queryByTestId('transcript-selector')).toBeInTheDocument();
expect(screen.queryByTestId('transcript_content_1')).toBeInTheDocument();
expect(screen.queryByTestId('no-transcript')).not.toBeInTheDocument();
expect(
screen.queryAllByTestId('transcript_time')[0]
).toHaveTextContent('00:00:22');
expect(screen.queryAllByTestId('transcript_text')[0]).toHaveTextContent(
'Transcript text line 1'
);
expect(screen.queryByTestId('transcript-machinegen-msg')).not.toBeInTheDocument();
});
});
});
});
30 changes: 19 additions & 11 deletions src/services/transcript-parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,16 @@ export async function readSupplementingAnnotations(manifestURL, title = '') {
let annotations = getAnnotations(canvas.annotations, 'supplementing');
let canvasTranscripts = [];
if (annotations.length > 0) {
let annotBody = annotations[0].body;
// Check if 'body' property is an array
let annotBody = annotations[0].body?.length > 0
? annotations[0].body[0] : annotations[0].body;
// Get AnnotationPage label if it is available
let annotationLabel = canvas.annotations?.length > 0 && canvas.annotations[0].label
? getLabelValue(canvas.annotations[0].label) : title;
if (annotBody.type === 'TextualBody') {
let label = title.length > 0
? title
: (annotBody.label ? getLabelValue(annotBody.label) : `Canvas-${index}`
);
: (annotationLabel ? annotationLabel : `Canvas-${index}`);
let { isMachineGen, labelText } = identifyMachineGen(label);
canvasTranscripts.push({
url: annotBody.id === undefined ? manifestURL : annotBody.id,
Expand Down Expand Up @@ -422,7 +426,9 @@ export function parseManifestTranscript(manifest, manifestURL, canvasIndex) {
// a list of transcript fragments
if (annotations.length > 0) {
let annotation = annotations[0];
let tType = annotation.body.type;
// 'body' property can be either an array or an object
let tType = annotation.body?.length > 0
? annotation.body[0].type : annotation.body.type;
if (tType == 'TextualBody') {
isExternalAnnotation = false;
} else {
Expand Down Expand Up @@ -520,14 +526,16 @@ function createTData(annotations) {
let tData = [];
annotations.map((a) => {
if (a.id != null) {
const tBody = a.body;
const tBody = a.body?.length > 0 ? a.body : [a.body];
const { start, end } = getMediaFragment(a.target);
tData.push({
text: tBody.value,
format: tBody.format,
begin: parseFloat(start),
end: parseFloat(end),
tag: TRANSCRIPT_CUE_TYPES.timedCue
tBody.map((t) => {
tData.push({
text: t.value,
format: t.format,
begin: parseFloat(start),
end: parseFloat(end),
tag: TRANSCRIPT_CUE_TYPES.timedCue
});
});
}
});
Expand Down
63 changes: 59 additions & 4 deletions src/services/transcript-parser.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import * as transcriptParser from './transcript-parser';
import manifestTranscript from '@TestData/volleyball-for-boys';
import multipleCanvas from '@TestData/transcript-multiple-canvas';
import annotationTranscript from '@TestData/transcript-annotation';
import multiSourceManifest from '@TestData/multi-source-manifest';
import mammoth from 'mammoth';
import { cleanup } from '@testing-library/react';
const utils = require('./utility-helpers');
Expand Down Expand Up @@ -109,6 +110,33 @@ describe('transcript-parser', () => {
}
]);
});

test('for transcript as an AnnotationPage (Aviary)', async () => {
const fetchSpy = jest.spyOn(global, 'fetch').mockResolvedValueOnce({
status: 200,
headers: { get: jest.fn(() => 'application/json') },
json: jest.fn(() => multiSourceManifest),
});

const transcripts = await transcriptParser.readSupplementingAnnotations(
'https://example.com/multi-source/manifest.json'
);

expect(fetchSpy).toHaveBeenCalledTimes(1);
expect(fetchSpy).toHaveBeenCalledWith('https://example.com/multi-source/manifest.json');
expect(transcripts).toHaveLength(2);
expect(transcripts[0].items).toHaveLength(0);
expect(transcripts[1].items).toHaveLength(1);
expect(transcripts[1].items).toEqual([
{
title: 'Aviary Supplementing Annotations',
id: 'Aviary Supplementing Annotations-1',
url: 'https://example.com/multi-source/manifest.json',
isMachineGen: false,
format: ''
}
]);
});
});
});

Expand Down Expand Up @@ -588,20 +616,20 @@ describe('transcript-parser', () => {
1
00:00:01.200 --> 00:00:21.000
[music]

2
00:00:22.200 --> 00:00:26.600
Just before lunch one day, a puppet show
was put on at school.

3
00:00:26.700 --> 00:00:31.500
It was called "Mister Bungle Goes to Lunch".

4
00:00:31.600 --> 00:00:34.500
It was fun to watch.

5
00:00:36.100 --> 00:00:41.300
In the puppet show, Mr. Bungle came to the
Expand Down Expand Up @@ -674,6 +702,33 @@ describe('transcript-parser', () => {
expect(tFileExt).toEqual('json');
expect(tType).toEqual(1);
});

test('as an AnnotationPage with a list of annotations (Aviary)', () => {
const { tData, tUrl, tType, tFileExt } = transcriptParser.parseManifestTranscript(
multiSourceManifest,
'https://example.com/multi-source-manifest.json',
1
);

expect(tData).toHaveLength(2);
expect(tData[0]).toEqual({
text: 'Transcript text line 1',
format: 'text/plain',
begin: 22.2,
end: 26.6,
tag: 'TIMED_CUE'
});
expect(tData[1]).toEqual({
text: 'Transcript text line 2',
format: 'text/plain',
begin: 26.7,
end: 31.5,
tag: 'TIMED_CUE'
});
expect(tUrl).toEqual('https://example.com/multi-source-manifest.json');
expect(tFileExt).toEqual('json');
expect(tType).toEqual(1);
});
});
});

Expand Down
18 changes: 13 additions & 5 deletions src/services/utility-helpers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,10 +323,18 @@ export function parseResourceAnnotations(annotation, duration, motivation, start
poster = '',
error = 'No resources found in Canvas';

const parseAnnotation = (a) => {
const source = getResourceInfo(a, start, duration, motivation);
// Check if the parsed sources has a resource URL
(source && source.src) && resources.push(source);
const parseAnnotation = (annotationItems) => {
/**
* Convert annotation items to an array, because 'body' property
* can sometimes contain an array instead of an object.
* Ex: Aviary annotations: https://weareavp.aviaryplatform.com/iiif/hm52f7jz70/manifest
*/
annotationItems = annotationItems?.length > 0 ? annotationItems : [annotationItems];
annotationItems.map((a) => {
const source = getResourceInfo(a, start, duration, motivation);
// Check if the parsed sources has a resource URL
(source && source.src) && resources.push(source);
});
};

if (annotation && annotation != undefined) {
Expand All @@ -338,7 +346,7 @@ export function parseResourceAnnotations(annotation, duration, motivation, start
poster: getPlaceholderCanvas(annotation)
};
}
// When multiple resources are in a single Canvas
// When multiple resources/annotations are in a single Canvas
else if (items?.length > 1) {
items.map((p, index) => {
if (p.motivation === motivation) {
Expand Down
87 changes: 87 additions & 0 deletions src/test_data/multi-source-manifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,93 @@ export default {
]
}
],
},
{
id: 'https://example.com/multi-source-manifest/canvas/2',
type: 'Canvas',
height: 360,
width: 480,
duration: 572.034,
label: { en: ['Aviary Annotation Style Canvas'] },
items: [
{
id: 'https://example.com/multi-source-manifest/canvas/2/page/1',
type: 'AnnotationPage',
items: [
{
id: 'https://example.com/multi-source-manifest/canvas/2/page/1/annotation/1',
type: 'Annotation',
motivation: 'painting',
body: {
type: 'Choice',
choiceHint: 'user',
items: [
{
id: 'https://example.com/sample/transcript-annotation/high/media.mp4',
type: 'Video',
format: 'video/mp4',
label: {
en: ['High'],
},
},
{
id: 'https://example.com/sample/transcript-annotation/medium/media.mp4',
type: 'Video',
format: 'video/mp4',
label: {
en: ['Medium'],
},
},
{
id: 'https://example.com/sample/transcript-annotation/low/media.mp4',
type: 'Video',
format: 'video/mp4',
label: {
en: ['Low'],
},
},
],
},
target: 'https://example.com/sample/transcript-annotation/canvas/1',
},
],
}
],
annotations: [
{
id: 'https://example.com/multi-source-manifest/canvas/2/page/2',
type: 'AnnotationPage',
label: { en: ['Aviary Supplementing Annotations'] },
items: [
{
id: 'https://example.com/multi-source-manifest/canvas/2/page/2/annotation/1',
type: 'Annotation',
motivation: 'supplementing',
body: [
{
type: 'TextualBody',
value: 'Transcript text line 1',
format: 'text/plain',
}
],
target: 'https://example.com/multi-source-manifest/canvas/2#t=22.2,26.6',
},
{
id: 'https://example.com/multi-source-manifest/canvas/2/page/2/annotation/2',
type: 'Annotation',
motivation: 'supplementing',
body: [
{
type: 'TextualBody',
value: 'Transcript text line 2',
format: 'text/plain',
}
],
target: 'https://example.com/multi-source-manifest/canvas/2#t=26.7,31.5',
},
],
},
]
}
],
structures: [
Expand Down