diff --git a/api-extractor/report/hls.js.api.md b/api-extractor/report/hls.js.api.md index 0dd5b746a23..39020d6f738 100644 --- a/api-extractor/report/hls.js.api.md +++ b/api-extractor/report/hls.js.api.md @@ -4620,6 +4620,8 @@ export interface SubtitleFragProcessedData { // (undocumented) frag: Fragment; // (undocumented) + part: Part | null; + // (undocumented) success: boolean; } diff --git a/src/controller/base-stream-controller.ts b/src/controller/base-stream-controller.ts index bbedf5d4871..e76dfeab8bc 100644 --- a/src/controller/base-stream-controller.ts +++ b/src/controller/base-stream-controller.ts @@ -1127,10 +1127,6 @@ export default class BaseStreamController // Buffer must be ahead of first part + duration of parts after last segment // and playback must be at or past segment adjacent to part list const firstPart = details.partList[0]; - // Loading of VTT subtitle parts is not implemented in subtitle-stream-controller (#7460) - if (firstPart.fragment.type === PlaylistLevelType.SUBTITLE) { - return false; - } const safePartStart = firstPart.end + (details.fragmentHint?.duration || 0); if (bufferEnd >= safePartStart) { diff --git a/src/controller/subtitle-stream-controller.ts b/src/controller/subtitle-stream-controller.ts index 93d9ea9dcca..3f219fcd68b 100644 --- a/src/controller/subtitle-stream-controller.ts +++ b/src/controller/subtitle-stream-controller.ts @@ -1,5 +1,4 @@ import BaseStreamController, { State } from './base-stream-controller'; -import { findFragmentByPTS } from './fragment-finders'; import { FragmentState } from './fragment-tracker'; import { ErrorDetails, ErrorTypes } from '../errors'; import { Events } from '../events'; @@ -48,7 +47,7 @@ export class SubtitleStreamController implements NetworkComponentAPI { private currentTrackId: number = -1; - private tracksBuffered: Array = []; + private tracksBuffered: Array = []; private mainDetails: LevelDetails | null = null; constructor( @@ -128,16 +127,7 @@ export class SubtitleStreamController event: Events.SUBTITLE_FRAG_PROCESSED, data: SubtitleFragProcessed, ) { - const { frag, success } = data; - if (!this.fragContextChanged(frag)) { - if (isMediaFragment(frag)) { - this.fragPrevious = frag; - } - this.state = State.IDLE; - } - if (!success) { - return; - } + const { frag, part } = data; const buffered = this.tracksBuffered[this.currentTrackId]; if (!buffered) { @@ -147,28 +137,36 @@ export class SubtitleStreamController // Create/update a buffered array matching the interface used by BufferHelper.bufferedInfo // so we can re-use the logic used to detect how much has been buffered let timeRange: TimeRange | undefined; - const fragStart = frag.start; + const start = (part || frag).start; for (let i = 0; i < buffered.length; i++) { - if (fragStart >= buffered[i].start && fragStart <= buffered[i].end) { + if (start >= buffered[i].start && start <= buffered[i].end) { timeRange = buffered[i]; break; } } - const fragEnd = frag.start + frag.duration; + const end = start + (part || frag).duration; if (timeRange) { - timeRange.end = fragEnd; + timeRange.end = end; } else { - timeRange = { - start: fragStart, - end: fragEnd, - }; + timeRange = { start, end }; buffered.push(timeRange); } - this.fragmentTracker.fragBuffered(frag as MediaFragment); - this.fragBufferedComplete(frag, null); - if (this.media) { - this.tick(); + + const isFragHint = !frag.relurl; + const endOfSegmentLoaded = + !part || !isMediaFragment(frag) || (end >= frag.end && !isFragHint); + this.fragBufferedComplete(frag, part); + if (endOfSegmentLoaded) { + if (!this.fragContextChanged(frag)) { + if (isMediaFragment(frag)) { + this.fragmentTracker.fragBuffered(frag); + this.fragPrevious = frag; + } + } + if (this.media) { + this.tickImmediate(); + } } } @@ -184,6 +182,7 @@ export class SubtitleStreamController } data.endOffsetSubtitles = Math.max(0, endOffsetSubtitles); this.tracksBuffered.forEach((buffered) => { + if (!buffered) return; for (let i = 0; i < buffered.length; ) { if (buffered[i].end <= endOffsetSubtitles) { buffered.shift(); @@ -259,13 +258,13 @@ export class SubtitleStreamController } // Check if track has the necessary details to load fragments - const currentTrack = this.levels[this.currentTrackId]; - if (currentTrack?.details) { - this.mediaBuffer = this.mediaBufferTimeRanges; - } else { + const currentTrack = this.levels[this.currentTrackId] as Level | undefined; + if (!currentTrack?.details) { this.mediaBuffer = null; + return; } - if (currentTrack && this.state !== State.STOPPED) { + this.mediaBuffer = this.mediaBufferTimeRanges; + if (this.state !== State.STOPPED) { this.setInterval(TICK_INTERVAL); } } @@ -281,7 +280,7 @@ export class SubtitleStreamController this.warn(`Subtitle tracks were reset while loading level ${trackId}`); return; } - const track: Level = levels[trackId]; + const track = levels[trackId] as Level | undefined; if (trackId >= levels.length || !track) { return; } @@ -346,26 +345,7 @@ export class SubtitleStreamController }); // trigger handler right now - this.tick(); - - // If playlist is misaligned because of bad PDT or drift, delete details to resync with main on reload - if ( - newDetails.live && - !this.fragCurrent && - this.media && - this.state === State.IDLE - ) { - const foundFrag = findFragmentByPTS( - null, - newDetails.fragments, - this.media.currentTime, - 0, - ); - if (!foundFrag) { - this.warn('Subtitle playlist not aligned with playback'); - track.details = undefined; - } - } + this.tickImmediate(); } _handleFragmentLoadComplete(fragLoadedData: FragLoadedData) { @@ -423,18 +403,23 @@ export class SubtitleStreamController } doTick() { - if (!this.media) { - this.state = State.IDLE; - return; - } - if (this.state === State.IDLE) { - const { currentTrackId, levels } = this; - const track = levels?.[currentTrackId]; - if (!track || !levels.length || !track.details) { + if ( + !this.media && + !this.primaryPrefetch && + (this.startFragRequested || !this.config.startFragPrefetch) + ) { return; } - if (this.waitForLive(track)) { + const { currentTrackId, levels } = this; + const track = levels?.[currentTrackId]; + const trackDetails = track?.details; + if ( + !trackDetails || + this.waitForLive(track) || + this.waitForCdnTuneIn(trackDetails) + ) { + this.startFragRequested = false; return; } const { config } = this; @@ -445,62 +430,33 @@ export class SubtitleStreamController config.maxBufferHole, ); const { end: targetBufferTime, len: bufferLen } = bufferedInfo; - const trackDetails = track.details as LevelDetails; const maxBufLen = this.hls.maxBufferLength + trackDetails.levelTargetDuration; - if (bufferLen > maxBufLen) { + if (bufferLen > maxBufLen || (bufferLen && !this.buffering)) { return; } - const fragments = trackDetails.fragments; - const fragLen = fragments.length; - const end = trackDetails.edge; - - let foundFrag: MediaFragment | null = null; - const fragPrevious = this.fragPrevious; - if (targetBufferTime < end) { - const tolerance = config.maxFragLookUpTolerance; - const lookupTolerance = - targetBufferTime > end - tolerance ? 0 : tolerance; - foundFrag = findFragmentByPTS( - fragPrevious, - fragments, - Math.max(fragments[0].start, targetBufferTime), - lookupTolerance, - ); - if ( - !foundFrag && - fragPrevious && - fragPrevious.start < fragments[0].start - ) { - foundFrag = fragments[0]; - } - } else { - foundFrag = fragments[fragLen - 1]; - } - foundFrag = this.filterReplacedPrimary(foundFrag, track.details); - if (!foundFrag) { + let frag = this.getNextFragment(targetBufferTime, trackDetails); + + if (!frag) { return; } // Load earlier fragment in same discontinuity to make up for misaligned playlists and cues that extend beyond end of segment - const curSNIdx = foundFrag.sn - trackDetails.startSN; - const prevFrag = fragments[curSNIdx - 1]; - if ( - prevFrag && - prevFrag.cc === foundFrag.cc && - this.fragmentTracker.getState(prevFrag) === FragmentState.NOT_LOADED - ) { - foundFrag = prevFrag; - } - if ( - this.fragmentTracker.getState(foundFrag) === FragmentState.NOT_LOADED - ) { - // only load if fragment is not loaded - const fragToLoad = this.mapToInitFragWhenRequired(foundFrag); - if (fragToLoad) { - this.loadFragment(fragToLoad, track, targetBufferTime); + if (isMediaFragment(frag)) { + const curSNIdx = frag.sn - trackDetails.startSN; + const prevFrag = trackDetails.fragments[curSNIdx - 1] as + | MediaFragment + | undefined; + if ( + prevFrag && + prevFrag.cc === frag.cc && + !trackDetails.partList?.length && + this.fragmentTracker.getState(prevFrag) === FragmentState.NOT_LOADED + ) { + frag = prevFrag; } } + this.loadFragment(frag, track, targetBufferTime); } } @@ -509,10 +465,17 @@ export class SubtitleStreamController level: Level, targetBufferTime: number, ) { - if (!isMediaFragment(frag)) { - this._loadInitSegment(frag, level); - } else { - super.loadFragment(frag, level, targetBufferTime); + // Check if fragment is not loaded + const fragState = this.fragmentTracker.getState(frag); + if ( + fragState === FragmentState.NOT_LOADED || + (this.loadingParts && isMediaFragment(frag)) + ) { + if (!isMediaFragment(frag)) { + this._loadInitSegment(frag, level); + } else { + super.loadFragment(frag, level, targetBufferTime); + } } } diff --git a/src/controller/subtitle-track-controller.ts b/src/controller/subtitle-track-controller.ts index 5b757336d42..7be7fa7a7d3 100644 --- a/src/controller/subtitle-track-controller.ts +++ b/src/controller/subtitle-track-controller.ts @@ -526,7 +526,6 @@ class SubtitleTrackController extends BasePlaylistController { // and we'll set subtitleTrack when onMediaAttached is triggered if (!this.media) { this.queuedDefaultTrack = newId; - return; } // exit if track id as already set or invalid diff --git a/src/controller/timeline-controller.ts b/src/controller/timeline-controller.ts index b90c2968aba..524684c93ec 100644 --- a/src/controller/timeline-controller.ts +++ b/src/controller/timeline-controller.ts @@ -203,6 +203,7 @@ export class TimelineController implements ComponentAPI { this.hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, { success: false, frag: data.frag, + part: null, error: new Error( 'Subtitle discontinuity domain does not match main', ), @@ -400,7 +401,9 @@ export class TimelineController implements ComponentAPI { const decrypted = 'stats' in data; // If the subtitles are not encrypted, parse VTTs now. Otherwise, we need to wait. if (decryptData == null || !decryptData.encrypted || decrypted) { - const trackPlaylistMedia = this.tracks[frag.level]; + const trackPlaylistMedia = this.tracks[frag.level] as + | MediaPlaylist + | undefined; const vttCCs = this.vttCCs; if (!vttCCs[frag.cc]) { vttCCs[frag.cc] = { @@ -410,24 +413,28 @@ export class TimelineController implements ComponentAPI { }; this.prevCC = frag.cc; } - if (trackPlaylistMedia.textCodec === IMSC1_CODEC) { - this._parseIMSC1(frag, payload); + if (trackPlaylistMedia?.textCodec === IMSC1_CODEC) { + this._parseIMSC1(data); } else { this._parseVTTs(data); } } } else { // In case there is no payload, finish unsuccessfully. + const part = 'part' in data ? data.part : null; this.hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, { success: false, frag, + part, error: new Error('Empty subtitle payload'), }); } } } - private _parseIMSC1(frag: Fragment, payload: ArrayBuffer) { + private _parseIMSC1(data: FragDecryptedData | FragLoadedData) { + const { frag, payload } = data; + const part = 'part' in data ? data.part : null; const hls = this.hls; parseIMSC1( payload, @@ -436,14 +443,16 @@ export class TimelineController implements ComponentAPI { this._appendCues(cues, frag.level); hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, { success: true, - frag: frag, + frag, + part, }); }, (error) => { hls.logger.log(`Failed to parse IMSC1: ${error}`); hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, { success: false, - frag: frag, + frag, + part, error, }); }, @@ -452,6 +461,7 @@ export class TimelineController implements ComponentAPI { private _parseVTTs(data: FragDecryptedData | FragLoadedData) { const { frag, payload } = data; + const part = 'part' in data ? data.part : null; // We need an initial synchronisation PTS. Store fragments as long as none has arrived const { initPTS, unparsedVttFrags } = this; const maxAvCC = initPTS.length - 1; @@ -462,20 +472,22 @@ export class TimelineController implements ComponentAPI { const hls = this.hls; // Parse the WebVTT file contents. - const payloadWebVTT = frag.initSegment?.data - ? appendUint8Array(frag.initSegment.data, new Uint8Array(payload)).buffer + const vttHeader = frag.initSegment?.data; + const payloadWebVTT = vttHeader + ? appendUint8Array(vttHeader, new Uint8Array(payload)).buffer : payload; parseWebVTT( payloadWebVTT, this.initPTS[frag.cc], this.vttCCs, frag.cc, - frag.start, + (part && !vttHeader ? part : frag).start, (cues) => { this._appendCues(cues, frag.level); hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, { success: true, - frag: frag, + frag, + part, }); }, (error) => { @@ -484,7 +496,7 @@ export class TimelineController implements ComponentAPI { if (missingInitPTS) { unparsedVttFrags.push(data); } else if (this.config.enableIMSC1) { - this._fallbackToIMSC1(frag, payload); + this._fallbackToIMSC1(data); } // Something went wrong while parsing. Trigger event with success false. hls.logger.log(`Failed to parse VTT cue: ${error}`); @@ -493,14 +505,16 @@ export class TimelineController implements ComponentAPI { } hls.trigger(Events.SUBTITLE_FRAG_PROCESSED, { success: false, - frag: frag, + frag, + part, error, }); }, ); } - private _fallbackToIMSC1(frag: Fragment, payload: ArrayBuffer) { + private _fallbackToIMSC1(data: FragDecryptedData | FragLoadedData) { + const { frag, payload } = data; // If textCodec is unknown, try parsing as IMSC1. Set textCodec based on the result const trackPlaylistMedia = this.tracks[frag.level]; if (!trackPlaylistMedia.textCodec) { @@ -509,7 +523,7 @@ export class TimelineController implements ComponentAPI { this.initPTS[frag.cc], () => { trackPlaylistMedia.textCodec = IMSC1_CODEC; - this._parseIMSC1(frag, payload); + this._parseIMSC1(data); }, () => { trackPlaylistMedia.textCodec = 'wvtt'; diff --git a/src/loader/fragment.ts b/src/loader/fragment.ts index 621647eb563..790085e1c53 100644 --- a/src/loader/fragment.ts +++ b/src/loader/fragment.ts @@ -1,11 +1,11 @@ import { buildAbsoluteURL } from 'url-toolkit'; import { LoadStats } from './load-stats'; +import { PlaylistLevelType } from '../types/loader'; import type { LevelKey } from './level-key'; import type { FragmentLoaderContext, KeyLoaderContext, Loader, - PlaylistLevelType, } from '../types/loader'; import type { AttrList } from '../utils/attr-list'; import type { KeySystemFormats } from '../utils/mediakeys-helper'; @@ -453,7 +453,8 @@ export class Part extends BaseSegment { return !!( elementaryStreams.audio || elementaryStreams.video || - elementaryStreams.audiovideo + elementaryStreams.audiovideo || + (this.fragment.type === PlaylistLevelType.SUBTITLE && this.stats.loaded) ); } } diff --git a/src/loader/m3u8-parser.ts b/src/loader/m3u8-parser.ts index 1b24c734a60..11e04e1a8e4 100644 --- a/src/loader/m3u8-parser.ts +++ b/src/loader/m3u8-parser.ts @@ -662,7 +662,7 @@ export default class M3U8Parser { if (!partList) { partList = level.partList = []; } - const previousFragmentPart = + const previousPart = currentPart > 0 ? partList[partList.length - 1] : undefined; const index = currentPart++; const partAttrs = new AttrList(value1, level); @@ -671,7 +671,7 @@ export default class M3U8Parser { frag as MediaFragment, base, index, - previousFragmentPart, + previousPart, ); partList.push(part); frag.duration += part.duration; diff --git a/src/types/events.ts b/src/types/events.ts index 186da9ae0f0..63812b74e27 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -284,6 +284,7 @@ export interface TrackSwitchedData { export interface SubtitleFragProcessed { success: boolean; frag: Fragment; + part: Part | null; } export interface FragChangedData { @@ -349,6 +350,7 @@ export interface ErrorData { export interface SubtitleFragProcessedData { success: boolean; frag: Fragment; + part: Part | null; error?: Error; } diff --git a/src/types/vtt.ts b/src/types/vtt.ts index 0c5434970e0..f5044b707a6 100644 --- a/src/types/vtt.ts +++ b/src/types/vtt.ts @@ -1,9 +1,11 @@ export type VTTCCs = { ccOffset: number; presentationOffset: number; - [key: number]: { - start: number; - prevCC: number; - new: boolean; - }; + [key: number]: + | { + start: number; + prevCC: number; + new: boolean; + } + | undefined; }; diff --git a/src/utils/webvtt-parser.ts b/src/utils/webvtt-parser.ts index 57a683ee518..00eb8ae8c20 100644 --- a/src/utils/webvtt-parser.ts +++ b/src/utils/webvtt-parser.ts @@ -8,17 +8,6 @@ import type { VTTCCs } from '../types/vtt'; const LINEBREAKS = /\r\n|\n\r|\n|\r/g; -// String.prototype.startsWith is not supported in IE11 -const startsWith = function ( - inputString: string, - searchString: string, - position: number = 0, -) { - return ( - inputString.slice(position, position + searchString.length) === searchString - ); -}; - const cueString2millis = function (timeString: string) { let ts = parseInt(timeString.slice(-3)); const secs = parseInt(timeString.slice(-6, -4)); @@ -54,30 +43,6 @@ export function generateCueId( return hash(startTime.toString()) + hash(endTime.toString()) + hash(text); } -const calculateOffset = function (vttCCs: VTTCCs, cc, presentationTime) { - let currCC = vttCCs[cc]; - let prevCC = vttCCs[currCC.prevCC]; - - // This is the first discontinuity or cues have been processed since the last discontinuity - // Offset = current discontinuity time - if (!prevCC || (!prevCC.new && currCC.new)) { - vttCCs.ccOffset = vttCCs.presentationOffset = currCC.start; - currCC.new = false; - return; - } - - // There have been discontinuities since cues were last parsed. - // Offset = time elapsed - while (prevCC?.new) { - vttCCs.ccOffset += currCC.start - prevCC.start; - currCC.new = false; - currCC = prevCC; - prevCC = vttCCs[currCC.prevCC]; - } - - vttCCs.presentationOffset = presentationTime; -}; - export function parseWebVTT( vttByteArray: ArrayBuffer, initPTS: TimestampOffset | undefined, @@ -101,7 +66,7 @@ export function parseWebVTT( let cueTime = '00:00.000'; let timestampMapMPEGTS = 0; let timestampMapLOCAL = 0; - let parsingError: Error; + let parsingError: Error | undefined; let inHeader = true; parser.oncue = function (cue: VTTCue) { @@ -114,12 +79,8 @@ export function parseWebVTT( // Update offsets for new discontinuities if (currCC?.new) { - if (timestampMapLOCAL !== undefined) { - // When local time is provided, offset = discontinuity start time - local time - cueOffset = vttCCs.ccOffset = currCC.start; - } else { - calculateOffset(vttCCs, cc, webVttMpegTsMapOffset); - } + // When local time is provided, offset = discontinuity start time - local time + cueOffset = vttCCs.ccOffset = currCC.start; } if (webVttMpegTsMapOffset) { if (!initPTS) { @@ -171,7 +132,7 @@ export function parseWebVTT( vttLines.forEach((line) => { if (inHeader) { // Look for X-TIMESTAMP-MAP in header. - if (startsWith(line, 'X-TIMESTAMP-MAP=')) { + if (line.startsWith('X-TIMESTAMP-MAP=')) { // Once found, no more are allowed anyway, so stop searching. inHeader = false; // Extract LOCAL and MPEGTS. @@ -179,9 +140,9 @@ export function parseWebVTT( .slice(16) .split(',') .forEach((timestamp) => { - if (startsWith(timestamp, 'LOCAL:')) { + if (timestamp.startsWith('LOCAL:')) { cueTime = timestamp.slice(6); - } else if (startsWith(timestamp, 'MPEGTS:')) { + } else if (timestamp.startsWith('MPEGTS:')) { timestampMapMPEGTS = parseInt(timestamp.slice(7)); } }); diff --git a/tests/unit/controller/subtitle-track-controller.ts b/tests/unit/controller/subtitle-track-controller.ts index 086768a2428..e3c010acd17 100644 --- a/tests/unit/controller/subtitle-track-controller.ts +++ b/tests/unit/controller/subtitle-track-controller.ts @@ -52,6 +52,7 @@ describe('SubtitleTrackController', function () { subtitleTrackController = new SubtitleTrackController( hls as unknown as Hls, ); + (hls as any).subtitleTrackController = subtitleTrackController; hls.networkControllers.push(subtitleTrackController); hls.levelController = { levels: [ @@ -654,8 +655,13 @@ describe('SubtitleTrackController', function () { expect(subtitleTracks[1].details).not.to.be.undefined; expect((subtitleTrackController as any).timer).to.equal(-1); - // We will still emit playlist loaded since we did load and store the details - expect(playlistLoadedSpy).to.have.been.called; + // hls.js will not emit playlist loaded since the trackId does not match the loaded event id + expect(playlistLoadedSpy).to.have.not.been.called; + + expect((subtitleTrackController as any).tracksInGroup[1]).not.to.be + .undefined; + expect((subtitleTrackController as any).tracksInGroup[1].details).not.to + .be.undefined; }); it('does not set the reload timer if loading has not started', function () {