From 5ab318b5a9bb251d44d7f38c8b733ae4331e1242 Mon Sep 17 00:00:00 2001 From: R-J Lim Date: Sat, 7 Sep 2024 11:42:04 +0900 Subject: [PATCH] Fix issue where firefox audio gets louder on each recording - Maintain single captured stream per video --- extension/src/services/audio-recorder.ts | 32 ++++++++++++++++------- extension/src/services/binding.ts | 33 ++++++++++++++---------- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/extension/src/services/audio-recorder.ts b/extension/src/services/audio-recorder.ts index 70a91cc3..7bdb0aa7 100755 --- a/extension/src/services/audio-recorder.ts +++ b/extension/src/services/audio-recorder.ts @@ -17,7 +17,12 @@ export default class AudioRecorder { this.blobPromise = null; } - startWithTimeout(stream: MediaStream, time: number, onStartedCallback: () => void): Promise { + startWithTimeout( + stream: MediaStream, + time: number, + onStartedCallback: () => void, + doNotManageStream: boolean = false + ): Promise { return new Promise(async (resolve, reject) => { try { if (this.recording) { @@ -26,12 +31,12 @@ export default class AudioRecorder { return; } - await this.start(stream); + await this.start(stream, doNotManageStream); onStartedCallback(); this.timeoutResolve = resolve; this.timeoutId = setTimeout(async () => { this.timeoutId = undefined; - resolve(await this.stop()); + resolve(await this.stop(doNotManageStream)); }, time); } catch (e) { reject(e); @@ -39,7 +44,7 @@ export default class AudioRecorder { }); } - start(stream: MediaStream): Promise { + start(stream: MediaStream, doNotManageStream: boolean = false): Promise { return new Promise((resolve, reject) => { if (this.recording) { reject('Already recording, cannot start'); @@ -58,9 +63,12 @@ export default class AudioRecorder { }; }); recorder.start(); - const output = new AudioContext(); - const source = output.createMediaStreamSource(stream); - source.connect(output.destination); + + if (!doNotManageStream) { + const output = new AudioContext(); + const source = output.createMediaStreamSource(stream); + source.connect(output.destination); + } this.recorder = recorder; this.recording = true; @@ -72,7 +80,7 @@ export default class AudioRecorder { }); } - async stop(): Promise { + async stop(doNotManageStream: boolean = false): Promise { if (!this.recording) { throw new Error('Not recording, unable to stop'); } @@ -80,8 +88,12 @@ export default class AudioRecorder { this.recording = false; this.recorder?.stop(); this.recorder = null; - this.stream?.getTracks()?.forEach((t) => t.stop()); - this.stream = null; + + if (!doNotManageStream) { + this.stream?.getTracks()?.forEach((t) => t.stop()); + this.stream = null; + } + const blob = await this.blobPromise; this.blobPromise = null; const base64 = await bufferToBase64(await blob!.arrayBuffer()); diff --git a/extension/src/services/binding.ts b/extension/src/services/binding.ts index 576643f8..531029c5 100755 --- a/extension/src/services/binding.ts +++ b/extension/src/services/binding.ts @@ -144,6 +144,10 @@ export default class Binding { ) => void; private heartbeatInterval?: NodeJS.Timeout; + // In the case of firefox, we need to avoid capturing the audio stream more than once, + // so we keep a reference to the first one we capture here. + private audioStream?: MediaStream; + private readonly frameId?: string; constructor(video: HTMLMediaElement, syncAvailable: boolean, frameId?: string) { @@ -740,13 +744,13 @@ export default class Binding { this._audioRecorder.startWithTimeout( stream, startRecordingAudioWithTimeoutMessage.timeout, - () => sendResponse(true) + () => sendResponse(true), + true ) ) .then((audioBase64) => this._sendAudioBase64(audioBase64, startRecordingAudioWithTimeoutMessage.preferMp3) ) - .then(() => this._resumeAudioAfterRecording()) .catch((e) => { console.error(e instanceof Error ? e.message : String(e)); sendResponse(false); @@ -754,9 +758,8 @@ export default class Binding { return true; case 'start-recording-audio': this._captureStream() - .then((stream) => this._audioRecorder.start(stream)) + .then((stream) => this._audioRecorder.start(stream, true)) .then(() => sendResponse(true)) - .then(() => this._resumeAudioAfterRecording()) .catch((e) => { console.error(e instanceof Error ? e.message : String(e)); sendResponse(false); @@ -765,7 +768,7 @@ export default class Binding { case 'stop-recording-audio': const stopRecordingAudioMessage = request.message as StopRecordingAudioMessage; this._audioRecorder - .stop() + .stop(true) .then((audioBase64) => { sendResponse(true); this._sendAudioBase64(audioBase64, stopRecordingAudioMessage.preferMp3); @@ -1327,6 +1330,11 @@ export default class Binding { private _captureStream(): Promise { return new Promise((resolve, reject) => { + if (this.audioStream !== undefined) { + resolve(this.audioStream); + return; + } + try { let stream: MediaStream | undefined; @@ -1355,6 +1363,12 @@ export default class Binding { } } + // Ensure audio keeps playing through computer speakers + const output = new AudioContext(); + const source = output.createMediaStreamSource(audioStream); + source.connect(output.destination); + + this.audioStream = audioStream; resolve(audioStream); } catch (e) { reject(e); @@ -1362,15 +1376,6 @@ export default class Binding { }); } - private async _resumeAudioAfterRecording() { - // On Firefox the audio is muted once audio recording is stopped. - // Below is a hack to resume the audio. - const stream = await this._captureStream(); - const output = new AudioContext(); - const source = output.createMediaStreamSource(stream); - source.connect(output.destination); - } - private async _sendAudioBase64(base64: string, preferMp3: boolean) { if (preferMp3) { const blob = await (await fetch('data:audio/webm;base64,' + base64)).blob();