FG-2933: Add mp4a.40.2 capture to MCAP recording demo

foxglove · Dec 16, 2024 · 107767e · 107767e
1 parent 0bb7c29
commit 107767e
Show file tree

Hide file tree

Showing 3 changed files with 134 additions and 38 deletions.
diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
@@ -20,6 +20,7 @@ import {
   CompressedAudioData,
   startAudioCapture,
   supportsOpusEncoding,
+  supportsMP4AEncoding,
 } from "./audioCapture";
 import {
   CompressedVideoFrame,
@@ -144,6 +145,7 @@ export function McapRecordingDemo(): JSX.Element {
   const [recordH265, setRecordH265] = useState(false);
   const [recordVP9, setRecordVP9] = useState(false);
   const [recordAV1, setRecordAV1] = useState(false);
+  const [recordMP4A, setRecordMP4A] = useState(false);
   const [recordOpus, setRecordOpus] = useState(false);
   const [recordMouse, setRecordMouse] = useState(true);
   const [recordOrientation, setRecordOrientation] = useState(true);
@@ -164,6 +166,7 @@ export function McapRecordingDemo(): JSX.Element {
   const { data: h265Support } = useAsync(supportsH265Encoding);
   const { data: vp9Support } = useAsync(supportsVP9Encoding);
   const { data: av1Support } = useAsync(supportsAV1Encoding);
+  const { data: mp4aSupport } = useAsync(supportsMP4AEncoding);
   const { data: opusSupport } = useAsync(supportsOpusEncoding);
 
   const canStartRecording =
@@ -174,6 +177,7 @@ export function McapRecordingDemo(): JSX.Element {
     (recordH265 && !videoError) ||
     (recordH264 && !videoError) ||
     (recordJpeg && !videoError) ||
+    (recordMP4A && !audioError) ||
     (recordOpus && !audioError);
 
   // Automatically pause recording after 30 seconds to avoid unbounded growth
@@ -304,7 +308,7 @@ export function McapRecordingDemo(): JSX.Element {
     undefined,
   );
 
-  const enableMicrophone = recordOpus;
+  const enableMicrophone = recordMP4A || recordOpus;
   useEffect(() => {
     const progress = audioProgressRef.current;
     if (!progress || !enableMicrophone) {
@@ -327,14 +331,15 @@ export function McapRecordingDemo(): JSX.Element {
       setAudioStream(undefined);
       setAudioError(undefined);
     };
-  }, [enableMicrophone, recordOpus]);
+  }, [enableMicrophone]);
 
   useEffect(() => {
     if (!enableMicrophone || !recording || !audioStream) {
       return;
     }
 
     const cleanup = startAudioCapture({
+      enableMP4A: recordMP4A,
       enableOpus: recordOpus,
       stream: audioStream,
       onAudioData: (data) => {
@@ -348,7 +353,14 @@ export function McapRecordingDemo(): JSX.Element {
     return () => {
       cleanup?.();
     };
-  }, [addAudioData, enableMicrophone, recordOpus, audioStream, recording]);
+  }, [
+    addAudioData,
+    enableMicrophone,
+    recordMP4A,
+    recordOpus,
+    audioStream,
+    recording,
+  ]);
 
   const onRecordClick = useCallback(
     (event: React.MouseEvent) => {
@@ -488,6 +500,18 @@ export function McapRecordingDemo(): JSX.Element {
             />
             Camera (JPEG)
           </label>
+          {mp4aSupport === true && (
+            <label>
+              <input
+                type="checkbox"
+                checked={recordMP4A}
+                onChange={(event) => {
+                  setRecordMP4A(event.target.checked);
+                }}
+              />
+              Microphone (mp4a.40.2)
+            </label>
+          )}
           {opusSupport === true && (
             <label>
               <input

diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts
@@ -64,6 +64,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
   #vp9ChannelSeq = 0;
   #av1Channel?: ProtobufChannelInfo;
   #av1ChannelSeq = 0;
+  #mp4aChannel?: ProtobufChannelInfo;
+  #mp4aChannelSeq = 0;
   #opusChannel?: ProtobufChannelInfo;
   #opusChannelSeq = 0;
 
@@ -118,6 +120,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     this.#h265ChannelSeq = 0;
     this.#av1Channel = undefined;
     this.#av1ChannelSeq = 0;
+    this.#mp4aChannel = undefined;
+    this.#mp4aChannelSeq = 0;
     this.#opusChannel = undefined;
     this.#opusChannelSeq = 0;
   }
@@ -293,6 +297,14 @@ export class Recorder extends EventEmitter<RecorderEvents> {
       let channel: ProtobufChannelInfo;
       let sequence: number;
       switch (data.format) {
+        case "mp4a.40.2":
+          channel = this.#mp4aChannel ??= await addProtobufChannel(
+            this.#writer,
+            "microphone_mp4a",
+            foxgloveMessageSchemas.CompressedAudio,
+          );
+          sequence = this.#mp4aChannelSeq++;
+          break;
         case "opus":
           channel = this.#opusChannel ??= await addProtobufChannel(
             this.#writer,

diff --git a/website/src/components/McapRecordingDemo/audioCapture.ts b/website/src/components/McapRecordingDemo/audioCapture.ts
@@ -117,7 +117,66 @@ export const supportsOpusEncoding = async (): Promise<boolean> => {
   return support.supported === true;
 };
 
-type CompressedAudioFormat = "opus";
+const DEFAULT_MP4A_CONFIG: AudioEncoderConfig = {
+  codec: "mp4a.40.2",
+  numberOfChannels: 1,
+  sampleRate: 48000,
+};
+
+/**
+ * Determine whether AudioEncoder can be used to encode audio with mp4a.40.2.
+ */
+export const supportsMP4AEncoding = async (): Promise<boolean> => {
+  if (!supportsMediaCaptureTransformAndWebCodecs()) {
+    return false;
+  }
+
+  const support = await AudioEncoder.isConfigSupported(DEFAULT_MP4A_CONFIG);
+  return support.supported === true;
+};
+
+const configureEncoder = ({
+  config,
+  framePool,
+  onAudioData,
+  onError,
+}: {
+  config: AudioEncoderConfig;
+  framePool: ArrayBuffer[];
+  onAudioData: (data: CompressedAudioData) => void;
+  onError: (error: Error) => void;
+}): AudioEncoder => {
+  const encoder = new AudioEncoder({
+    output: (chunk) => {
+      let buffer = framePool.pop();
+      if (!buffer || buffer.byteLength < chunk.byteLength) {
+        buffer = new ArrayBuffer(chunk.byteLength);
+      }
+      chunk.copyTo(buffer);
+      onAudioData({
+        format: config.codec as CompressedAudioFormat,
+        type: chunk.type as CompressedAudioType,
+        timestamp: chunk.timestamp,
+        data: new Uint8Array(buffer, 0, chunk.byteLength),
+        sampleRate: config.sampleRate,
+        numberOfChannels: config.numberOfChannels,
+        release() {
+          if (buffer) {
+            framePool.push(buffer);
+          }
+        },
+      });
+    },
+    error: (error) => {
+      onError(error);
+    },
+  });
+  encoder.configure(config);
+
+  return encoder;
+};
+
+type CompressedAudioFormat = "opus" | "mp4a.40.2";
 type CompressedAudioType = "key" | "delta";
 export type CompressedAudioData = {
   format: CompressedAudioFormat;
@@ -132,6 +191,7 @@ export type CompressedAudioData = {
 };
 
 interface AudioCaptureParams {
+  enableMP4A: boolean;
   enableOpus: boolean;
   /** MediaStream from startAudioStream */
   stream: MediaStream;
@@ -141,12 +201,13 @@ interface AudioCaptureParams {
 }
 
 export function startAudioCapture({
+  enableMP4A,
   enableOpus,
   stream,
   onAudioData,
   onError,
 }: AudioCaptureParams): (() => void) | undefined {
-  if (!enableOpus) {
+  if (!enableMP4A && !enableOpus) {
     onError(new Error("Invariant: expected Opus encoding to be enabled"));
     return undefined;
   }
@@ -170,52 +231,49 @@ export function startAudioCapture({
     track,
   });
 
-  const settings = track.getSettings();
   const framePool: ArrayBuffer[] = [];
 
-  const encoder = new AudioEncoder({
-    output: (chunk) => {
-      let buffer = framePool.pop();
-      if (!buffer || buffer.byteLength < chunk.byteLength) {
-        buffer = new ArrayBuffer(chunk.byteLength);
-      }
-      chunk.copyTo(buffer);
-      onAudioData({
-        format: "opus",
-        type: chunk.type as CompressedAudioType,
-        timestamp: chunk.timestamp,
-        data: new Uint8Array(buffer, 0, chunk.byteLength),
-        sampleRate: settings.sampleRate ?? 0,
-        numberOfChannels: settings.channelCount ?? 0,
-        release() {
-          if (buffer) {
-            framePool.push(buffer);
-          }
-        },
-      });
-    },
-    error: (error) => {
-      onError(error);
-    },
-  });
-  encoder.configure({
-    codec: "opus",
-    sampleRate: settings.sampleRate ?? 0,
-    numberOfChannels: settings.channelCount ?? 0,
-  });
+  const encoders = [
+    ...(enableMP4A
+      ? [
+          configureEncoder({
+            config: DEFAULT_MP4A_CONFIG,
+            framePool,
+            onAudioData,
+            onError,
+          }),
+        ]
+      : []),
+    ...(enableOpus
+      ? [
+          configureEncoder({
+            config: DEFAULT_OPUS_CONFIG,
+            framePool,
+            onAudioData,
+            onError,
+          }),
+        ]
+      : []),
+  ];
 
   const reader = trackProcessor.readable.getReader();
   let canceled = false;
 
   const readAndEncode = () => {
+    if (canceled) {
+      return;
+    }
+
     reader
       .read()
       .then((result) => {
         if (result.done || canceled) {
           return;
         }
 
-        encoder.encode(result.value);
+        for (const encoder of encoders) {
+          encoder.encode(result.value);
+        }
 
         readAndEncode();
       })
@@ -228,6 +286,8 @@ export function startAudioCapture({
 
   return () => {
     canceled = true;
-    encoder.close();
+    for (const encoder of encoders) {
+      encoder.close();
+    }
   };
 }