feat(skeleton-viewer): render video using VideoEncoder (#119)

* feat(skeleton-viewer): render video

* feat(skeleton-viewer): render video using VideoEncoder

CHANGELOG.md

@@ -6,11 +6,21 @@ This file is used to list important changes made over time to `sign/translate`.
 
 ### 0.0.3
 
+#### Documentation
+
+- **wiki**: redo wiki to be more transparent about the project's progress
+-
+
 #### Features
 
 - **api**: add text-normalization API with appCheck support
 - **spoken-to-signed**: add language detection suggestion "Translate from: \_\_\_"
 - **spoken-to-signed**: add text edit suggestions "Did you mean: \_\_\_"
+- **core**: upgraded to angular 17, and all relevant dependencies
+
+#### Fixes
+
+- **normalization**: cancel normalization requests when user changes text
 
 ## Released
 

README.md

@@ -23,6 +23,9 @@
   <a href="https://github.com/sign/translate/actions/workflows/client.yml">
     <img src="https://github.com/sign/translate/actions/workflows/client.yml/badge.svg" alt="Client Build Test Status Badge" />
   </a>
+  <a href="https://github.com/sign/translate/actions/workflows/server.yml">
+    <img src="https://github.com/sign/translate/actions/workflows/server.yml/badge.svg" alt="Server Build Test Status Badge" />
+  </a>
   <a href="https://coveralls.io/github/sign/translate?branch=master">
     <img src="https://coveralls.io/repos/github/sign/translate/badge.svg?branch=master" alt="Coverage Status Badge" />
   </a>
@@ -137,9 +140,9 @@ npm test
 Run the application on iOS:
 
 ```bash
-bun run build && \
-bun x cap sync && \
-bun x cap run ios
+npm run build && \
+npx cap sync && \
+npx cap run ios
 ```
 
 [node.js]: https://nodejs.org/

functions/package.json

@@ -29,7 +29,7 @@
     "firebase-functions": "4.5.0",
     "http-errors": "2.0.0",
     "node-fetch": "2.6.7",
-    "openai": "4.17.0"
+    "openai": "4.17.4"
   },
   "devDependencies": {
     "@firebase/firestore-types": "3.0.0",

package.json

@@ -33,20 +33,20 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/animations": "17.0.1",
+    "@angular/animations": "17.0.2",
     "@angular/cdk": "17.0.0",
-    "@angular/common": "17.0.1",
-    "@angular/compiler": "17.0.1",
-    "@angular/core": "17.0.1",
-    "@angular/forms": "17.0.1",
+    "@angular/common": "17.0.2",
+    "@angular/compiler": "17.0.2",
+    "@angular/core": "17.0.2",
+    "@angular/forms": "17.0.2",
     "@angular/material": "17.0.0",
-    "@angular/platform-browser": "17.0.1",
-    "@angular/platform-browser-dynamic": "17.0.1",
-    "@angular/platform-server": "17.0.1",
-    "@angular/router": "17.0.1",
-    "@angular/service-worker": "17.0.1",
+    "@angular/platform-browser": "17.0.2",
+    "@angular/platform-browser-dynamic": "17.0.2",
+    "@angular/platform-server": "17.0.2",
+    "@angular/router": "17.0.2",
+    "@angular/service-worker": "17.0.2",
     "@angular/ssr": "^17.0.0",
-    "@asymmetrik/ngx-leaflet": "16.0.1",
+    "@asymmetrik/ngx-leaflet": "17.0.0",
     "@capacitor-firebase/analytics": "5.2.0",
     "@capacitor-firebase/app": "5.2.0",
     "@capacitor-firebase/app-check": "5.2.0",
@@ -82,13 +82,13 @@
     "cld3-asm": "3.1.1",
     "comlink": "4.4.1",
     "filesize": "9.0.11",
-    "firebase": "10.5.2",
+    "firebase": "10.6.0",
     "flag-icons": "6.15.0",
     "ionicons": "7.2.1",
     "leaflet": "1.9.4",
     "mp4-muxer": "3.0.2",
     "ngx-filesize": "3.0.2",
-    "pose-viewer": "0.7.2",
+    "pose-viewer": "0.7.4",
     "rxjs": "7.8.1",
     "stats.js": "0.17.0",
     "three": "0.158.0",
@@ -106,7 +106,7 @@
     "@angular-eslint/schematics": "17.0.1",
     "@angular-eslint/template-parser": "17.0.1",
     "@angular/cli": "17.0.0",
-    "@angular/compiler-cli": "17.0.1",
+    "@angular/compiler-cli": "17.0.2",
     "@capacitor/assets": "3.0.1",
     "@capacitor/cli": "5.5.1",
     "@ionic/angular-server": "7.5.4",
@@ -149,7 +149,7 @@
     "sitemap": "7.1.1",
     "tiny-async-pool": "2.1.0",
     "ts-node": "10.9.1",
-    "tsx": "4.0.0",
+    "tsx": "4.1.1",
     "typescript": "5.2.2",
     "webpack-bundle-analyzer": "4.9.1",
     "zod": "3.22.4"

src/app/components/speech-to-text/speech-to-text.component.ts

@@ -74,7 +74,6 @@ export class SpeechToTextComponent extends BaseComponent implements OnInit, OnCh
   }
 
   requestPermission() {
-    alert();
     navigator.mediaDevices.getUserMedia({video: false, audio: true}).then(stream => {
       stream.getTracks().forEach(track => track.stop());
       this.supportError = null;

src/app/pages/translate/pose-viewers/human-pose-viewer/human-pose-viewer.component.ts

@@ -5,7 +5,6 @@ import {takeUntil, tap} from 'rxjs/operators';
 import {BasePoseViewerComponent} from '../pose-viewer.component';
 import {Store} from '@ngxs/store';
 import {transferableImage} from '../../../../core/helpers/image/transferable';
-import {wait} from '../../../../core/helpers/wait/wait';
 
 @Component({
   selector: 'app-human-pose-viewer',
@@ -92,9 +91,10 @@ export class HumanPoseViewerComponent extends BasePoseViewerComponent implements
     this.modelReady = true; // Stop loading after first model inference
 
     const imageData = new ImageData(uint8Array, canvas.width, canvas.height);
-    await this.addCacheFrame(imageData);
-
     ctx.putImageData(imageData, 0, 0);
+
+    const imageBitmap = await createImageBitmap(imageData);
+    await this.addCacheFrame(imageBitmap);
   }
 
   override reset(): void {
@@ -124,7 +124,12 @@ export class HumanPoseViewerComponent extends BasePoseViewerComponent implements
         tap(() => {
           i++;
           if (i < this.cache.length) {
-            ctx.putImageData(this.cache[i], 0, 0);
+            ctx.clearRect(0, 0, canvas.width, canvas.height);
+            if (this.background) {
+              ctx.fillStyle = this.background;
+              ctx.fillRect(0, 0, canvas.width, canvas.height);
+            }
+            ctx.drawImage(this.cache[i], 0, 0);
             delete this.cache[i]; // Free up memory after cached frame is no longer necessary
           } else {
             this.cacheSubscription.unsubscribe();

src/app/pages/translate/pose-viewers/pose-viewer.component.ts

@@ -4,12 +4,17 @@ import {fromEvent, Subscription} from 'rxjs';
 import {takeUntil, tap} from 'rxjs/operators';
 import {Store} from '@ngxs/store';
 import {SetSignedLanguageVideo} from '../../../modules/translate/translate.actions';
-import {ArrayBufferTarget as WebmArrayBufferTarget, Muxer as WebmMuxer} from 'webm-muxer';
-import {Muxer as Mp4Muxer, ArrayBufferTarget as Mp4ArrayBufferTarget} from 'mp4-muxer';
+import type {ArrayBufferTarget as WebmArrayBufferTarget, Muxer as WebmMuxer} from 'webm-muxer';
+import type {ArrayBufferTarget as Mp4ArrayBufferTarget, Muxer as Mp4Muxer} from 'mp4-muxer';
 import {isSafari} from '../../../core/constants';
 
 const BPS = 1_000_000_000; // 1GBps, to act as infinity
 
+interface VideoCodecInfo {
+  codec: string;
+  forceEvenSizedFrames: boolean;
+}
+
 @Component({
   selector: 'app-pose-viewer',
   template: ``,
@@ -18,17 +23,20 @@ const BPS = 1_000_000_000; // 1GBps, to act as infinity
 export abstract class BasePoseViewerComponent extends BaseComponent implements OnInit, OnDestroy {
   @ViewChild('poseViewer') poseEl: ElementRef<HTMLPoseViewerElement>;
 
+  background: string = '';
+
   // Using cache and MediaRecorder for older browsers, and safari
-  mimeTypes = ['video/webm;codecs:vp9', 'video/webm;codecs:vp8', 'video/webm', 'video/mp4', 'video/ogv'];
+  mimeTypes = ['video/webm; codecs:vp9', 'video/webm; codecs:vp8', 'video/webm', 'video/mp4', 'video/ogv'];
   mediaRecorder: MediaRecorder;
   mediaSubscriptions: Subscription[] = [];
 
   // Use a video encoder on supported browsers
+  supportsVideoEncoder = 'VideoEncoder' in window;
   videoEncoder: VideoEncoder;
   videoType: 'webm' | 'mp4';
   muxer: WebmMuxer<WebmArrayBufferTarget> | Mp4Muxer<Mp4ArrayBufferTarget>;
 
-  cache: ImageData[] = [];
+  cache: ImageBitmap[] = [];
   cacheSubscription: Subscription;
 
   frameIndex = 0;
@@ -40,6 +48,16 @@ export abstract class BasePoseViewerComponent extends BaseComponent implements O
   }
 
   async ngOnInit() {
+    // Some browsers videos can't have a transparent background
+    const isTransparencySupported =
+      'chrome' in window && // transparency is currently not supported in firefox and safari
+      !this.supportsVideoEncoder; // alpha is not yet supported in chrome VideoEncoder
+    if (!isTransparencySupported) {
+      // Make the video background the same as the element's background
+      const el = document.querySelector('app-signed-language-output');
+      this.background = getComputedStyle(el).backgroundColor;
+    }
+
     await this.definePoseViewerElement();
   }
 
@@ -70,7 +88,23 @@ export abstract class BasePoseViewerComponent extends BaseComponent implements O
     return pose.body.fps;
   }
 
-  async createMuxer(image: ImageData): Promise<string> {
+  videoDimensions(image: ImageBitmap, forceEvenSizedFrames: boolean = false) {
+    let width = image.width;
+    let height = image.height;
+
+    if (forceEvenSizedFrames) {
+      if (image.width % 2 !== 0) {
+        width += 1;
+      }
+      if (image.height % 2 !== 0) {
+        height += 1;
+      }
+    }
+
+    return {width, height};
+  }
+
+  async createMuxer(image: ImageBitmap): Promise<VideoCodecInfo> {
     // Creates the muxer and returns the relevant codec
 
     const fps = await this.fps();
@@ -80,17 +114,21 @@ export abstract class BasePoseViewerComponent extends BaseComponent implements O
     if (isSafari) {
       const {Muxer, ArrayBufferTarget} = await import('mp4-muxer');
       this.videoType = 'mp4';
+
       this.muxer = new Muxer({
         target: new ArrayBufferTarget(),
         fastStart: 'in-memory',
         video: {
           codec: 'avc',
-          width: image.width,
-          height: image.height,
+          ...this.videoDimensions(image, true),
         },
       });
 
-      return 'avc1.42001f';
+      return {
+        codec: 'avc1.42001f',
+        // H264 only supports even sized frames
+        forceEvenSizedFrames: true,
+      };
     }
 
     const {Muxer, ArrayBufferTarget} = await import('webm-muxer');
@@ -99,31 +137,33 @@ export abstract class BasePoseViewerComponent extends BaseComponent implements O
       target: new ArrayBufferTarget(),
       video: {
         codec: 'V_VP9',
-        width: image.width,
-        height: image.height,
+        ...this.videoDimensions(image),
         frameRate: fps,
         alpha: true,
       },
     });
 
-    return 'vp09.00.10.08';
+    return {
+      codec: 'avc1.42001f',
+      forceEvenSizedFrames: false,
+    };
   }
 
-  async initVideoEncoder(image: ImageData) {
-    const codec = await this.createMuxer(image);
+  async initVideoEncoder(image: ImageBitmap) {
+    const {codec, forceEvenSizedFrames} = await this.createMuxer(image);
 
     this.videoEncoder = new VideoEncoder({
       output: (chunk, meta) => this.muxer.addVideoChunk(chunk, meta),
       error: e => console.error(e),
     });
-    this.videoEncoder.configure({
+    const config = {
       codec,
-      width: image.width,
-      height: image.height,
+      ...this.videoDimensions(image, forceEvenSizedFrames),
       bitrate: BPS,
       framerate: await this.fps(),
-      // alpha: 'keep' TODO: this is not yet supported in Chrome https://chromium.googlesource.com/chromium/src/+/master/third_party/blink/renderer/modules/webcodecs/video_encoder.cc#242
-    });
+      // alpha: 'keep' as AlphaOption // TODO: this is not yet supported in Chrome https://chromium.googlesource.com/chromium/src/+/master/third_party/blink/renderer/modules/webcodecs/video_encoder.cc#242
+    };
+    this.videoEncoder.configure(config);
   }
 
   async createEncodedVideo() {
@@ -203,14 +243,14 @@ export abstract class BasePoseViewerComponent extends BaseComponent implements O
     }
   }
 
-  async addCacheFrame(image: ImageData): Promise<void> {
-    if ('VideoEncoder' in window) {
+  async addCacheFrame(image: ImageBitmap): Promise<void> {
+    if (this.supportsVideoEncoder) {
       if (!this.videoEncoder) {
         await this.initVideoEncoder(image);
       }
       const ms = 1_000_000; // 1µs
       const fps = await this.fps();
-      const frame = new VideoFrame(await createImageBitmap(image), {
+      const frame = new VideoFrame(image, {
         timestamp: (ms * this.frameIndex) / fps,
         duration: ms / fps,
       });