feat(spoken-to-signed): add language detection suggestions

fix(language-selector): correctly apply language change to UI
fix(language-selector): fix "detected" languages in non-Enlgish UI languages

functions/package.json

@@ -20,7 +20,7 @@
   "main": "lib/index.js",
   "dependencies": {
     "@firebase/database-types": "^1.0.0",
-    "@google-cloud/storage": "^7.3.0",
+    "@google-cloud/storage": "^7.3.1",
     "@sign-mt/browsermt": "^0.2.1",
     "cors": "^2.8.5",
     "express": "^4.18.2",
@@ -38,7 +38,7 @@
     "@types/node-fetch": "^2.6.7",
     "@typescript-eslint/eslint-plugin": "^6.8.0",
     "@typescript-eslint/parser": "^6.8.0",
-    "eslint": "^8.51.0",
+    "eslint": "^8.52.0",
     "firebase-functions-test": "^3.1.0",
     "firebase-tools": "^12.7.0",
     "jest": "^29.7.0",

package.json

@@ -97,14 +97,14 @@
     "zone.js": "0.14.0"
   },
   "devDependencies": {
-    "@angular-devkit/architect": "0.1602.6",
-    "@angular-devkit/build-angular": "16.2.6",
+    "@angular-devkit/architect": "0.1602.7",
+    "@angular-devkit/build-angular": "16.2.7",
     "@angular-eslint/builder": "16.2.0",
     "@angular-eslint/eslint-plugin": "16.2.0",
     "@angular-eslint/eslint-plugin-template": "16.2.0",
     "@angular-eslint/schematics": "16.2.0",
     "@angular-eslint/template-parser": "16.2.0",
-    "@angular/cli": "16.2.6",
+    "@angular/cli": "16.2.7",
     "@angular/compiler-cli": "16.2.10",
     "@capacitor/assets": "3.0.1",
     "@capacitor/cli": "5.5.0",
@@ -127,7 +127,7 @@
     "@typescript-eslint/parser": "6.8.0",
     "deepmerge": "4.3.1",
     "dotenv": "16.3.1",
-    "eslint": "8.51.0",
+    "eslint": "8.52.0",
     "fuzzy": "0.1.3",
     "husky": "8.0.3",
     "inquirer": "9.2.11",

src/app/modules/translate/translate.state.ts

@@ -69,7 +69,18 @@ export class TranslateState implements NgxsOnInit {
     this.poseViewerSetting$ = this.store.select<PoseViewerSetting>(state => state.settings.poseViewer);
   }
 
-  ngxsOnInit({dispatch}: StateContext<TranslateStateModel>): any {
+  ngxsOnInit({dispatch, patchState}: StateContext<TranslateStateModel>): any {
+    const searchParams = 'window' in globalThis ? window.location.search : '';
+    const urlParams = new URLSearchParams(searchParams);
+    const urlSignedLanguage = urlParams.get('sil');
+    if (urlSignedLanguage) {
+      patchState({signedLanguage: urlSignedLanguage});
+    }
+    const urlSpokenLanguage = urlParams.get('spl');
+    if (urlSpokenLanguage) {
+      patchState({spokenLanguage: urlSpokenLanguage});
+    }
+
     dispatch(ChangeTranslation);
 
     // Reset video whenever viewer setting changes
@@ -118,21 +129,28 @@ export class TranslateState implements NgxsOnInit {
     }
   }
 
+  async detectLanguage(spokenLanguageText: string, patchState: StateContext<TranslateStateModel>['patchState']) {
+    if (spokenLanguageText.length === 0) {
+      patchState({detectedLanguage: null});
+      return;
+    }
+
+    await this.service.initCld();
+    const detectedLanguage = await this.service.detectSpokenLanguage(spokenLanguageText);
+    patchState({detectedLanguage});
+  }
+
   @Action(SetSpokenLanguage)
   async setSpokenLanguage(
     {patchState, getState, dispatch}: StateContext<TranslateStateModel>,
     {language}: SetSpokenLanguage
   ): Promise<void> {
-    patchState({spokenLanguage: language, detectedLanguage: null});
+    patchState({spokenLanguage: language});
 
     // Load and apply language detection if selected
     if (!language) {
-      await this.service.initCld();
       const {spokenLanguageText} = getState();
-      if (spokenLanguageText) {
-        const detectedLanguage = await this.service.detectSpokenLanguage(spokenLanguageText);
-        patchState({detectedLanguage});
-      }
+      await this.detectLanguage(spokenLanguageText, patchState);
     }
 
     dispatch(ChangeTranslation);
@@ -154,10 +172,14 @@ export class TranslateState implements NgxsOnInit {
   ): Promise<void> {
     const {spokenLanguage} = getState();
     const trimmedText = text.trim();
-    patchState({
-      spokenLanguageText: text,
-      detectedLanguage: !trimmedText || spokenLanguage ? null : await this.service.detectSpokenLanguage(trimmedText),
-    });
+
+    patchState({spokenLanguageText: text});
+    const detectLanguage = this.detectLanguage(trimmedText, patchState);
+
+    // Wait for language detection if language is not selected
+    if (!spokenLanguage) {
+      await detectLanguage;
+    }
 
     dispatch(ChangeTranslation);
   }

src/app/pages/translate/language-selector/language-selector.component.html

@@ -4,7 +4,7 @@
     [(selectedIndex)]="selectedIndex"
     [disablePagination]="true">
     <ng-container *ngIf="hasLanguageDetection">
-      <ng-container *ngIf="detectedLanguage$ | async as detectedLanguage; else detectLanguage">
+      <ng-container *ngIf="detectedLanguage && !language; else detectLanguage">
         <mat-tab>
           <ng-template mat-tab-label>
             {{ t('detected', {lang: langNames[detectedLanguage]}) }}
@@ -24,7 +24,7 @@
   </mat-tab-group>
 
   <!-- Language button for small screens -->
-  <ng-container *ngIf="hasLanguageDetection && detectedLanguage$ | async as detectedLanguage; else detectLanguage">
+  <ng-container *ngIf="hasLanguageDetection && detectedLanguage && !language; else detectLanguage">
     <ion-button
       [matMenuTriggerFor]="signedLanguagesMenu"
       class="menu-language-button"

src/app/pages/translate/language-selector/language-selector.component.ts

@@ -1,6 +1,6 @@
-import {Component, EventEmitter, Input, OnInit, Output} from '@angular/core';
+import {Component, EventEmitter, Input, OnChanges, OnInit, Output, SimpleChanges} from '@angular/core';
 import {Store} from '@ngxs/store';
-import {Observable, switchMap} from 'rxjs';
+import {switchMap} from 'rxjs';
 import {TranslocoService} from '@ngneat/transloco';
 import {filter, takeUntil, tap} from 'rxjs/operators';
 import {BaseComponent} from '../../../components/base/base.component';
@@ -13,16 +13,15 @@ const IntlTypeMap: {[key: string]: Intl.DisplayNamesType} = {languages: 'languag
   templateUrl: './language-selector.component.html',
   styleUrls: ['./language-selector.component.scss'],
 })
-export class LanguageSelectorComponent extends BaseComponent implements OnInit {
-  detectedLanguage$: Observable<string>;
+export class LanguageSelectorComponent extends BaseComponent implements OnInit, OnChanges {
+  detectedLanguage: string;
 
   @Input() flags = false;
   @Input() hasLanguageDetection = false;
   @Input() languages: string[];
   @Input() translationKey: string;
-  @Input() urlParameter: string;
 
-  @Input() language: string;
+  @Input() language: string | null;
 
   @Output() languageChange = new EventEmitter<string>();
 
@@ -35,17 +34,12 @@ export class LanguageSelectorComponent extends BaseComponent implements OnInit {
 
   constructor(private store: Store, private transloco: TranslocoService) {
     super();
-
-    this.detectedLanguage$ = this.store.select<string>(state => state.translate.detectedLanguage);
   }
 
   ngOnInit(): void {
-    this.topLanguages = this.languages.slice(0, 3);
-
-    const searchParams = 'window' in globalThis ? window.location.search : '';
-    const urlParams = new URLSearchParams(searchParams);
-    const initial = urlParams.get(this.urlParameter) || this.languages[0];
-    this.selectLanguage(initial);
+    if (!this.language) {
+      this.selectLanguage(this.languages[0]);
+    }
 
     // Initialize langNames, relevant for SSR
     this.setLangNames(this.transloco.getActiveLang());
@@ -60,6 +54,21 @@ export class LanguageSelectorComponent extends BaseComponent implements OnInit {
       .subscribe();
 
     this.setLangCountries();
+
+    // Get detected language
+    this.store
+      .select<string>(state => state.translate.detectedLanguage)
+      .pipe(
+        tap(detectedLanguage => (this.detectedLanguage = detectedLanguage)),
+        takeUntil(this.ngUnsubscribe)
+      )
+      .subscribe();
+  }
+
+  ngOnChanges(changes: SimpleChanges) {
+    if (changes.language) {
+      this.selectLanguage(changes.language.currentValue);
+    }
   }
 
   langName(lang: string): string {
@@ -100,19 +109,21 @@ export class LanguageSelectorComponent extends BaseComponent implements OnInit {
   }
 
   selectLanguage(lang: string): void {
-    if (lang === this.language) {
-      return;
+    if (!this.topLanguages) {
+      this.topLanguages = this.languages.slice(0, 3);
+    }
+
+    if (lang !== this.language) {
+      // Update selected language
+      this.language = lang;
+      this.languageChange.emit(this.language);
     }
 
     if (lang && !this.topLanguages.includes(lang)) {
       this.topLanguages.unshift(lang);
       this.topLanguages.pop();
     }
 
-    // Update selected language
-    this.language = lang;
-    this.languageChange.emit(this.language);
-
     const index = this.topLanguages.indexOf(this.language);
     this.selectedIndex = index + Number(this.hasLanguageDetection);
   }

src/app/pages/translate/language-selectors/language-selectors.component.html

@@ -2,8 +2,8 @@
   [flags]="true"
   [hasLanguageDetection]="false"
   [languages]="translation.signedLanguages"
+  [language]="signedLanguage$ | async"
   translationKey="signedLanguagesShort"
-  urlParameter="sil"
   (languageChange)="setSignedLanguage($event)"></app-language-selector>
 
 <ion-button
@@ -21,6 +21,6 @@
 <app-language-selector
   [hasLanguageDetection]="spokenToSigned"
   [languages]="translation.spokenLanguages"
+  [language]="spokenLanguage$ | async"
   translationKey="languages"
-  urlParameter="spl"
   (languageChange)="setSpokenLanguage($event)"></app-language-selector>

src/app/pages/translate/language-selectors/language-selectors.component.ts

@@ -17,16 +17,18 @@ import {takeUntil, tap} from 'rxjs/operators';
 })
 export class LanguageSelectorsComponent extends BaseComponent implements OnInit {
   spokenToSigned$: Observable<boolean>;
-  spokenLanguage$: Observable<boolean>;
-  detectedLanguage$: Observable<boolean>;
+  spokenLanguage$: Observable<string>;
+  signedLanguage$: Observable<string>;
+  detectedLanguage$: Observable<string>;
 
   @HostBinding('class.spoken-to-signed') spokenToSigned: boolean;
 
   constructor(private store: Store, public translation: TranslationService) {
     super();
     this.spokenToSigned$ = this.store.select<boolean>(state => state.translate.spokenToSigned);
-    this.spokenLanguage$ = this.store.select<boolean>(state => state.translate.spokenLanguage);
-    this.detectedLanguage$ = this.store.select<boolean>(state => state.translate.detectedLanguage);
+    this.spokenLanguage$ = this.store.select<string>(state => state.translate.spokenLanguage);
+    this.signedLanguage$ = this.store.select<string>(state => state.translate.signedLanguage);
+    this.detectedLanguage$ = this.store.select<string>(state => state.translate.detectedLanguage);
   }
 
   ngOnInit() {

src/app/pages/translate/pose-viewers/pose-viewer.component.ts

@@ -166,7 +166,6 @@ export abstract class BasePoseViewerComponent extends BaseComponent implements O
       tap(async () => {
         stream.getTracks().forEach(track => track.stop());
         const blob = new Blob(recordedChunks, {type: this.mediaRecorder.mimeType});
-        console.log('recordedChunks', recordedChunks);
         console.log('blob', blob.size, blob.type);
         // TODO: this does not work in iOS. The blob above is of size 0, and the video does not play.
         //       Should open an issue that ios mediarecorder dataavailable blob size is 0

src/app/pages/translate/spoken-to-signed/spoken-language-input/spoken-language-input.component.html

@@ -9,12 +9,19 @@
       enterkeyhint="enter"
       dir="auto"
       aria-labelledby="spoken-language-label"
-      aria-autocomplete="list"
-      autocapitalize="off"
-      autocomplete="off"
-      autocorrect="off"
       autofocus
       data-gramm_editor="false"></textarea>
+    <div id="suggestions" *transloco="let t; read: 'translate.suggestions'">
+      <ng-container *ngIf="detectedLanguage && detectedLanguage !== spokenLanguage">
+        <ion-icon name="sparkles" color="primary"></ion-icon>
+        {{ t('translate-from') }}:
+        <a (click)="setDetectedLanguage()">{{ 'languages.' + detectedLanguage | transloco }}</a>
+      </ng-container>
+      <ng-container *ngIf="false">
+        <ion-icon name="sparkles"></ion-icon>
+        {{ t('did-you-mean') }}: <a>todo add text suggestion</a>
+      </ng-container>
+    </div>
     <div class="actions-row">
       <app-speech-to-text [lang]="spokenLanguage" (changeText)="text.setValue($event)"></app-speech-to-text>
       <app-text-to-speech [lang]="spokenLanguage" [text]="text.value"></app-text-to-speech>
@@ -43,10 +50,6 @@
       enterkeyhint="enter"
       dir="auto"
       aria-labelledby="spoken-language-label"
-      aria-autocomplete="list"
-      autocapitalize="off"
-      autocomplete="off"
-      autocorrect="off"
       autofocus
       [attr.data-gramm_editor]="false"></ion-textarea>
 

src/app/pages/translate/spoken-to-signed/spoken-language-input/spoken-language-input.component.scss

@@ -27,6 +27,18 @@ textarea#desktop {
   height: calc(100% - 40px);
 }
 
+#suggestions {
+  position: absolute;
+  bottom: 60px;
+  white-space: break-spaces;
+  ion-icon {
+    padding: 0 4px;
+  }
+  a {
+    cursor: pointer;
+  }
+}
+
 #char-count {
   flex-grow: 1;
   align-self: center;

src/app/pages/translate/spoken-to-signed/spoken-language-input/spoken-language-input.component.ts

@@ -2,7 +2,7 @@ import {Component, Input, OnInit} from '@angular/core';
 import {FormControl} from '@angular/forms';
 import {debounce, distinctUntilChanged, skipWhile, takeUntil, tap} from 'rxjs/operators';
 import {interval, Observable} from 'rxjs';
-import {SetSpokenLanguageText} from '../../../../modules/translate/translate.actions';
+import {SetSpokenLanguage, SetSpokenLanguageText} from '../../../../modules/translate/translate.actions';
 import {Store} from '@ngxs/store';
 import {TranslateStateModel} from '../../../../modules/translate/translate.state';
 import {BaseComponent} from '../../../../components/base/base.component';
@@ -18,6 +18,7 @@ export class SpokenLanguageInputComponent extends BaseComponent implements OnIni
 
   text = new FormControl();
   maxTextLength = 500;
+  detectedLanguage: string;
   spokenLanguage: string;
 
   @Input() isMobile = false;
@@ -31,7 +32,10 @@ export class SpokenLanguageInputComponent extends BaseComponent implements OnIni
   ngOnInit() {
     this.translate$
       .pipe(
-        tap(({spokenLanguage, detectedLanguage}) => (this.spokenLanguage = spokenLanguage || detectedLanguage)),
+        tap(({spokenLanguage, detectedLanguage}) => {
+          this.detectedLanguage = detectedLanguage;
+          this.spokenLanguage = spokenLanguage ?? detectedLanguage;
+        }),
         takeUntil(this.ngUnsubscribe)
       )
       .subscribe();
@@ -55,4 +59,8 @@ export class SpokenLanguageInputComponent extends BaseComponent implements OnIni
       )
       .subscribe();
   }
+
+  setDetectedLanguage() {
+    this.store.dispatch(new SetSpokenLanguage(this.detectedLanguage));
+  }
 }