diff --git a/README.md b/README.md index 0642b7dcf8..4bf7ddf7b8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ * Download VSCode if you do not already have it installed. This is the preferred IDE for development with Pangea Chat. * Download flutter on your device using this guide: https://docs.flutter.dev/get-started/install -* Test to make sure that flutter is properly installed by running “flutter –version” +* Test to make sure that flutter is properly installed by running “flutter --version” * You may need to add flutter to your path manually. Instructions can be found here: https://docs.flutter.dev/get-started/install/macos/mobile-ios?tab=download#add-flutter-to-your-path * Ensure that Google Chrome is installed * Install the latest version of XCode @@ -25,9 +25,9 @@ * To run on Android: * Download Android File Transfer here: ​​https://www.android.com/filetransfer/ * To run the app from VSCode terminal: - * On web, run `flutter run -d chrome –hot` + * On web, run `flutter run -d chrome --hot` * Or as a web server (Usage with WSL or remote connect) `flutter run --release -d web-server -web-port=3000` - * On mobile device or simulator, run `flutter run –hot -d ` + * On mobile device or simulator, run `flutter run --hot -d ` # Special thanks diff --git a/lib/pages/chat/chat.dart b/lib/pages/chat/chat.dart index 98cdfa43b5..41a692d289 100644 --- a/lib/pages/chat/chat.dart +++ b/lib/pages/chat/chat.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:developer'; import 'dart:io'; +import 'package:fluffychat/pangea/common/constants/model_keys.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; @@ -938,6 +939,10 @@ class ChatController extends State 'duration': result.duration, 'waveform': result.waveform, }, + // #Pangea + //TODO: add results of transcription + // ModelKey.botTranscription: result.sttModel?.toJson(), + // Pangea# }, // #Pangea // ).catchError((e) { diff --git a/lib/pages/chat/recording_dialog.dart b/lib/pages/chat/recording_dialog.dart index 03fe569221..c9abfc9e36 100644 --- a/lib/pages/chat/recording_dialog.dart +++ b/lib/pages/chat/recording_dialog.dart @@ -5,14 +5,17 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter_gen/gen_l10n/l10n.dart'; +import 'package:flutter_sound/flutter_sound.dart'; import 'package:path/path.dart' as path_lib; import 'package:path_provider/path_provider.dart'; import 'package:record/record.dart'; import 'package:wakelock_plus/wakelock_plus.dart'; +import 'package:web_socket_channel/web_socket_channel.dart'; import 'package:fluffychat/config/app_config.dart'; import 'package:fluffychat/pangea/toolbar/utils/update_version_dialog.dart'; import 'package:fluffychat/utils/platform_infos.dart'; +import 'package:fluffychat/pangea/transcription/transcription_repo.dart'; import 'events/audio_player.dart'; class RecordingDialog extends StatefulWidget { @@ -33,12 +36,16 @@ class RecordingDialogState extends State { final _audioRecorder = AudioRecorder(); final List amplitudeTimeline = []; + FlutterSoundRecorder? _audioRecorderStream; + WebSocketChannel? realtimeTranscriptionChannel; + String? fileName; static const int bitRate = 64000; // #Pangea // static const int samplingRate = 44100; static const int samplingRate = 22050; + static const int samplingRateTranscription = 16000; // Pangea# Future startRecording() async { @@ -70,22 +77,33 @@ class RecordingDialogState extends State { await WakelockPlus.enable(); // #Pangea + const audioConfig = RecordConfig( + bitRate: bitRate, + sampleRate: samplingRate, + numChannels: 1, + autoGain: true, + echoCancel: true, + noiseSuppress: true, + encoder: codec, + ); + final streamBytes = StreamController(); + _audioRecorderStream = await FlutterSoundRecorder().openRecorder(); + final audioRecorderStartFuture = _audioRecorderStream!.startRecorder( + toStream: streamBytes.sink, + codec: Codec.pcm16, + numChannels: 1, + sampleRate: samplingRateTranscription, + bitRate: samplingRateTranscription, + ); final isNotError = await showUpdateVersionDialog( - future: () => - // Pangea# - - _audioRecorder.start( - const RecordConfig( - bitRate: bitRate, - sampleRate: samplingRate, - numChannels: 1, - autoGain: true, - echoCancel: true, - noiseSuppress: true, - encoder: codec, + // Pangea# + future: () => Future.wait([ + audioRecorderStartFuture, + _audioRecorder.start( + audioConfig, + path: path ?? '', ), - path: path ?? '', - ), + ]), // #Pangea context: context, ); @@ -107,6 +125,14 @@ class RecordingDialogState extends State { _duration += const Duration(milliseconds: 100); }); }); + + // init websocket with transcription API + realtimeTranscriptionChannel = + await TranscriptionRepo.connectTranscriptionChannel(); + realtimeTranscriptionChannel!.sink + .addStream(streamBytes.stream) + .then((_) {}) + .catchError((error) {}); } catch (_) { setState(() => error = true); rethrow; @@ -124,12 +150,20 @@ class RecordingDialogState extends State { WakelockPlus.disable(); _recorderSubscription?.cancel(); _audioRecorder.stop(); + if (_audioRecorderStream != null) { + _audioRecorderStream!.closeRecorder(); + _audioRecorderStream = null; + } super.dispose(); } void _stopAndSend() async { _recorderSubscription?.cancel(); final path = await _audioRecorder.stop(); + if (_audioRecorderStream != null) { + await _audioRecorderStream!.stopRecorder(); + } + await realtimeTranscriptionChannel!.sink.close(); if (path == null) throw ('Recording failed!'); const waveCount = AudioPlayerWidget.wavesCount; @@ -159,44 +193,63 @@ class RecordingDialogState extends State { '${_duration.inMinutes.toString().padLeft(2, '0')}:${(_duration.inSeconds % 60).toString().padLeft(2, '0')}'; final content = error ? Text(L10n.of(context).oopsSomethingWentWrong) - : Row( + : Column( + mainAxisSize: MainAxisSize.min, children: [ - Container( - width: 16, - height: 16, - decoration: BoxDecoration( - borderRadius: BorderRadius.circular(32), - color: Colors.red, - ), + Row( + children: [ + Container( + width: 16, + height: 16, + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(32), + color: Colors.red, + ), + ), + Expanded( + child: Row( + mainAxisSize: MainAxisSize.min, + mainAxisAlignment: MainAxisAlignment.end, + children: amplitudeTimeline.reversed + .take(26) + .toList() + .reversed + .map( + (amplitude) => Container( + margin: const EdgeInsets.only(left: 2), + width: 4, + decoration: BoxDecoration( + color: theme.colorScheme.primary, + borderRadius: BorderRadius.circular( + AppConfig.borderRadius), + ), + height: maxDecibalWidth * (amplitude / 100), + ), + ) + .toList(), + ), + ), + const SizedBox(width: 8), + SizedBox( + width: 48, + child: Text(time), + ), + ], ), - Expanded( - child: Row( - mainAxisSize: MainAxisSize.min, - mainAxisAlignment: MainAxisAlignment.end, - children: amplitudeTimeline.reversed - .take(26) - .toList() - .reversed - .map( - (amplitude) => Container( - margin: const EdgeInsets.only(left: 2), - width: 4, - decoration: BoxDecoration( - color: theme.colorScheme.primary, - borderRadius: - BorderRadius.circular(AppConfig.borderRadius), - ), - height: maxDecibalWidth * (amplitude / 100), + realtimeTranscriptionChannel != null + ? Row( + children: [ + StreamBuilder( + stream: realtimeTranscriptionChannel!.stream, + builder: (context, snapshot) { + return Text( + snapshot.hasData ? '${snapshot.data}' : '', + ); + }, ), - ) - .toList(), - ), - ), - const SizedBox(width: 8), - SizedBox( - width: 48, - child: Text(time), - ), + ], + ) + : const Row(), ], ); if (PlatformInfos.isCupertinoStyle) { diff --git a/lib/pangea/common/network/urls.dart b/lib/pangea/common/network/urls.dart index 6851369a9f..0f281acfad 100644 --- a/lib/pangea/common/network/urls.dart +++ b/lib/pangea/common/network/urls.dart @@ -75,6 +75,9 @@ class PApiUrls { static String morphFeaturesAndTags = "${PApiUrls.choreoEndpoint}/morphs"; + static String realtimeTranscriptionSession = + "${PApiUrls.choreoEndpoint}/realtime_transcription_session"; + ///-------------------------------- revenue cat -------------------------- static String rcAppsChoreo = "${PApiUrls.subscriptionEndpoint}/app_ids"; static String rcProductsChoreo = diff --git a/lib/pangea/transcription/transcription_models.dart b/lib/pangea/transcription/transcription_models.dart new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pangea/transcription/transcription_repo.dart b/lib/pangea/transcription/transcription_repo.dart new file mode 100644 index 0000000000..00b14f6ee1 --- /dev/null +++ b/lib/pangea/transcription/transcription_repo.dart @@ -0,0 +1,45 @@ +import 'dart:convert'; + +import 'package:fluffychat/pages/chat/recording_dialog.dart'; +import 'package:fluffychat/pangea/common/config/environment.dart'; +import 'package:fluffychat/pangea/common/network/requests.dart'; +import 'package:fluffychat/pangea/common/network/urls.dart'; +import 'package:fluffychat/widgets/matrix.dart'; +import 'package:http/http.dart'; +import 'package:web_socket_channel/io.dart'; + +class TranscriptionRepo { + static Future connectTranscriptionChannel() async { + final Requests reqToken = Requests( + choreoApiKey: Environment.choreoApiKey, + accessToken: MatrixState.pangeaController.userController.accessToken, + ); + + final Response resToken = await reqToken.get( + url: PApiUrls.realtimeTranscriptionSession, + ); + final json = jsonDecode(resToken.body); + final String key = json['key']; + + final uri = Uri( + scheme: 'wss', + host: 'api.deepgram.com', + path: 'v1/listen', + queryParameters: { + 'encoding': 'linear16', + 'sample_rate': '${RecordingDialogState.samplingRateTranscription}', + 'endpointing': 'false', + // TODO: accept language code as a parameter and set params based on it + // 'language': languageCode, + // 'model': languageCode == 'en' ? 'nova-3' : 'nova-2', + }, + ); + + final channel = IOWebSocketChannel.connect( + uri, + headers: {"Authorization": "Token $key"}, + ); + + return channel; + } +} diff --git a/pubspec.lock b/pubspec.lock index eff000cfe6..331efda57c 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -943,6 +943,30 @@ packages: url: "https://pub.dev" source: hosted version: "2.0.0" + flutter_sound: + dependency: "direct main" + description: + name: flutter_sound + sha256: ac0bee62b9255014aa1ce629dbbc8e8bab40337403ec405213c0d7848ff9ad49 + url: "https://pub.dev" + source: hosted + version: "9.25.1" + flutter_sound_platform_interface: + dependency: transitive + description: + name: flutter_sound_platform_interface + sha256: c57f996bd8049c71ac744b84439a951ea0bf495f6a229f3981dceb4dde0176a1 + url: "https://pub.dev" + source: hosted + version: "9.25.1" + flutter_sound_web: + dependency: transitive + description: + name: flutter_sound_web + sha256: "2fcedb2291a465e3bebc1f3dd680ffeb4efe2a3d5fedd06d2f75701ede5de722" + url: "https://pub.dev" + source: hosted + version: "9.25.1" flutter_svg: dependency: "direct main" description: @@ -1956,6 +1980,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.3.1" + recase: + dependency: transitive + description: + name: recase + sha256: e4eb4ec2dcdee52dcf99cb4ceabaffc631d7424ee55e56f280bc039737f89213 + url: "https://pub.dev" + source: hosted + version: "4.1.0" receive_sharing_intent: dependency: "direct main" description: @@ -2200,10 +2232,10 @@ packages: dependency: transitive description: name: shelf_web_socket - sha256: "9ca081be41c60190ebcb4766b2486a7d50261db7bd0f5d9615f2d653637a84c1" + sha256: cc36c297b52866d203dbf9332263c94becc2fe0ceaa9681d07b6ef9807023b67 url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "2.0.1" shimmer: dependency: "direct main" description: @@ -2729,14 +2761,22 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.0" - web_socket_channel: + web_socket: dependency: transitive + description: + name: web_socket + sha256: "3c12d96c0c9a4eec095246debcea7b86c0324f22df69893d538fcc6f1b8cce83" + url: "https://pub.dev" + source: hosted + version: "0.1.6" + web_socket_channel: + dependency: "direct main" description: name: web_socket_channel - sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b + sha256: "0b8e2457400d8a859b7b2030786835a28a8e80836ef64402abef392ff4f1d0e5" url: "https://pub.dev" source: hosted - version: "2.4.0" + version: "3.0.2" webdriver: dependency: transitive description: @@ -2818,5 +2858,5 @@ packages: source: hosted version: "3.1.2" sdks: - dart: ">=3.5.1 <4.0.0" + dart: ">=3.6.0 <4.0.0" flutter: ">=3.24.0" diff --git a/pubspec.yaml b/pubspec.yaml index 4e6b01c239..4814373be8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -133,6 +133,8 @@ dependencies: text_to_speech: ^0.2.3 flutter_tts: ^4.2.0 dart_levenshtein: ^1.0.1 + web_socket_channel: ^3.0.2 + flutter_sound: ^9.25.1 # Pangea# dev_dependencies: