Add RTSP streams support (#169)

* Start rtsp viewer

* Working rtsp stream viewer draft

* Remove debug change

* Improve quality and add flipped and rotated configs

* Update readme

* Small fixes

* Cleanup code

* Fix build

README.md

@@ -86,7 +86,27 @@ roslaunch web_video_server web_video_server.launch
 roslaunch rosbridge_server rosbridge_websocket.launch
 ```
 
-### Audio
+## System dependencies
+
+To view RTSP streams your system requires FFmpeg to be installed.
+
+### On Windows
+
+With Chocolatey:
+
+`choco install ffmpeg`
+
+### On Linux (Debian)
+
+`sudo apt install ffmpeg`
+
+### On Mac
+
+With homebrew:
+
+`brew install ffmpeg`
+
+## Audio
 
 Currently, the audio IO is handled by the [capra_audio](https://github.com/clubcapra/capra_audio_common) node. The UI will simply launch the node as a child process. This only works on linux. Windows support should work with wsl but is not currently implemented. For audio to work, you also need to make sure that the capra_audio launch files are sourced in the terminal you are using to launch the UI.
 

package.json

@@ -42,9 +42,13 @@
       "output": "dist"
     },
     "artifactName": "capra_web_ui_setup.${ext}",
-    "extends": "electron-snowpack/config/electron-builder.js"
+    "extends": "electron-snowpack/config/electron-builder.js",
+    "extraFiles": [
+      "script/**"
+    ]
   },
   "dependencies": {
+    "@cycjimmy/jsmpeg-player": "^6.0.5",
     "@reduxjs/toolkit": "^1.8.0",
     "@xstate/react": "^1.6.3",
     "chalk": "^5.0.1",
@@ -55,6 +59,7 @@
     "execa": "^6.1.0",
     "lodash": "^4.17.21",
     "nanoid": "^3.3.1",
+    "node-rtsp-stream": "^0.0.9",
     "polished": "^4.1.4",
     "qr-scanner": "^1.4.1",
     "react": "^17.0.2",

script/rtspServer.js

@@ -0,0 +1,25 @@
+const Stream = require('node-rtsp-stream');
+
+// Get node args
+const args = process.argv.slice(2);
+
+// Check if required arguments are present
+if (args.length < 2) {
+  const filename = __filename.split('/').pop();
+  // eslint-disable-next-line no-console
+  console.error(
+    `Missing Arguments! Usage: node ${filename} <rtsp-url> <websocket-port>`
+  );
+  process.exit(1);
+}
+
+new Stream({
+  name: 'RTSP-Stream',
+  streamUrl: args[0],
+  wsPort: parseInt(args[1]),
+  ffmpegOptions: {
+    // options ffmpeg flags
+    '-r': 30,
+    '-q': 0, // quality video in scale [0, 32]
+  },
+});

snowpack.config.js

@@ -12,4 +12,4 @@ module.exports = {
   packageOptions: {
     knownEntrypoints: ['date-fns/fp/format', 'react-is'],
   },
-}
+};

src/main/index.ts

@@ -1,3 +1,4 @@
+import '@/main/rtspServer';
 import '@/main/audio';
 import { log } from '@/main/logger';
 import { APP_INFO_QUERY, APP_INFO_TYPE, AUDIO_STOP } from '@/main/preload';

src/main/preload.ts

@@ -18,6 +18,9 @@ export type AUDIO_MSG_TYPE = {
   stdout: string;
 };
 
+export const RTSP_START = 'rtsp_start';
+export const RTSP_STOP = 'rtsp_stop';
+
 if (contextBridge) {
   // This is done this way because otherwise tests would try to import preload and it wouldn't work
   // It's possible the issue is mostly in the import order.

src/main/preload/api.ts

@@ -7,6 +7,8 @@ import {
   AUDIO_STOP,
   LOG_MSG,
   LOG_MSG_TYPE,
+  RTSP_START,
+  RTSP_STOP,
 } from '@/main/preload';
 import { ipcRenderer } from 'electron';
 
@@ -29,4 +31,8 @@ export const preload = {
       ipcRenderer.on(AUDIO_MSG, (_event, args) => cb(args as AUDIO_MSG_TYPE));
     },
   },
+  rtsp: {
+    start: (url: string) => ipcRenderer.invoke(RTSP_START, url),
+    stop: (port: number) => ipcRenderer.send(RTSP_STOP, port),
+  },
 };

src/main/rtspServer.ts

@@ -0,0 +1,45 @@
+import { log } from '@/main/logger';
+import { app, ipcMain } from 'electron';
+import { ExecaChildProcess, execa } from 'execa';
+import path from 'path';
+import { RTSP_START, RTSP_STOP } from './preload';
+import process from 'process';
+
+interface RtspProcess {
+  process: ExecaChildProcess;
+  wsPort: number;
+}
+
+const rtspServers: Map<number, RtspProcess> = new Map();
+
+// Stack array of ports from (9000 to 9060) to use for rtsp servers
+const ports = Array.from({ length: 61 }, (_, i) => i + 9000);
+
+ipcMain.handle(RTSP_START, (_, url: string) => {
+  const nextPort = ports.shift() ?? 9000;
+  const rtspProcess = execa('node', [
+    !app.isPackaged
+      ? './script/rtspServer.js'
+      : path.resolve(`${process.resourcesPath}/../script/rtspServer.js`),
+    url,
+    nextPort.toString(),
+  ]);
+  rtspServers.set(nextPort, {
+    process: rtspProcess,
+    wsPort: nextPort,
+  });
+
+  return nextPort;
+});
+
+ipcMain.on(RTSP_STOP, (_, port: number) => {
+  const rtspProcess = rtspServers.get(port);
+
+  if (!rtspProcess) {
+    return;
+  }
+  log.info('stopping rtsp process');
+  ports.push(rtspProcess.wsPort);
+  rtspProcess.process.kill();
+  rtspServers.delete(port);
+});

src/renderer/components/Feed/Feeds/CameraFeed.tsx

@@ -9,6 +9,7 @@ import * as React from 'react';
 import { FC, useEffect, useRef, useState } from 'react';
 import { log } from '@/renderer/logger';
 import { QRFeed } from './QRFeed/QRFeed';
+import { RTSPFeed } from './RTSPFeed';
 
 interface Props {
   feed: ICameraFeed;
@@ -145,6 +146,14 @@ const View: FC<Props> = ({ feed }) => {
           />
         </QRFeed>
       );
+    case CameraType.RTSP:
+      return (
+        <RTSPFeed
+          url={feed.camera.topic}
+          flipped={feed.camera.flipped}
+          rotated={feed.camera.rotated}
+        />
+      );
     default:
       return <TextFeed text="stream type not supported" />;
   }

src/renderer/components/Feed/Feeds/RTSPFeed.tsx

@@ -0,0 +1,71 @@
+import React, { useEffect, useRef } from 'react';
+import JSMpeg from '@cycjimmy/jsmpeg-player';
+import { log } from '@/renderer/logger';
+
+interface Props {
+  url: string;
+  flipped: boolean;
+  rotated: boolean;
+}
+
+export const RTSPFeed = ({ url, flipped, rotated }: Props) => {
+  const videoRef = useRef<HTMLDivElement>(null);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+
+  useEffect(() => {
+    const videoWrapper = videoRef.current;
+    const canvas = canvasRef.current;
+    let videoElement: JSMpeg.VideoElement | null = null;
+    let port: number | null = null;
+    const startServer = async () => {
+      port = (await window.preloadApi.rtsp.start(url)) as number;
+
+      if (videoWrapper && canvas) {
+        log.info(`RTSP server started on port ${port}`);
+        videoElement = new JSMpeg.VideoElement(
+          videoWrapper,
+          `ws://localhost:${port}`,
+          {
+            canvas,
+            autoplay: true,
+            audio: false,
+          },
+          { videoBufferSize: 2048 * 2048 }
+        );
+      }
+    };
+
+    startServer().catch((e) => log.error(e));
+
+    return () => {
+      if (videoElement && port) {
+        log.info(`Stopping RTSP server for ${url}`);
+        window.preloadApi.rtsp.stop(port);
+        videoElement.destroy();
+      }
+    };
+  }, [url]);
+
+  return (
+    <div
+      ref={videoRef}
+      style={{
+        height: '100%',
+        objectFit: 'contain',
+        overflow: 'hidden',
+        transform: `${flipped ? 'scaleX(-1)' : ''} ${
+          rotated ? 'rotate(180deg)' : ''
+        }`,
+      }}
+    >
+      <canvas
+        ref={canvasRef}
+        style={{
+          height: '100%',
+          objectFit: 'contain',
+          overflow: 'hidden',
+        }}
+      />
+    </div>
+  );
+};

src/renderer/store/modules/feed.ts

@@ -31,6 +31,7 @@ export enum CameraType {
   VP8 = 'vp8',
   WEBCAM = 'webcam',
   QR_CODE = 'qr_code',
+  RTSP = 'rtsp',
 }
 
 export type FeedType =

src/renderer/types/jsmpeg.d.ts

@@ -0,0 +1,19 @@
+declare module '@cycjimmy/jsmpeg-player' {
+  class VideoElement {
+    constructor(
+      videoWrapper: HTMLDivElement,
+      url: string,
+      options: {
+        canvas?: HTMLCanvasElement;
+        autoplay?: boolean;
+        audio?: boolean;
+        autoSetWrapperSize?: boolean;
+      },
+      overlayOptions?: {
+        videoBufferSize?: number;
+      }
+    );
+    paused: boolean;
+    destroy(): void;
+  }
+}