Feat/trame viewport

packages/core/package.json

@@ -34,7 +34,9 @@
     "comlink": "^4.4.1",
     "detect-gpu": "^5.0.22",
     "gl-matrix": "^3.4.3",
-    "lodash.clonedeep": "4.5.0"
+    "lodash.clonedeep": "4.5.0",
+    "wslink": "^2.1.1",
+    "html2canvas": "1.4.1"
   },
   "contributors": [
     {

packages/core/src/RenderingEngine/RenderingEngine.ts

@@ -25,6 +25,7 @@ import type {
 } from '../types/IViewport';
 import { OrientationAxis } from '../enums';
 import VolumeViewport3D from './VolumeViewport3D';
+import TrameViewport from './trameViewport';
 
 type ViewportDisplayCoords = {
   sxStartDisplayCoords: number;
@@ -789,6 +790,8 @@ class RenderingEngine implements IRenderingEngine {
       viewport = new VolumeViewport(viewportInput);
     } else if (type === ViewportType.VOLUME_3D) {
       viewport = new VolumeViewport3D(viewportInput);
+    } else if (type === ViewportType.TRAME) {
+      viewport = new TrameViewport(viewportInput);
     } else {
       throw new Error(`Viewport Type ${type} is not supported`);
     }

packages/core/src/RenderingEngine/helpers/viewportTypeToViewportClass.ts

@@ -5,6 +5,7 @@ import ViewportType from '../../enums/ViewportType';
 import VolumeViewport3D from '../VolumeViewport3D';
 import VideoViewport from '../VideoViewport';
 import WSIViewport from '../WSIViewport';
+import TrameViewport from '../trameViewport';
 
 const viewportTypeToViewportClass = {
   [ViewportType.ORTHOGRAPHIC]: VolumeViewport,
@@ -13,6 +14,7 @@ const viewportTypeToViewportClass = {
   [ViewportType.VOLUME_3D]: VolumeViewport3D,
   [ViewportType.VIDEO]: VideoViewport,
   [ViewportType.WholeSlide]: WSIViewport,
+  [ViewportType.TRAME]: TrameViewport,
 };
 
 export default viewportTypeToViewportClass;

packages/core/src/RenderingEngine/trameViewport.ts

@@ -0,0 +1,161 @@
+import Viewport from './Viewport';
+import { ViewportInput } from '../types/IViewport';
+import wslink from './wslink';
+import vtkRemoteView from '@kitware/vtk.js/Rendering/Misc/RemoteView';
+
+import type { Point3, Point2 } from '../types';
+
+interface RemoteConnection {
+  connection;
+  viewId;
+  viewStream;
+  view;
+  session;
+}
+
+export default class TrameViewport extends Viewport {
+  remoteConnection: RemoteConnection;
+
+  ////////////////////////////////////////////////////////////////////////////////
+  private getWorldToCanvasRatio() {
+    return 1.0;
+  }
+
+  private getCanvasToWorldRatio() {
+    return 1.0;
+  }
+
+  /**
+   * Converts a VideoViewport canvas coordinate to a video coordinate.
+   *
+   * @param canvasPos - to convert to world
+   * @returns World position
+   */
+  public canvasToWorld = (
+    canvasPos: Point2,
+    destPos: Point3 = [0, 0, 0]
+  ): Point3 => {
+    const pan: Point2 = [0, 0]; // In world coordinates
+    const worldToCanvasRatio: number = this.getWorldToCanvasRatio();
+
+    const panOffsetCanvas: Point2 = [
+      pan[0] * worldToCanvasRatio,
+      pan[1] * worldToCanvasRatio,
+    ];
+
+    const subCanvasPos: Point2 = [
+      canvasPos[0] - panOffsetCanvas[0],
+      canvasPos[1] - panOffsetCanvas[1],
+    ];
+
+    // Replace the x,y values only in place in the world position
+    // as the z is unchanging for video display
+    destPos.splice(
+      0,
+      2,
+      subCanvasPos[0] / worldToCanvasRatio,
+      subCanvasPos[1] / worldToCanvasRatio
+    );
+    return destPos;
+  };
+
+  /**
+   * Converts `[x, y, 0]` world video coordinate to canvas CSS coordinates.
+   *
+   * @param  worldPos - world coord to convert to canvas
+   * @returns Canvas position
+   */
+  public worldToCanvas = (worldPos: Point3): Point2 => {
+    const pan: Point2 = [0, 0];
+    const worldToCanvasRatio: number = this.getWorldToCanvasRatio();
+
+    const canvasPos: Point2 = [
+      (worldPos[0] + pan[0]) * worldToCanvasRatio,
+      (worldPos[1] + pan[1]) * worldToCanvasRatio,
+    ];
+
+    return canvasPos;
+  };
+
+  public getFrameOfReferenceUID = (): string => {
+    return 'REMOTE';
+  };
+
+  ////////////////////////////////////////////////////////////////////////////////
+
+  protected configureElement() {
+    this.element.style.position = 'relative';
+    this.element.style.overflow = 'hidden';
+  }
+
+  protected configureConnection() {
+    this.remoteConnection = {
+      connection: wslink.createClient(),
+      viewId: '',
+      viewStream: '',
+      view: '',
+      session: '',
+    };
+    this.remoteConnection.connection.onConnectionError((httpReq) => {
+      const message =
+        (httpReq && httpReq.response && httpReq.response.error) ||
+        `Connection error`;
+      console.error(message);
+      console.log(httpReq);
+    });
+
+    // Close
+    this.remoteConnection.connection.onConnectionClose((httpReq) => {
+      const message =
+        (httpReq && httpReq.response && httpReq.response.error) ||
+        `Connection close`;
+      console.error(message);
+      console.log(httpReq);
+    });
+  }
+
+  constructor(props: ViewportInput) {
+    super(props);
+    this.configureConnection();
+  }
+
+  public connect = (config, isTrameConnection = false): void => {
+    this.remoteConnection.connection
+      .connect(config)
+      .then(async (validClient) => {
+        let remoteObject = undefined;
+        if (isTrameConnection) {
+          remoteObject = await this.remoteConnection.connection
+            .getRemote()
+            .Trame.getState();
+        }
+        this.remoteConnection.viewId = remoteObject?.state?.viewId || '1';
+        this.remoteConnection.viewStream = this.remoteConnection.connection
+          .getImageStream()
+          .createViewStream(this.remoteConnection.viewId);
+
+        this.remoteConnection.view = vtkRemoteView.newInstance({
+          rpcWheelEvent: 'viewport.mouse.zoom.wheel',
+          viewStream: this.remoteConnection.viewStream,
+        });
+        const session = validClient.getConnection().getSession();
+        this.remoteConnection.view.setSession(session);
+        this.remoteConnection.view.setContainer(this.element.children[0]);
+        // make the canvas from remoteView be the first one canvas
+        this.element.children[0].insertBefore(
+          this.element.children[0].children[2],
+          this.element.children[0].children[0]
+        );
+        this.remoteConnection.view.setInteractiveRatio(0.7); // the scaled image compared to the clients view resolution
+        this.remoteConnection.view.setInteractiveQuality(50); // jpeg quality
+
+        this.element.addEventListener(
+          'resize',
+          this.remoteConnection.view.resize
+        );
+      })
+      .catch((error) => {
+        console.error(error);
+      });
+  };
+}

packages/core/src/RenderingEngine/wslink/index.js

@@ -0,0 +1,88 @@
+import vtkURLExtract from '@kitware/vtk.js/Common/Core/URLExtract';
+import SmartConnect from 'wslink/src/SmartConnect';
+import vtkWSLinkClient from '@kitware/vtk.js/IO/Core/WSLinkClient';
+
+import protocols from './protocols';
+
+vtkWSLinkClient.setSmartConnectClass(SmartConnect);
+
+const WS_PROTOCOL = {
+  'http:': 'ws:',
+  'https:': 'wss:',
+  'ws:': 'ws:',
+  'wss:': 'wss:',
+};
+
+const NOT_BUSY_LIST = [
+  // *
+  'unsubscribe',
+  // SyncView
+  'subscribeToViewChange',
+  // Trame
+  'subscribeToStateUpdate',
+  'subscribeToActions',
+  'subscribeToViewChange',
+];
+
+function configDecorator(config) {
+  const outputConfig = { ...config };
+
+  // Process sessionURL
+  if (outputConfig.sessionURL) {
+    let sessionURL = outputConfig.sessionURL.toLowerCase();
+    const httpURL = window.location;
+
+    // handle protocol mapping http(s) => ws(s)
+    if (sessionURL.includes('use_')) {
+      const wsURL = new URL(sessionURL);
+      wsURL.protocol = WS_PROTOCOL[httpURL.protocol];
+      sessionURL = wsURL.toString();
+    }
+
+    // handle variable replacement
+    const useMapping = {
+      use_hostname: httpURL.hostname,
+      use_host: httpURL.host,
+    };
+    // eslint-disable-next-line no-restricted-syntax
+    for (const [key, value] of Object.entries(useMapping)) {
+      sessionURL = sessionURL.replaceAll(key, value);
+    }
+
+    // update config
+    outputConfig.sessionURL = sessionURL;
+  }
+
+  // Extract app-name from html
+  outputConfig.application =
+    document.querySelector('html').dataset.appName || outputConfig.application;
+
+  const sessionManagerURL =
+    document.querySelector('html').dataset.sessionManagerUrl ||
+    outputConfig.sessionManagerURL;
+  if (sessionManagerURL) {
+    outputConfig.sessionManagerURL = sessionManagerURL;
+  }
+
+  // Process arguments from URL
+  if (outputConfig.useUrl) {
+    return {
+      ...outputConfig,
+      ...vtkURLExtract.extractURLParameters(),
+    };
+  }
+  return outputConfig;
+}
+
+function createClient() {
+  return vtkWSLinkClient.newInstance({
+    protocols,
+    configDecorator,
+    notBusyList: NOT_BUSY_LIST,
+  });
+}
+
+export default {
+  configDecorator,
+  createClient,
+};

packages/core/src/RenderingEngine/wslink/protocols/Trame.js

@@ -0,0 +1,26 @@
+export default (session) => ({
+  lifeCycleUpdate(phaseName) {
+    return session.call('trame.lifecycle.update', [phaseName]);
+  },
+  sendError(message) {
+    return session.call('trame.error.client', [message]);
+  },
+  getState() {
+    return session.call('trame.state.get', []);
+  },
+  trigger(name, args = [], kwargs = {}) {
+    return session.call('trame.trigger', [name, args, kwargs]);
+  },
+  updateState(changes) {
+    return session.call('trame.state.update', [changes]);
+  },
+  subscribeToStateUpdate(callback) {
+    return session.subscribe('trame.state.topic', callback);
+  },
+  subscribeToActions(callback) {
+    return session.subscribe('trame.actions.topic', callback);
+  },
+  unsubscribe(subscription) {
+    return session.unsubscribe(subscription);
+  },
+});

packages/core/src/RenderingEngine/wslink/protocols/index.js

@@ -0,0 +1,5 @@
+import Trame from './Trame';
+
+export default {
+  Trame,
+};

packages/core/src/enums/ViewportType.ts

@@ -17,6 +17,7 @@ enum ViewportType {
   PERSPECTIVE = 'perspective',
   VOLUME_3D = 'volume3d',
   VIDEO = 'video',
+  TRAME = 'trame',
   /**
    * Whole slide imaging viewport
    */

packages/core/src/index.ts

@@ -1,6 +1,7 @@
 import * as Enums from './enums';
 import * as CONSTANTS from './constants';
 import { Events } from './enums';
+import wslinkConnection from './RenderingEngine/wslink';
 //
 import {
   createVolumeActor,
@@ -156,4 +157,5 @@ export {
   // Geometry Loader
   geometryLoader,
   ProgressiveRetrieveImages,
+  wslinkConnection,
 };