diff --git a/src/components/CallView/shared/LocalMediaControls.vue b/src/components/CallView/shared/LocalMediaControls.vue index 746be8f5f8c..31905c103db 100644 --- a/src/components/CallView/shared/LocalMediaControls.vue +++ b/src/components/CallView/shared/LocalMediaControls.vue @@ -160,7 +160,7 @@ {{ raiseHandButtonLabel }} { + this._loadPromiseResolve = resolve + this._loadPromiseReject = reject + }) this._loaded = false + this._loadFailed = false if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) { this._virtualImage = document.createElement('img') @@ -101,6 +106,11 @@ export default class JitsiStreamBackgroundEffect { break case 'loaded': this._loaded = true + this._loadPromiseResolve() + break + case 'loadFailed': + this._loadFailed = true + this._loadPromiseReject() break default: console.error('_startFx: Something went wrong.') @@ -108,6 +118,31 @@ export default class JitsiStreamBackgroundEffect { } } + /** + * Helper method to know when the model was loaded after creating the + * object. + * + * Note that it is not needed to call this method to actually load the + * effect; the load will automatically start as soon as the object is + * created, but it can be waited on this method to know once it has finished + * (or failed). + * + * @return {Promise} promise resolved or rejected once the load has finished + * or failed. + */ + async load() { + return this._loadPromise + } + + /** + * Returns whether loading the TFLite model failed or not. + * + * @return {boolean} true if loading failed, false otherwise + */ + didLoadFail() { + return this._loadFailed + } + /** * Represents the run post processing. * diff --git a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js index aa3963d8592..4135f30a246 100644 --- a/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js +++ b/src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js @@ -62,13 +62,24 @@ async function makeTFLite(isSimd) { await self.tflite._loadModel(self.model.byteLength) + // Even if the wrong tflite file is downloaded (for example, if an HTML + // error is downloaded instead of the file) loading the model will + // succeed. However, if the model does not have certain values it could + // be assumed that the model failed to load. + if (!self.tflite._getInputWidth() || !self.tflite._getInputHeight() + || !self.tflite._getOutputWidth() || !self.tflite._getOutputHeight()) { + throw new Error('Failed to load tflite model!') + } + self.compiled = true self.postMessage({ message: 'loaded' }) } catch (error) { console.error(error) - console.error('JitsiStreamBackgroundEffect.worker: tflite compilation failed.') + console.error('JitsiStreamBackgroundEffect.worker: tflite compilation failed. The web server may not be properly configured to send wasm and/or tflite files.') + + self.postMessage({ message: 'loadFailed' }) } } diff --git a/src/utils/media/pipeline/VirtualBackground.js b/src/utils/media/pipeline/VirtualBackground.js index 5b16dd09443..77e384b92ea 100644 --- a/src/utils/media/pipeline/VirtualBackground.js +++ b/src/utils/media/pipeline/VirtualBackground.js @@ -34,10 +34,16 @@ import JitsiStreamBackgroundEffect from '../effects/virtual-background/JitsiStre * * The virtual background node requires Web Assembly to be enabled in the * browser as well as support for canvas filters. Whether the virtual background - * is supported or not can be checked by calling - * "VirtualBackground.isSupported()". If a virtual background node is tried to - * be used when it is not supported its input will be just bypassed to its - * output. + * is available or not can be checked by calling + * "VirtualBackground.isSupported()". Besides that, it needs to + * download and compile a Tensor Flow Lite model; until the model has not + * finished loading or failed to load it is not possible to know for sure if + * virtual background is available, so if it is supported it is assumed to be + * available unless the model fails to load. In that case "loadFailed" is + * emitted and the virtual background is disabled. Whether the virtual + * background is available or not can be checked by calling "isAvailable()" on + * the object. If a virtual background node is tried to be used when it is not + * available its input will be just bypassed to its output. * * The virtual background is automatically stopped and started again when the * input track is disabled and enabled (which changes the output track). The @@ -140,6 +146,22 @@ export default class VirtualBackground extends TrackSinkSource { } this._jitsiStreamBackgroundEffect = new JitsiStreamBackgroundEffect(options) + this._jitsiStreamBackgroundEffect.load().catch(() => { + this._trigger('loadFailed') + + this.setEnabled(false) + }) + } + + isAvailable() { + if (!VirtualBackground.isSupported()) { + return false + } + + // If VirtualBackground is supported it is assumed to be available + // unless the load has failed (so it is seen as available even when + // still loading). + return !this._jitsiStreamBackgroundEffect.didLoadFail() } isEnabled() { @@ -147,6 +169,10 @@ export default class VirtualBackground extends TrackSinkSource { } setEnabled(enabled) { + if (!this.isAvailable()) { + enabled = false + } + if (this.enabled === enabled) { return } @@ -170,9 +196,9 @@ export default class VirtualBackground extends TrackSinkSource { } _handleInputTrack(trackId, newTrack, oldTrack) { - // If not supported or enabled the input track is just bypassed to the + // If not available or enabled the input track is just bypassed to the // output. - if (!VirtualBackground.isSupported() || !this._enabled) { + if (!this.isAvailable() || !this._enabled) { this._setOutputTrack('default', newTrack) return @@ -190,9 +216,9 @@ export default class VirtualBackground extends TrackSinkSource { } _handleInputTrackEnabled(trackId, enabled) { - // If not supported or enabled the input track is just bypassed to the + // If not available or enabled the input track is just bypassed to the // output. - if (!VirtualBackground.isSupported() || !this._enabled) { + if (!this.isAvailable() || !this._enabled) { this._setOutputTrackEnabled('default', enabled) return diff --git a/src/utils/webrtc/models/LocalMediaModel.js b/src/utils/webrtc/models/LocalMediaModel.js index 9e43c30ff6f..41dad137500 100644 --- a/src/utils/webrtc/models/LocalMediaModel.js +++ b/src/utils/webrtc/models/LocalMediaModel.js @@ -40,6 +40,7 @@ export default function LocalMediaModel() { volumeThreshold: -100, videoAvailable: false, videoEnabled: false, + virtualBackgroundAvailable: false, virtualBackgroundEnabled: false, localScreen: null, token: '', @@ -61,6 +62,7 @@ export default function LocalMediaModel() { this._handleStoppedSpeakingWhileMutedBound = this._handleStoppedSpeakingWhileMuted.bind(this) this._handleVideoOnBound = this._handleVideoOn.bind(this) this._handleVideoOffBound = this._handleVideoOff.bind(this) + this._handleVirtualBackgroundLoadFailedBound = this._handleVirtualBackgroundLoadFailed.bind(this) this._handleVirtualBackgroundOnBound = this._handleVirtualBackgroundOn.bind(this) this._handleVirtualBackgroundOffBound = this._handleVirtualBackgroundOff.bind(this) this._handleLocalScreenBound = this._handleLocalScreen.bind(this) @@ -101,6 +103,7 @@ LocalMediaModel.prototype = { this._webRtc.webrtc.off('stoppedSpeakingWhileMuted', this._handleStoppedSpeakingWhileMutedBound) this._webRtc.webrtc.off('videoOn', this._handleVideoOnBound) this._webRtc.webrtc.off('videoOff', this._handleVideoOffBound) + this._webRtc.webrtc.off('virtualBackgroundLoadFailed', this._handleVirtualBackgroundLoadFailedBound) this._webRtc.webrtc.off('virtualBackgroundOn', this._handleVirtualBackgroundOnBound) this._webRtc.webrtc.off('virtualBackgroundOff', this._handleVirtualBackgroundOffBound) this._webRtc.webrtc.off('localScreen', this._handleLocalScreenBound) @@ -118,6 +121,7 @@ LocalMediaModel.prototype = { this.set('volumeThreshold', -100) this.set('videoAvailable', false) this.set('videoEnabled', false) + this.set('virtualBackgroundAvailable', this._webRtc.webrtc.isVirtualBackgroundAvailable()) this.set('virtualBackgroundEnabled', this._webRtc.webrtc.isVirtualBackgroundEnabled()) this.set('localScreen', null) @@ -136,6 +140,7 @@ LocalMediaModel.prototype = { this._webRtc.webrtc.on('stoppedSpeakingWhileMuted', this._handleStoppedSpeakingWhileMutedBound) this._webRtc.webrtc.on('videoOn', this._handleVideoOnBound) this._webRtc.webrtc.on('videoOff', this._handleVideoOffBound) + this._webRtc.webrtc.on('virtualBackgroundLoadFailed', this._handleVirtualBackgroundLoadFailedBound) this._webRtc.webrtc.on('virtualBackgroundOn', this._handleVirtualBackgroundOnBound) this._webRtc.webrtc.on('virtualBackgroundOff', this._handleVirtualBackgroundOffBound) this._webRtc.webrtc.on('localScreen', this._handleLocalScreenBound) @@ -317,6 +322,10 @@ LocalMediaModel.prototype = { this.set('videoEnabled', false) }, + _handleVirtualBackgroundLoadFailed() { + this.set('virtualBackgroundAvailable', false) + }, + _handleVirtualBackgroundOn() { this.set('virtualBackgroundEnabled', true) }, diff --git a/src/utils/webrtc/simplewebrtc/localmedia.js b/src/utils/webrtc/simplewebrtc/localmedia.js index 5fbb66edfe1..adbd2b24285 100644 --- a/src/utils/webrtc/simplewebrtc/localmedia.js +++ b/src/utils/webrtc/simplewebrtc/localmedia.js @@ -53,6 +53,9 @@ function LocalMedia(opts) { this._videoTrackConstrainer = new TrackConstrainer() this._virtualBackground = new VirtualBackground() + this._virtualBackground.on('loadFailed', () => { + this.emit('virtualBackgroundLoadFailed') + }) this._speakingMonitor = new SpeakingMonitor() this._speakingMonitor.on('speaking', () => { @@ -363,6 +366,10 @@ LocalMedia.prototype.isVideoEnabled = function() { return enabled } +LocalMedia.prototype.isVirtualBackgroundAvailable = function() { + return this._virtualBackground.isAvailable() +} + LocalMedia.prototype.isVirtualBackgroundEnabled = function() { return this._virtualBackground.isEnabled() }