[stable23] Fix empty video when blur background model could not be loaded

src/components/CallView/shared/LocalMediaControls.vue

@@ -160,7 +160,7 @@
 					{{ raiseHandButtonLabel }}
 				</ActionButton>
 				<ActionButton
-					v-if="isVirtualBackgroundSupported"
+					v-if="isVirtualBackgroundAvailable"
 					:close-after-click="true"
 					@click="toggleVirtualBackground">
 					<BlurOff
@@ -253,7 +253,6 @@ import NetworkStrength2Alert from 'vue-material-design-icons/NetworkStrength2Ale
 import { Actions, ActionSeparator, ActionButton } from '@nextcloud/vue'
 import { callAnalyzer } from '../../../utils/webrtc/index'
 import { CONNECTION_QUALITY } from '../../../utils/webrtc/analyzers/PeerConnectionAnalyzer'
-import VirtualBackground from '../../../utils/media/pipeline/VirtualBackground'
 import isInCall from '../../../mixins/isInCall'
 
 export default {
@@ -330,8 +329,8 @@ export default {
 			return t('spreed', 'Lower hand (R)')
 		},
 
-		isVirtualBackgroundSupported() {
-			return VirtualBackground.isSupported()
+		isVirtualBackgroundAvailable() {
+			return this.model.attributes.virtualBackgroundAvailable
 		},
 
 		isVirtualBackgroundEnabled() {
@@ -566,7 +565,7 @@ export default {
 				return null
 			}
 
-			const virtualBackgroundEnabled = this.isVirtualBackgroundSupported && this.model.attributes.virtualBackgroundEnabled
+			const virtualBackgroundEnabled = this.isVirtualBackgroundAvailable && this.model.attributes.virtualBackgroundEnabled
 
 			const tooltip = {}
 			if (!this.model.attributes.audioEnabled && this.model.attributes.videoEnabled && virtualBackgroundEnabled && this.model.attributes.localScreen) {

src/components/DeviceChecker/DeviceChecker.vue

@@ -189,7 +189,6 @@ import { subscribe, unsubscribe } from '@nextcloud/event-bus'
 import CheckboxRadioSwitch from '@nextcloud/vue/dist/Components/CheckboxRadioSwitch'
 import BrowserStorage from '../../services/BrowserStorage'
 import VolumeIndicator from '../VolumeIndicator/VolumeIndicator.vue'
-import VirtualBackground from '../../utils/media/pipeline/VirtualBackground'
 
 export default {
 	name: 'DeviceChecker',
@@ -256,7 +255,7 @@ export default {
 		},
 
 		blurPreviewAvailable() {
-			return VirtualBackground.isSupported()
+			return this.virtualBackground.isAvailable()
 		},
 
 		audioButtonTooltip() {

src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.js

@@ -45,7 +45,12 @@ export default class JitsiStreamBackgroundEffect {
 	constructor(options) {
 		const isSimd = options.simd
 		this._options = options
+		this._loadPromise = new Promise((resolve, reject) => {
+			this._loadPromiseResolve = resolve
+			this._loadPromiseReject = reject
+		})
 		this._loaded = false
+		this._loadFailed = false
 
 		if (this._options.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
 			this._virtualImage = document.createElement('img')
@@ -101,13 +106,43 @@ export default class JitsiStreamBackgroundEffect {
 			break
 		case 'loaded':
 			this._loaded = true
+			this._loadPromiseResolve()
+			break
+		case 'loadFailed':
+			this._loadFailed = true
+			this._loadPromiseReject()
 			break
 		default:
 			console.error('_startFx: Something went wrong.')
 			break
 		}
 	}
 
+	/**
+	 * Helper method to know when the model was loaded after creating the
+	 * object.
+	 *
+	 * Note that it is not needed to call this method to actually load the
+	 * effect; the load will automatically start as soon as the object is
+	 * created, but it can be waited on this method to know once it has finished
+	 * (or failed).
+	 *
+	 * @return {Promise} promise resolved or rejected once the load has finished
+	 *         or failed.
+	 */
+	async load() {
+		return this._loadPromise
+	}
+
+	/**
+	 * Returns whether loading the TFLite model failed or not.
+	 *
+	 * @return {boolean} true if loading failed, false otherwise
+	 */
+	didLoadFail() {
+		return this._loadFailed
+	}
+
 	/**
 	 * Represents the run post processing.
 	 *

src/utils/media/effects/virtual-background/JitsiStreamBackgroundEffect.worker.js

@@ -62,13 +62,24 @@ async function makeTFLite(isSimd) {
 
 		await self.tflite._loadModel(self.model.byteLength)
 
+		// Even if the wrong tflite file is downloaded (for example, if an HTML
+		// error is downloaded instead of the file) loading the model will
+		// succeed. However, if the model does not have certain values it could
+		// be assumed that the model failed to load.
+		if (!self.tflite._getInputWidth() || !self.tflite._getInputHeight()
+			|| !self.tflite._getOutputWidth() || !self.tflite._getOutputHeight()) {
+			throw new Error('Failed to load tflite model!')
+		}
+
 		self.compiled = true
 
 		self.postMessage({ message: 'loaded' })
 
 	} catch (error) {
 		console.error(error)
-		console.error('JitsiStreamBackgroundEffect.worker: tflite compilation failed.')
+		console.error('JitsiStreamBackgroundEffect.worker: tflite compilation failed. The web server may not be properly configured to send wasm and/or tflite files.')
+
+		self.postMessage({ message: 'loadFailed' })
 	}
 }
 

src/utils/media/pipeline/VirtualBackground.js

@@ -34,10 +34,16 @@ import JitsiStreamBackgroundEffect from '../effects/virtual-background/JitsiStre
  *
  * The virtual background node requires Web Assembly to be enabled in the
  * browser as well as support for canvas filters. Whether the virtual background
- * is supported or not can be checked by calling
- * "VirtualBackground.isSupported()". If a virtual background node is tried to
- * be used when it is not supported its input will be just bypassed to its
- * output.
+ * is available or not can be checked by calling
+ * "VirtualBackground.isSupported()". Besides that, it needs to
+ * download and compile a Tensor Flow Lite model; until the model has not
+ * finished loading or failed to load it is not possible to know for sure if
+ * virtual background is available, so if it is supported it is assumed to be
+ * available unless the model fails to load. In that case "loadFailed" is
+ * emitted and the virtual background is disabled. Whether the virtual
+ * background is available or not can be checked by calling "isAvailable()" on
+ * the object. If a virtual background node is tried to be used when it is not
+ * available its input will be just bypassed to its output.
  *
  * The virtual background is automatically stopped and started again when the
  * input track is disabled and enabled (which changes the output track). The
@@ -140,13 +146,33 @@ export default class VirtualBackground extends TrackSinkSource {
 		}
 
 		this._jitsiStreamBackgroundEffect = new JitsiStreamBackgroundEffect(options)
+		this._jitsiStreamBackgroundEffect.load().catch(() => {
+			this._trigger('loadFailed')
+
+			this.setEnabled(false)
+		})
+	}
+
+	isAvailable() {
+		if (!VirtualBackground.isSupported()) {
+			return false
+		}
+
+		// If VirtualBackground is supported it is assumed to be available
+		// unless the load has failed (so it is seen as available even when
+		// still loading).
+		return !this._jitsiStreamBackgroundEffect.didLoadFail()
 	}
 
 	isEnabled() {
 		return this._enabled
 	}
 
 	setEnabled(enabled) {
+		if (!this.isAvailable()) {
+			enabled = false
+		}
+
 		if (this.enabled === enabled) {
 			return
 		}
@@ -170,9 +196,9 @@ export default class VirtualBackground extends TrackSinkSource {
 	}
 
 	_handleInputTrack(trackId, newTrack, oldTrack) {
-		// If not supported or enabled the input track is just bypassed to the
+		// If not available or enabled the input track is just bypassed to the
 		// output.
-		if (!VirtualBackground.isSupported() || !this._enabled) {
+		if (!this.isAvailable() || !this._enabled) {
 			this._setOutputTrack('default', newTrack)
 
 			return
@@ -190,9 +216,9 @@ export default class VirtualBackground extends TrackSinkSource {
 	}
 
 	_handleInputTrackEnabled(trackId, enabled) {
-		// If not supported or enabled the input track is just bypassed to the
+		// If not available or enabled the input track is just bypassed to the
 		// output.
-		if (!VirtualBackground.isSupported() || !this._enabled) {
+		if (!this.isAvailable() || !this._enabled) {
 			this._setOutputTrackEnabled('default', enabled)
 
 			return

src/utils/webrtc/models/LocalMediaModel.js

@@ -40,6 +40,7 @@ export default function LocalMediaModel() {
 		volumeThreshold: -100,
 		videoAvailable: false,
 		videoEnabled: false,
+		virtualBackgroundAvailable: false,
 		virtualBackgroundEnabled: false,
 		localScreen: null,
 		token: '',
@@ -61,6 +62,7 @@ export default function LocalMediaModel() {
 	this._handleStoppedSpeakingWhileMutedBound = this._handleStoppedSpeakingWhileMuted.bind(this)
 	this._handleVideoOnBound = this._handleVideoOn.bind(this)
 	this._handleVideoOffBound = this._handleVideoOff.bind(this)
+	this._handleVirtualBackgroundLoadFailedBound = this._handleVirtualBackgroundLoadFailed.bind(this)
 	this._handleVirtualBackgroundOnBound = this._handleVirtualBackgroundOn.bind(this)
 	this._handleVirtualBackgroundOffBound = this._handleVirtualBackgroundOff.bind(this)
 	this._handleLocalScreenBound = this._handleLocalScreen.bind(this)
@@ -101,6 +103,7 @@ LocalMediaModel.prototype = {
 			this._webRtc.webrtc.off('stoppedSpeakingWhileMuted', this._handleStoppedSpeakingWhileMutedBound)
 			this._webRtc.webrtc.off('videoOn', this._handleVideoOnBound)
 			this._webRtc.webrtc.off('videoOff', this._handleVideoOffBound)
+			this._webRtc.webrtc.off('virtualBackgroundLoadFailed', this._handleVirtualBackgroundLoadFailedBound)
 			this._webRtc.webrtc.off('virtualBackgroundOn', this._handleVirtualBackgroundOnBound)
 			this._webRtc.webrtc.off('virtualBackgroundOff', this._handleVirtualBackgroundOffBound)
 			this._webRtc.webrtc.off('localScreen', this._handleLocalScreenBound)
@@ -118,6 +121,7 @@ LocalMediaModel.prototype = {
 		this.set('volumeThreshold', -100)
 		this.set('videoAvailable', false)
 		this.set('videoEnabled', false)
+		this.set('virtualBackgroundAvailable', this._webRtc.webrtc.isVirtualBackgroundAvailable())
 		this.set('virtualBackgroundEnabled', this._webRtc.webrtc.isVirtualBackgroundEnabled())
 		this.set('localScreen', null)
 
@@ -136,6 +140,7 @@ LocalMediaModel.prototype = {
 		this._webRtc.webrtc.on('stoppedSpeakingWhileMuted', this._handleStoppedSpeakingWhileMutedBound)
 		this._webRtc.webrtc.on('videoOn', this._handleVideoOnBound)
 		this._webRtc.webrtc.on('videoOff', this._handleVideoOffBound)
+		this._webRtc.webrtc.on('virtualBackgroundLoadFailed', this._handleVirtualBackgroundLoadFailedBound)
 		this._webRtc.webrtc.on('virtualBackgroundOn', this._handleVirtualBackgroundOnBound)
 		this._webRtc.webrtc.on('virtualBackgroundOff', this._handleVirtualBackgroundOffBound)
 		this._webRtc.webrtc.on('localScreen', this._handleLocalScreenBound)
@@ -317,6 +322,10 @@ LocalMediaModel.prototype = {
 		this.set('videoEnabled', false)
 	},
 
+	_handleVirtualBackgroundLoadFailed() {
+		this.set('virtualBackgroundAvailable', false)
+	},
+
 	_handleVirtualBackgroundOn() {
 		this.set('virtualBackgroundEnabled', true)
 	},

src/utils/webrtc/simplewebrtc/localmedia.js

@@ -53,6 +53,9 @@ function LocalMedia(opts) {
 	this._videoTrackConstrainer = new TrackConstrainer()
 
 	this._virtualBackground = new VirtualBackground()
+	this._virtualBackground.on('loadFailed', () => {
+		this.emit('virtualBackgroundLoadFailed')
+	})
 
 	this._speakingMonitor = new SpeakingMonitor()
 	this._speakingMonitor.on('speaking', () => {
@@ -363,6 +366,10 @@ LocalMedia.prototype.isVideoEnabled = function() {
 	return enabled
 }
 
+LocalMedia.prototype.isVirtualBackgroundAvailable = function() {
+	return this._virtualBackground.isAvailable()
+}
+
 LocalMedia.prototype.isVirtualBackgroundEnabled = function() {
 	return this._virtualBackground.isEnabled()
 }