diff --git a/src/__mocks__/attachmediastream.js b/src/__mocks__/attachmediastream.js new file mode 100644 index 00000000000..74178ed2718 --- /dev/null +++ b/src/__mocks__/attachmediastream.js @@ -0,0 +1,17 @@ +/** + * Basic "attachmediastream" implementation without using "webrtc-adapter", as + * "browserDetails" is null in unit tests. + * + * @param {MediaStream} stream the stream to attach + * @param {HTMLElement} element the element to attach the stream to + * @param {object} options ignored + */ +export default function(stream, element, options) { + if (!element) { + element = document.createElement(options.audio ? 'audio' : 'video') + } + + element.srcObject = stream + + return element +} diff --git a/src/components/CallView/CallView.vue b/src/components/CallView/CallView.vue index 98b87f16355..b34a177076f 100644 --- a/src/components/CallView/CallView.vue +++ b/src/components/CallView/CallView.vue @@ -141,6 +141,7 @@ import { loadState } from '@nextcloud/initial-state' import Grid from './Grid/Grid' import { SIMULCAST } from '../../constants' import { localMediaModel, localCallParticipantModel, callParticipantCollection } from '../../utils/webrtc/index' +import RemoteVideoBlocker from '../../utils/webrtc/RemoteVideoBlocker' import { fetchPeers } from '../../services/callsService' import { showMessage } from '@nextcloud/dialogs' import EmptyCallView from './shared/EmptyCallView' @@ -212,7 +213,7 @@ export default { callParticipantModelsWithVideo() { return this.callParticipantModels.filter(callParticipantModel => { return callParticipantModel.attributes.videoAvailable - && this.sharedDatas[callParticipantModel.attributes.peerId].videoEnabled + && this.sharedDatas[callParticipantModel.attributes.peerId].remoteVideoBlocker.isVideoEnabled() && (typeof callParticipantModel.attributes.stream === 'object') }) }, @@ -404,7 +405,6 @@ export default { callParticipantCollection.on('remove', this._lowerHandWhenParticipantLeaves) - subscribe('talk:video:toggled', this.handleToggleVideo) subscribe('switch-screen-to-id', this._switchScreenToId) }, beforeDestroy() { @@ -412,7 +412,6 @@ export default { callParticipantCollection.off('remove', this._lowerHandWhenParticipantLeaves) - unsubscribe('talk:video:toggled', this.handleToggleVideo) unsubscribe('switch-screen-to-id', this._switchScreenToId) }, methods: { @@ -454,7 +453,7 @@ export default { addedModels.forEach(addedModel => { const sharedData = { promoted: false, - videoEnabled: true, + remoteVideoBlocker: new RemoteVideoBlocker(addedModel), screenVisible: false, } @@ -646,11 +645,6 @@ export default { } }, 1500), - // Toggles videos on and off - handleToggleVideo({ peerId, value }) { - this.sharedDatas[peerId].videoEnabled = value - }, - adjustSimulcastQuality() { this.callParticipantModels.forEach(callParticipantModel => { this.adjustSimulcastQualityForParticipant(callParticipantModel) diff --git a/src/components/CallView/Grid/Grid.vue b/src/components/CallView/Grid/Grid.vue index c7fe91e7370..63496c80833 100644 --- a/src/components/CallView/Grid/Grid.vue +++ b/src/components/CallView/Grid/Grid.vue @@ -584,7 +584,11 @@ export default { placeholderSharedData() { return { - videoEnabled: true, + videoEnabled: { + isVideoEnabled() { + return true + }, + }, screenVisible: false, } }, diff --git a/src/components/CallView/shared/Video.spec.js b/src/components/CallView/shared/Video.spec.js new file mode 100644 index 00000000000..224a9652116 --- /dev/null +++ b/src/components/CallView/shared/Video.spec.js @@ -0,0 +1,1147 @@ +/** + * + * @copyright Copyright (c) 2022, Daniel Calviño Sánchez (danxuliu@gmail.com) + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +import Vuex from 'vuex' +import { createLocalVue, shallowMount } from '@vue/test-utils' +import { cloneDeep } from 'lodash' +import storeConfig from '../../../store/storeConfig' + +import EmitterMixin from '../../../utils/EmitterMixin' +import CallParticipantModel from '../../../utils/webrtc/models/CallParticipantModel' + +import Video from './Video' + +describe('Video.vue', () => { + let localVue + let store + let testStoreConfig + + let callParticipantModel + + function PeerMock() { + this._superEmitterMixin() + + this.id = 'theId' + this.nick = 'The nick' + this.pc = { + connectionState: 'new', + iceConnectionState: 'new', + signalingState: 'stable', + } + } + PeerMock.prototype._setIceConnectionState = function(iceConnectionState) { + this.pc.iceConnectionState = iceConnectionState + this._trigger('extendedIceConnectionStateChange', [iceConnectionState]) + } + PeerMock.prototype._setSignalingState = function(signalingState) { + this.pc.signalingState = signalingState + this._trigger('signalingStateChange', [signalingState]) + } + EmitterMixin.apply(PeerMock.prototype) + // Override _trigger from EmitterMixin, as it includes "this" as the first + // argument. + PeerMock.prototype._trigger = function(event, args) { + let handlers = this._handlers[event] + if (!handlers) { + return + } + + if (!args) { + args = [] + } + + handlers = handlers.slice(0) + for (let i = 0; i < handlers.length; i++) { + const handler = handlers[i] + handler.apply(handler, args) + } + } + + beforeEach(() => { + localVue = createLocalVue() + localVue.use(Vuex) + + testStoreConfig = cloneDeep(storeConfig) + store = new Vuex.Store(testStoreConfig) + + const webRtcMock = { + on: jest.fn(), + off: jest.fn(), + } + callParticipantModel = new CallParticipantModel({ + peerId: 'theId', + webRtc: webRtcMock, + }) + }) + + describe('connection state feedback', () => { + const connectionMessage = { + NOT_ESTABLISHED: 'Connection could not be established. Trying again …', + NOT_ESTABLISHED_NOT_RETRYING: 'Connection could not be established …', + LOST: 'Connection lost. Trying to reconnect …', + LOST_NOT_RETRYING: 'Connection was lost and could not be re-established …', + PROBLEMS: 'Connection problems …', + NONE: null, + } + + let wrapper + + // "setupWrapper()" needs to be called right before checking the wrapper + // to ensure that the component state is updated. If the wrapper is + // created at the beginning of each test "await Vue.nextTick()" would + // need to be called instead (and for that the tests would need to be + // async). + function setupWrapper() { + wrapper = shallowMount(Video, { + localVue, + store, + propsData: { + model: callParticipantModel, + token: 'theToken', + sharedData: { + remoteVideoBlocker: { + increaseVisibleCounter: jest.fn(), + }, + promoted: false, + }, + }, + }) + } + + function assertConnectionMessageLabel(expectedText) { + const connectionMessageLabel = wrapper.find('.connection-message') + if (expectedText) { + expect(connectionMessageLabel.exists()).toBe(true) + expect(connectionMessageLabel.text()).toBe(expectedText) + } else { + expect(connectionMessageLabel.exists()).toBe(false) + } + } + + function assertLoadingIconIsShown(expected) { + const loadingIcon = wrapper.find('.icon-loading') + expect(loadingIcon.exists()).toBe(expected) + } + + function assertNotConnected(expected) { + const notConnected = wrapper.find('.not-connected') + expect(notConnected.exists()).toBe(expected) + } + + test('participant just created', () => { + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('no peer', () => { + callParticipantModel.setPeer(null) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + describe('original connection', () => { + let peerMock + + beforeEach(() => { + peerMock = new PeerMock() + callParticipantModel.setPeer(peerMock) + }) + + test('peer just set', () => { + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('sending offer', () => { + peerMock._setSignalingState('have-local-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('receiving offer', () => { + peerMock._setSignalingState('have-remote-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('negotiation finished', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('establishing connection', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('connection established', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('connection established (completed)', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('completed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('disconnected', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected without ever connecting', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + peerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long without ever connecting', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + peerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without ever connecting', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + }) + + describe('renegotiation', () => { + let peerMock + + beforeEach(() => { + peerMock = new PeerMock() + callParticipantModel.setPeer(peerMock) + }) + + test('started after connection established', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setSignalingState('have-remote-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('started after connection established and then finished', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('started before disconnected', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setSignalingState('have-remote-offer') + peerMock._setIceConnectionState('disconnected') + + setupWrapper() + + // FIXME The message should be "PROBLEMS" rather than "LOST", as + // the negotiation is not caused by the disconnection itself. + // However it does not seem to be an easy way to do it right + // now. + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('started before disconnected and then finished', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setSignalingState('have-remote-offer') + peerMock._setIceConnectionState('disconnected') + peerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('started after disconnected', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + peerMock._setSignalingState('have-remote-offer') + + setupWrapper() + + // FIXME The message should be "PROBLEMS" rather than "LOST", as + // the negotiation is not caused by the disconnection itself. + // However it does not seem to be an easy way to do it right + // now. + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('started after disconnected and then finished', () => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + }) + + describe('reconnection after no original connection', () => { + let newPeerMock + + beforeEach(() => { + callParticipantModel.setPeer(null) + + newPeerMock = new PeerMock() + callParticipantModel.setPeer(newPeerMock) + }) + + test('peer just set', () => { + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('sending offer', () => { + newPeerMock._setSignalingState('have-local-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('receiving offer', () => { + newPeerMock._setSignalingState('have-remote-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('negotiation finished', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('establishing connection', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('connection established', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('connection established (completed)', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('completed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('disconnected', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected without ever connecting', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + newPeerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long without ever connecting', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + newPeerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + newPeerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without ever connecting', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + }) + + describe('reconnection', () => { + let peerMock + let newPeerMock + + beforeEach(() => { + peerMock = new PeerMock() + callParticipantModel.setPeer(peerMock) + + newPeerMock = new PeerMock() + }) + + describe('without having been connected', () => { + beforeEach(() => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('failed') + + callParticipantModel.setPeer(newPeerMock) + }) + + test('peer just set', () => { + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('sending offer', () => { + newPeerMock._setSignalingState('have-local-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('receiving offer', () => { + newPeerMock._setSignalingState('have-remote-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('negotiation finished', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('establishing connection', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('connection established', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('disconnected', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected without ever connecting', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the + // connection has been disconnected for a few seconds + newPeerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long without ever connecting', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the + // connection has been disconnected for a few seconds + newPeerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + newPeerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without ever connecting', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without ever connecting and not retrying', () => { + newPeerMock._setSignalingState('have-remote-offer') + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('failed') + // Custom event emitted when there is no HPB and the + // connection has failed several times in a row + newPeerMock._trigger('extendedIceConnectionStateChange', ['failed-no-restart']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NOT_ESTABLISHED_NOT_RETRYING) + assertLoadingIconIsShown(false) + assertNotConnected(true) + }) + }) + + describe('after having been connected', () => { + beforeEach(() => { + peerMock._setSignalingState('have-remote-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + }) + + describe('without HPB', () => { + test('ICE restarted after disconnected long (no HPB)', () => { + peerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + peerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + peerMock._setSignalingState('have-local-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('ICE restarted after failed (no HPB)', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('negotiation finished', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('establishing connection', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('connection established', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('connection established (completed)', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('completed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('disconnected', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected without connecting the second time', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + peerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected long without connecting the second time', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('disconnected') + // Custom event emitted when there is no HPB and the connection + // has been disconnected for a few seconds + peerMock._trigger('extendedIceConnectionStateChange', ['disconnected-long']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('connected') + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without connecting the second time', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without connecting the second time and not retrying', () => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + peerMock._setSignalingState('have-local-offer') + peerMock._setSignalingState('stable') + peerMock._setIceConnectionState('checking') + peerMock._setIceConnectionState('failed') + // Custom event emitted when there is no HPB and the + // connection has failed several times in a row + peerMock._trigger('extendedIceConnectionStateChange', ['failed-no-restart']) + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST_NOT_RETRYING) + assertLoadingIconIsShown(false) + assertNotConnected(true) + }) + }) + + describe('with HPB', () => { + beforeEach(() => { + peerMock._setIceConnectionState('disconnected') + peerMock._setIceConnectionState('failed') + + callParticipantModel.setPeer(newPeerMock) + + newPeerMock._setSignalingState('have-remote-offer') + }) + + test('requested renegotiation after connection failed', () => { + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('negotiation finished', () => { + newPeerMock._setSignalingState('stable') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('establishing connection', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('connection established', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('connection established (completed)', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('completed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.NONE) + assertLoadingIconIsShown(false) + assertNotConnected(false) + }) + + test('disconnected', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.PROBLEMS) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('disconnected without connecting the second time', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('disconnected') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('connected') + newPeerMock._setIceConnectionState('disconnected') + newPeerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + + test('failed without connecting the second time', () => { + newPeerMock._setSignalingState('stable') + newPeerMock._setIceConnectionState('checking') + newPeerMock._setIceConnectionState('failed') + + setupWrapper() + + assertConnectionMessageLabel(connectionMessage.LOST) + assertLoadingIconIsShown(true) + assertNotConnected(true) + }) + }) + }) + }) + }) +}) diff --git a/src/components/CallView/shared/Video.vue b/src/components/CallView/shared/Video.vue index 617417a01ce..e1d792bece3 100644 --- a/src/components/CallView/shared/Video.vue +++ b/src/components/CallView/shared/Video.vue @@ -180,12 +180,12 @@ export default { return this.model.attributes.connectedAtLeastOnce }, - isNotConnected() { - return this.model.attributes.connectionState !== ConnectionState.CONNECTED && this.model.attributes.connectionState !== ConnectionState.COMPLETED + isConnected() { + return this.model.attributes.connectionState === ConnectionState.CONNECTED || this.model.attributes.connectionState === ConnectionState.COMPLETED }, isLoading() { - return this.isNotConnected && this.model.attributes.connectionState !== ConnectionState.FAILED_NO_RESTART + return !this.isConnected && this.model.attributes.connectionState !== ConnectionState.FAILED_NO_RESTART }, isDisconnected() { @@ -204,11 +204,15 @@ export default { * received yet). Similarly both "negotiating" and "connecting" need to * be checked, as the negotiation will start before the connection * attempt is started. + * + * If the negotiation is done while there is still a connection it is + * not regarded as reconnecting, as in that case it is a renegotiation + * to update the current connection. */ isReconnecting() { return this.model.attributes.connectionState === ConnectionState.FAILED || (!this.model.attributes.initialConnection - && (this.model.attributes.negotiating || this.model.attributes.connecting)) + && ((this.model.attributes.negotiating && !this.isConnected) || this.model.attributes.connecting)) }, isNoLongerTryingToReconnect() { @@ -242,7 +246,7 @@ export default { containerClass() { return { 'videoContainer-dummy': this.placeholderForPromoted, - 'not-connected': !this.placeholderForPromoted && this.isNotConnected, + 'not-connected': !this.placeholderForPromoted && !this.isConnected, speaking: !this.placeholderForPromoted && this.model.attributes.speaking, promoted: !this.placeholderForPromoted && this.sharedData.promoted && !this.isGrid, 'video-container-grid': this.isGrid, @@ -371,7 +375,7 @@ export default { }, hasVideo() { - return this.model.attributes.videoAvailable && this.sharedData.videoEnabled && (typeof this.model.attributes.stream === 'object') + return !this.model.attributes.videoBlocked && this.model.attributes.videoAvailable && this.sharedData.remoteVideoBlocker.isVideoEnabled() && (typeof this.model.attributes.stream === 'object') }, hasSelectedVideo() { @@ -469,10 +473,16 @@ export default { }, mounted() { + this.sharedData.remoteVideoBlocker.increaseVisibleCounter() + // Set initial state this._setStream(this.model.attributes.stream) }, + destroyed() { + this.sharedData.remoteVideoBlocker.decreaseVisibleCounter() + }, + methods: { _setStream(stream) { diff --git a/src/components/CallView/shared/VideoBottomBar.vue b/src/components/CallView/shared/VideoBottomBar.vue index ca4b23477e5..74baadec259 100644 --- a/src/components/CallView/shared/VideoBottomBar.vue +++ b/src/components/CallView/shared/VideoBottomBar.vue @@ -200,11 +200,11 @@ export default { }, showVideoButton() { - return this.sharedData.videoEnabled + return this.sharedData.remoteVideoBlocker.isVideoEnabled() }, videoButtonTooltip() { - if (this.sharedData.videoEnabled) { + if (this.sharedData.remoteVideoBlocker.isVideoEnabled()) { return t('spreed', 'Disable video') } @@ -220,7 +220,7 @@ export default { }, showNameIndicator() { - return !this.model.attributes.videoAvailable || !this.sharedData.videoEnabled || this.showVideoOverlay || this.isSelected || this.isPromoted || this.isSpeaking + return !this.model.attributes.videoAvailable || !this.sharedData.remoteVideoBlocker.isVideoEnabled() || this.showVideoOverlay || this.isSelected || this.isPromoted || this.isSpeaking }, boldenNameIndicator() { @@ -255,10 +255,7 @@ export default { }, toggleVideo() { - emit('talk:video:toggled', { - peerId: this.model.attributes.peerId, - value: !this.sharedData.videoEnabled, - }) + this.sharedData.remoteVideoBlocker.setVideoEnabled(!this.sharedData.remoteVideoBlocker.isVideoEnabled()) }, switchToScreen() { diff --git a/src/utils/signaling.js b/src/utils/signaling.js index 6b6fbab3223..c0d1fbb52d4 100644 --- a/src/utils/signaling.js +++ b/src/utils/signaling.js @@ -1292,7 +1292,7 @@ Signaling.Standalone.prototype.processRoomParticipantsEvent = function(data) { } } -Signaling.Standalone.prototype.requestOffer = function(sessionid, roomType) { +Signaling.Standalone.prototype.requestOffer = function(sessionid, roomType, sid = undefined) { if (!this.hasFeature('mcu')) { console.warn("Can't request an offer without a MCU.") return @@ -1302,7 +1302,7 @@ Signaling.Standalone.prototype.requestOffer = function(sessionid, roomType) { // Got a user object. sessionid = sessionid.sessionId || sessionid.sessionid } - console.debug('Request offer from', sessionid) + console.debug('Request offer from', sessionid, sid) this.doSend({ type: 'message', message: { @@ -1313,6 +1313,7 @@ Signaling.Standalone.prototype.requestOffer = function(sessionid, roomType) { data: { type: 'requestoffer', roomType, + sid, }, }, }) diff --git a/src/utils/webrtc/RemoteVideoBlocker.js b/src/utils/webrtc/RemoteVideoBlocker.js new file mode 100644 index 00000000000..2a8a65cbd6a --- /dev/null +++ b/src/utils/webrtc/RemoteVideoBlocker.js @@ -0,0 +1,131 @@ +/** + * + * @copyright Copyright (c) 2022, Daniel Calviño Sánchez (danxuliu@gmail.com) + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +/** + * Helper to block the remote video when not needed. + * + * A remote video is not needed if the local user explicitly disabled it + * (independently of whether the remote user (and thus owner of the remote + * video) has it enabled or not) or if it is not visible. + * + * The remote video is not immediately hidden when no longer visible; a few + * seconds are waited to avoid blocking and unblocking on layout changes. + * + * "increaseVisibleCounter()" can be called several times by the same view, but + * "decreaseVisibleCounter()" must have been called a corresponding number of + * times once the view is destroyed. + * + * A single RemoteVideoBlocker is assumed to be associated with its + * CallParticipantModel, and it is also assumed to be the only element blocking + * and unblocking the video. Otherwise the result is undefined. + * + * Note that the RemoteVideoBlocker can be used on participants that do not have + * a video at all (for example, because they do not have a camera or they do not + * have video permissions). In that case the CallParticipantModel will block the + * video if needed if it becomes available. + * + * @param {object} callParticipantModel the model to block/unblock the video on. + */ +export default function RemoteVideoBlocker(callParticipantModel) { + this._model = callParticipantModel + + // Keep track of the blocked state here, as the Peer object may not block + // the video if some features are missing, and even if the video is blocked + // the attribute will not be updated right away but once the renegotiation + // is done. + this._blocked = false + + this._enabled = true + this._visibleCounter = 1 + + this._blockVideoTimeout = null + + // Block by default if not shown after creation. + this.decreaseVisibleCounter() +} + +RemoteVideoBlocker.prototype = { + + isVideoEnabled() { + return this._enabled + }, + + setVideoEnabled(enabled) { + this._enabled = enabled + + const hadBlockVideoTimeout = this._blockVideoTimeout + + clearTimeout(this._blockVideoTimeout) + this._blockVideoTimeout = null + + if (!this._visibleCounter && !hadBlockVideoTimeout) { + return + } + + this._setVideoBlocked(!enabled) + }, + + increaseVisibleCounter() { + this._visibleCounter++ + + clearTimeout(this._blockVideoTimeout) + this._blockVideoTimeout = null + + if (!this._enabled) { + return + } + + this._setVideoBlocked(false) + }, + + decreaseVisibleCounter() { + if (this._visibleCounter <= 0) { + console.error('Visible counter decreased when not visible') + + return + } + + this._visibleCounter-- + + if (this._visibleCounter > 0 || !this._enabled) { + return + } + + clearTimeout(this._blockVideoTimeout) + + this._blockVideoTimeout = setTimeout(() => { + this._setVideoBlocked(true) + + this._blockVideoTimeout = null + }, 5000) + }, + + _setVideoBlocked(blocked) { + if (this._blocked === blocked) { + return + } + + this._blocked = blocked + + this._model.setVideoBlocked(blocked) + }, + +} diff --git a/src/utils/webrtc/RemoteVideoBlocker.spec.js b/src/utils/webrtc/RemoteVideoBlocker.spec.js new file mode 100644 index 00000000000..700b2918b5c --- /dev/null +++ b/src/utils/webrtc/RemoteVideoBlocker.spec.js @@ -0,0 +1,336 @@ +/** + * + * @copyright Copyright (c) 2022, Daniel Calviño Sánchez (danxuliu@gmail.com) + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +import RemoteVideoBlocker from './RemoteVideoBlocker' + +describe('RemoteVideoBlocker', () => { + let callParticipantModel + let remoteVideoBlocker + + beforeEach(() => { + jest.useFakeTimers() + + callParticipantModel = { + setVideoBlocked: jest.fn() + } + + remoteVideoBlocker = new RemoteVideoBlocker(callParticipantModel) + }) + + test('blocks the video by default if not shown in some seconds', () => { + jest.advanceTimersByTime(4000) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(0) + + jest.advanceTimersByTime(1000) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + describe('set video enabled', () => { + test('immediately blocks the video', () => { + remoteVideoBlocker.increaseVisibleCounter() + + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(false) + }) + test('immediately unblocks the video', () => { + remoteVideoBlocker.increaseVisibleCounter() + + remoteVideoBlocker.setVideoEnabled(false) + remoteVideoBlocker.setVideoEnabled(true) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(2) + expect(callParticipantModel.setVideoBlocked).toHaveBeenNthCalledWith(2, false) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + }) + + describe('set video visible', () => { + test('does nothing if shown', () => { + remoteVideoBlocker.increaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(0) + expect(remoteVideoBlocker._visibleCounter).toBe(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('does nothing if hidden without showing first', () => { + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(remoteVideoBlocker._visibleCounter).toBe(0) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('blocks the video after some seconds when hidden', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.advanceTimersByTime(4000) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(0) + + jest.advanceTimersByTime(1000) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + expect(remoteVideoBlocker._visibleCounter).toBe(0) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('does nothing if shown again before blocking', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.advanceTimersByTime(4000) + + remoteVideoBlocker.increaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(0) + expect(remoteVideoBlocker._visibleCounter).toBe(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('immediately unblocks the video after showing', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + remoteVideoBlocker.increaseVisibleCounter() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(2) + expect(callParticipantModel.setVideoBlocked).toHaveBeenNthCalledWith(2, false) + expect(remoteVideoBlocker._visibleCounter).toBe(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('does nothing if not fully hidden', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(0) + expect(remoteVideoBlocker._visibleCounter).toBe(2) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + }) + + describe('set video enabled and visible', () => { + test('immediately blocks the video if disabled when visible', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(false) + }) + + test('immediately blocks the video if disabled before blocking after hidden', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.advanceTimersByTime(4000) + + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(false) + }) + + test('blocks the video after some seconds if hidden when enabled', () => { + remoteVideoBlocker.setVideoEnabled(true) + + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.advanceTimersByTime(4000) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(0) + + jest.advanceTimersByTime(1000) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('does nothing if disabled when hidden', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(false) + }) + + test('does nothing if enabled when hidden', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.setVideoEnabled(false) + remoteVideoBlocker.setVideoEnabled(true) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('does nothing if hidden when disabled', () => { + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(false) + }) + + test('does nothing if shown when disabled', () => { + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(false) + }) + + test('immediately unblocks the video if enabled after showing', () => { + remoteVideoBlocker.setVideoEnabled(false) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.increaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + remoteVideoBlocker.setVideoEnabled(true) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(2) + expect(callParticipantModel.setVideoBlocked).toHaveBeenNthCalledWith(2, false) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + + test('immediately unblocks the video if shown after enabled', () => { + remoteVideoBlocker.increaseVisibleCounter() + remoteVideoBlocker.decreaseVisibleCounter() + + jest.runAllTimers() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledWith(true) + + remoteVideoBlocker.setVideoEnabled(false) + remoteVideoBlocker.setVideoEnabled(true) + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(1) + + remoteVideoBlocker.increaseVisibleCounter() + + expect(callParticipantModel.setVideoBlocked).toHaveBeenCalledTimes(2) + expect(callParticipantModel.setVideoBlocked).toHaveBeenNthCalledWith(2, false) + + expect(remoteVideoBlocker.isVideoEnabled()).toBe(true) + }) + }) +}) diff --git a/src/utils/webrtc/models/CallParticipantModel.js b/src/utils/webrtc/models/CallParticipantModel.js index 35406745476..6362e14e320 100644 --- a/src/utils/webrtc/models/CallParticipantModel.js +++ b/src/utils/webrtc/models/CallParticipantModel.js @@ -65,6 +65,9 @@ export default function CallParticipantModel(options) { audioElement: null, audioAvailable: undefined, speaking: undefined, + // "videoBlocked" is "true" only if the video is blocked and it would + // have been available in the remote peer if not blocked. + videoBlocked: undefined, videoAvailable: undefined, screen: null, // The audio element is part of the model to ensure that it can be @@ -89,6 +92,7 @@ export default function CallParticipantModel(options) { this._handleSignalingStateChangeBound = this._handleSignalingStateChange.bind(this) this._handleChannelMessageBound = this._handleChannelMessage.bind(this) this._handleRaisedHandBound = this._handleRaisedHand.bind(this) + this._handleRemoteVideoBlockedBound = this._handleRemoteVideoBlocked.bind(this) this._webRtc.on('peerStreamAdded', this._handlePeerStreamAddedBound) this._webRtc.on('peerStreamRemoved', this._handlePeerStreamRemovedBound) @@ -105,6 +109,7 @@ CallParticipantModel.prototype = { if (this.get('peer')) { this.get('peer').off('extendedIceConnectionStateChange', this._handleExtendedIceConnectionStateChangeBound) this.get('peer').off('signalingStateChange', this._handleSignalingStateChangeBound) + this.get('peer').off('remoteVideoBlocked', this._handleRemoteVideoBlockedBound) } this._webRtc.off('peerStreamAdded', this._handlePeerStreamAddedBound) @@ -249,6 +254,7 @@ CallParticipantModel.prototype = { if (this.get('peer')) { this.get('peer').off('extendedIceConnectionStateChange', this._handleExtendedIceConnectionStateChangeBound) this.get('peer').off('signalingStateChange', this._handleSignalingStateChangeBound) + this.get('peer').off('remoteVideoBlocked', this._handleRemoteVideoBlockedBound) } this.set('peer', peer) @@ -261,6 +267,7 @@ CallParticipantModel.prototype = { this.set('audioAvailable', false) this.set('speaking', false) this.set('videoAvailable', false) + this.set('videoBlocked', false) return } @@ -275,9 +282,19 @@ CallParticipantModel.prototype = { } this._handleSignalingStateChange(this.get('peer').pc.signalingState) this._handlePeerStreamAdded(this.get('peer')) + this._handleRemoteVideoBlocked(undefined) this.get('peer').on('extendedIceConnectionStateChange', this._handleExtendedIceConnectionStateChangeBound) this.get('peer').on('signalingStateChange', this._handleSignalingStateChangeBound) + this.get('peer').on('remoteVideoBlocked', this._handleRemoteVideoBlockedBound) + + // Set expected state in Peer object. + if (this._simulcastVideoQuality !== undefined) { + this.setSimulcastVideoQuality(this._simulcastVideoQuality) + } + if (this._videoBlocked !== undefined) { + this.setVideoBlocked(this._videoBlocked) + } }, _handleExtendedIceConnectionStateChange(extendedIceConnectionState) { @@ -363,6 +380,11 @@ CallParticipantModel.prototype = { // Reset state that depends on the screen Peer object. this._handlePeerStreamAdded(this.get('screenPeer')) + + // Set expected state in screen Peer object. + if (this._simulcastScreenQuality !== undefined) { + this.setSimulcastScreenQuality(this._simulcastScreenQuality) + } }, setUserId(userId) { @@ -373,7 +395,25 @@ CallParticipantModel.prototype = { this.set('nextcloudSessionId', nextcloudSessionId) }, + setVideoBlocked(videoBlocked) { + // Store value to be able to apply it again if a new Peer object is set. + this._videoBlocked = videoBlocked + + if (!this.get('peer')) { + return + } + + this.get('peer').setRemoteVideoBlocked(videoBlocked) + }, + + _handleRemoteVideoBlocked(remoteVideoBlocked) { + this.set('videoBlocked', remoteVideoBlocked) + }, + setSimulcastVideoQuality(simulcastVideoQuality) { + // Store value to be able to apply it again if a new Peer object is set. + this._simulcastVideoQuality = simulcastVideoQuality + if (!this.get('peer') || !this.get('peer').enableSimulcast) { return } @@ -383,6 +423,10 @@ CallParticipantModel.prototype = { }, setSimulcastScreenQuality(simulcastScreenQuality) { + // Store value to be able to apply it again if a new screen Peer object + // is set. + this._simulcastScreenQuality = simulcastScreenQuality + if (!this.get('screenPeer') || !this.get('screenPeer').enableSimulcast) { return } diff --git a/src/utils/webrtc/simplewebrtc/peer.js b/src/utils/webrtc/simplewebrtc/peer.js index 6d64b20a490..6b0a987a8f4 100644 --- a/src/utils/webrtc/simplewebrtc/peer.js +++ b/src/utils/webrtc/simplewebrtc/peer.js @@ -31,6 +31,7 @@ function Peer(options) { this.oneway = options.oneway || false this.sharemyscreen = options.sharemyscreen || false this.stream = options.stream + this.receiverOnly = options.receiverOnly this.sendVideoIfAvailable = options.sendVideoIfAvailable === undefined ? true : options.sendVideoIfAvailable this.enableDataChannels = options.enableDataChannels === undefined ? this.parent.config.enableDataChannels : options.enableDataChannels this.enableSimulcast = options.enableSimulcast === undefined ? this.parent.config.enableSimulcast : options.enableSimulcast @@ -418,12 +419,49 @@ Peer.prototype.offer = function(options) { Peer.prototype.handleOffer = function(offer) { this.pc.setRemoteDescription(offer).then(function() { + this._blockRemoteVideoIfNeeded() + this.answer() }.bind(this)).catch(function(error) { console.warn('setRemoteDescription for offer failed: ', error) }) } +/** + * Blocks remote video based on "_remoteVideoShouldBeBlocked". + * + * 'remoteVideoBlocked' is emitted if the blocked state changes. + * + * Currently remote video can be blocked only when the HPB is used, so this + * method should be called immediately before creating the answer (the answer + * must be created in the same "tick" that this method is called). + * + * Note that if the transceiver direction changes after creating the answer but + * before setting it as the local description the "negotiationneeded" event will + * be automatically emitted again. + */ +Peer.prototype._blockRemoteVideoIfNeeded = function() { + const remoteVideoWasBlocked = this._remoteVideoBlocked + + this._remoteVideoBlocked = undefined + + this.pc.getTransceivers().forEach(transceiver => { + if (transceiver.mid === 'video' && !transceiver.stopped) { + if (this._remoteVideoShouldBeBlocked) { + transceiver.direction = 'inactive' + + this._remoteVideoBlocked = true + } else { + this._remoteVideoBlocked = false + } + } + }) + + if (remoteVideoWasBlocked !== this._remoteVideoBlocked) { + this.emit('remoteVideoBlocked', this._remoteVideoBlocked) + } +} + Peer.prototype.answer = function() { this.pc.createAnswer().then(function(answer) { this.pc.setLocalDescription(answer).then(function() { @@ -816,6 +854,36 @@ Peer.prototype.handleLocalTrackEnabledChanged = function(track, stream) { } } +Peer.prototype.setRemoteVideoBlocked = function(remoteVideoBlocked) { + // If the HPB is not used or if it is used and this is a sender peer the + // remote video can not be blocked. + // Besides that the remote video is not blocked either if the signaling + // server does not support updating the subscribers; in that case a new + // connection would need to be established and due to this the audio would + // be interrupted during the connection change. + if (!this.receiverOnly || !this.parent.config.connection.hasFeature('update-sdp')) { + return + } + + this._remoteVideoShouldBeBlocked = remoteVideoBlocked + + // The "negotiationneeded" event is emitted if needed based on the direction + // changes. + // Note that there will be a video transceiver even if the remote + // participant is sending a null video track (either because there is a + // camera but the video is disabled or because the camera was removed during + // the call), so a renegotiation could be needed also in that case. + this.pc.getTransceivers().forEach(transceiver => { + if (transceiver.mid === 'video' && !transceiver.stopped) { + if (remoteVideoBlocked) { + transceiver.direction = 'inactive' + } else { + transceiver.direction = 'recvonly' + } + } + }) +} + Peer.prototype.handleRemoteStreamAdded = function(event) { const self = this if (this.stream) { diff --git a/src/utils/webrtc/webrtc.js b/src/utils/webrtc/webrtc.js index ec2c54d7754..c5b97f7abbc 100644 --- a/src/utils/webrtc/webrtc.js +++ b/src/utils/webrtc/webrtc.js @@ -367,6 +367,10 @@ function usersChanged(signaling, newUsers, disconnectedSessionIds) { // TODO(jojo): Already create peer object to avoid duplicate offers. signaling.requestOffer(user, 'video') + // Clearing the previous delayedConnectionToPeer should not be + // needed here, but just in case. + clearInterval(delayedConnectionToPeer[user.sessionId]) + delayedConnectionToPeer[user.sessionId] = setInterval(function() { console.debug('No offer received for new peer, request offer again') @@ -781,6 +785,8 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local signaling.requestOffer(peer.id, 'video') + clearInterval(delayedConnectionToPeer[peer.id]) + delayedConnectionToPeer[peer.id] = setInterval(function() { console.debug('No offer received, request offer again', peer) @@ -926,6 +932,41 @@ export default function initWebRtc(signaling, _callParticipantCollection, _local */ function setHandlerForNegotiationNeeded(peer) { peer.pc.addEventListener('negotiationneeded', function() { + // When the HPB is used and the negotiation is needed for a receiver + // peer (for example, to block the received video) there is no need + // to force a full reconnection, it is enough to reconnect only that + // peer. + if (signaling.hasFeature('mcu') && peer.id !== signaling.getSessionId()) { + // If possible update connection rather than creating a new one. + let update = signaling.hasFeature('update-sdp') + + // Create a connection if the current one has failed, as it + // would require an ICE restart rather than update to recover. + if (update && (peer.pc.iceConnectionState === 'failed' || peer.pc.connectionState === 'failed')) { + update = false + } + + // If the connection needs to be updated but a new connection + // (or another update) is already pending ignore the new update. + // If a new connection needs to be created rather than updated + // then force it even if there is another one already pending. + if (update && delayedConnectionToPeer[peer.id]) { + return + } + + signaling.requestOffer(peer.id, 'video', update ? peer.sid : undefined) + + clearInterval(delayedConnectionToPeer[peer.id]) + + delayedConnectionToPeer[peer.id] = setInterval(function() { + console.debug('No offer received, request offer again' + update ? '(update)' : '', peer) + + signaling.requestOffer(peer.id, 'video', update ? peer.sid : undefined) + }, 10000) + + return + } + // Negotiation needed will be first triggered before the connection // is established, but forcing a reconnection should be done only // once the connection was established.