Merge pull request #12497 from nextcloud/fix-audio-in-inactive-tab-in-safari

Fix audio in inactive tab in Safari

4 files changed, 905 insertions, 20 deletions

diff --git a/src/utils/webrtc/CallParticipantsAudioPlayer.js b/src/utils/webrtc/CallParticipantsAudioPlayer.js
new file mode 100644
index 000000000..4c366e900
--- /dev/null
+++ b/src/utils/webrtc/CallParticipantsAudioPlayer.js
@@ -0,0 +1,178 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+import attachMediaStream from '../attachmediastream.js'
+
+/**
+ * Player for audio of call participants.
+ *
+ * The player keeps track of the participants added and removed to the
+ * CallParticipantCollection and plays their audio as needed. Note that in the
+ * case of regular audio whether the audio is muted or not depends on
+ * "audioAvailable"; screen share audio, on the other hand, is always treated as
+ * unmuted.
+ *
+ * By default, the audio or screen audio of each participant is played on its
+ * own audio element. Alternatively, the audio of all participants can be mixed
+ * and played by a single element created when the player is created by setting
+ * "mixAudio = true" in the constructor.
+ *
+ * Once the player is no longer needed "destroy()" must be called to stop
+ * tracking the participants and playing audio.
+ *
+ * @param {object} callParticipantCollection the CallParticipantCollection.
+ * @param {boolean} mixAudio true to mix and play all audio in a single audio
+ *        element, false to play each audio on its own audio element.
+ */
+export default function CallParticipantsAudioPlayer(callParticipantCollection, mixAudio = false) {
+	this._callParticipantCollection = callParticipantCollection
+
+	this._mixAudio = mixAudio
+
+	if (this._mixAudio) {
+		this._audioContext = new (window.AudioContext || window.webkitAudioContext)()
+		this._audioDestination = this._audioContext.createMediaStreamDestination()
+		this._audioElement = attachMediaStream(this._audioDestination.stream, null, { audio: true })
+		this._audioNodes = new Map()
+	} else {
+		this._audioElements = new Map()
+	}
+
+	this._handleCallParticipantAddedBound = this._handleCallParticipantAdded.bind(this)
+	this._handleCallParticipantRemovedBound = this._handleCallParticipantRemoved.bind(this)
+	this._handleStreamChangedBound = this._handleStreamChanged.bind(this)
+	this._handleScreenChangedBound = this._handleScreenChanged.bind(this)
+	this._handleAudioAvailableChangedBound = this._handleAudioAvailableChanged.bind(this)
+
+	this._callParticipantCollection.on('add', this._handleCallParticipantAddedBound)
+	this._callParticipantCollection.on('remove', this._handleCallParticipantRemovedBound)
+
+	this._callParticipantCollection.callParticipantModels.value.forEach(callParticipantModel => {
+		this._handleCallParticipantAddedBound(this._callParticipantCollection, callParticipantModel)
+	})
+}
+
+CallParticipantsAudioPlayer.prototype = {
+
+	destroy() {
+		this._callParticipantCollection.off('add', this._handleCallParticipantAddedBound)
+		this._callParticipantCollection.off('remove', this._handleCallParticipantRemovedBound)
+
+		this._callParticipantCollection.callParticipantModels.value.forEach(callParticipantModel => {
+			this._handleCallParticipantRemovedBound(this._callParticipantCollection, callParticipantModel)
+		})
+
+		if (this._mixAudio) {
+			this._audioElement.srcObject = null
+			this._audioContext.close()
+		}
+	},
+
+	_handleCallParticipantAdded(callParticipantCollection, callParticipantModel) {
+		callParticipantModel.on('change:stream', this._handleStreamChangedBound)
+		callParticipantModel.on('change:screen', this._handleScreenChangedBound)
+		callParticipantModel.on('change:audioAvailable', this._handleAudioAvailableChangedBound)
+
+		this._handleStreamChangedBound(callParticipantModel, callParticipantModel.get('stream'))
+		this._handleScreenChangedBound(callParticipantModel, callParticipantModel.get('screen'))
+	},
+
+	_handleCallParticipantRemoved(callParticipantCollection, callParticipantModel) {
+		callParticipantModel.off('change:stream', this._handleStreamChangedBound)
+		callParticipantModel.off('change:screen', this._handleScreenChangedBound)
+		callParticipantModel.off('change:audioAvailable', this._handleAudioAvailableChangedBound)
+
+		this._handleStreamChangedBound(callParticipantModel, null)
+		this._handleScreenChangedBound(callParticipantModel, null)
+	},
+
+	_handleStreamChanged(callParticipantModel, stream) {
+		const id = callParticipantModel.get('peerId') + '-stream'
+		const mute = !callParticipantModel.get('audioAvailable')
+		if (this._mixAudio) {
+			this._setAudioNode(id, stream, mute)
+		} else {
+			this._setAudioElement(id, stream, mute)
+		}
+	},
+
+	_handleScreenChanged(callParticipantModel, screen) {
+		const id = callParticipantModel.get('peerId') + '-screen'
+		if (this._mixAudio) {
+			this._setAudioNode(id, screen)
+		} else {
+			this._setAudioElement(id, screen)
+		}
+	},
+
+	_setAudioNode(id, stream, mute = false) {
+		const audioNode = this._audioNodes.get(id)
+		if (audioNode) {
+			if (audioNode.connected) {
+				audioNode.audioSource.disconnect(this._audioDestination)
+			}
+
+			this._audioNodes.delete(id)
+		}
+
+		if (!stream) {
+			return
+		}
+
+		const audioSource = this._audioContext.createMediaStreamSource(stream)
+		if (!mute) {
+			audioSource.connect(this._audioDestination)
+		}
+
+		this._audioNodes.set(id, { audioSource, connected: !mute })
+	},
+
+	_setAudioElement(id, stream, mute = false) {
+		let audioElement = this._audioElements.get(id)
+		if (audioElement) {
+			audioElement.srcObject = null
+
+			this._audioElements.delete(id)
+		}
+
+		if (!stream) {
+			return
+		}
+
+		audioElement = attachMediaStream(stream, null, { audio: true })
+		if (mute) {
+			audioElement.muted = true
+		}
+
+		this._audioElements.set(id, audioElement)
+	},
+
+	_handleAudioAvailableChanged(callParticipantModel, audioAvailable) {
+		if (this._mixAudio) {
+			const audioNode = this._audioNodes.get(callParticipantModel.get('peerId') + '-stream')
+			if (!audioNode) {
+				return
+			}
+
+			if (audioAvailable && !audioNode.connected) {
+				audioNode.audioSource.connect(this._audioDestination)
+				audioNode.connected = true
+			} else if (!audioAvailable && audioNode.connected) {
+				audioNode.audioSource.disconnect(this._audioDestination)
+				audioNode.connected = false
+			}
+
+			return
+		}
+
+		const audioElement = this._audioElements.get(callParticipantModel.get('peerId') + '-stream')
+		if (!audioElement) {
+			return
+		}
+
+		audioElement.muted = !audioAvailable
+	},
+
+}
diff --git a/src/utils/webrtc/CallParticipantsAudioPlayer.spec.js b/src/utils/webrtc/CallParticipantsAudioPlayer.spec.js
new file mode 100644
index 000000000..1e25172ab
--- /dev/null
+++ b/src/utils/webrtc/CallParticipantsAudioPlayer.spec.js
@@ -0,0 +1,716 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+import { ref } from 'vue'
+
+import CallParticipantsAudioPlayer from './CallParticipantsAudioPlayer.js'
+import EmitterMixin from '../EmitterMixin.js'
+
+/**
+ * Stub of CallParticipantModel with just the attributes and methods used by
+ * CallParticipantsAudioPlayer.
+ *
+ * @param {string} peerId the ID of the peer
+ */
+function CallParticipantModelStub(peerId) {
+	this._superEmitterMixin()
+
+	this.attributes = {
+		peerId,
+	}
+
+	this.get = (key) => {
+		return this.attributes[key]
+	}
+
+	this.set = (key, value) => {
+		this.attributes[key] = value
+
+		this._trigger('change:' + key, [value])
+	}
+}
+EmitterMixin.apply(CallParticipantModelStub.prototype)
+
+/**
+ * Stub of CallParticipantCollection with just the attributes and methods used
+ * by CallParticipantsAudioPlayer.
+ */
+function CallParticipantCollectionStub() {
+	this._superEmitterMixin()
+
+	this.callParticipantModels = ref([])
+}
+EmitterMixin.apply(CallParticipantCollectionStub.prototype)
+
+/**
+ * Mock of MediaStream with an id for easier tracking in tests.
+ *
+ * HTMLAudioElement requires srcObject to conform to the MediaStream interface,
+ * but in jsdom anything can be assigned, so the mock is kept to a minimum.
+ *
+ * @param {string} id the id for the stream.
+ */
+function MediaStreamMock(id) {
+	this.id = id
+}
+
+describe('CallParticipantsAudioPlayer', () => {
+
+	let callParticipantCollection
+	let callParticipantsAudioPlayer
+
+	/**
+	 * Adds a CallParticipantModel to the collection.
+	 *
+	 * The participant must not be yet in the collection.
+	 *
+	 * @param {object} callParticipantModel the CallParticipantModel to add.
+	 */
+	function addCallParticipantModel(callParticipantModel) {
+		callParticipantCollection.callParticipantModels.value.push(callParticipantModel)
+
+		callParticipantCollection._trigger('add', [callParticipantModel])
+	}
+
+	/**
+	 * Removes a CallParticipantModel from the collection.
+	 *
+	 * The participant must be in the collection.
+	 *
+	 * @param {object} callParticipantModel the CallParticipantModel to remove.
+	 */
+	function removeCallParticipantModel(callParticipantModel) {
+		const index = callParticipantCollection.callParticipantModels.value.indexOf(callParticipantModel)
+		callParticipantCollection.callParticipantModels.value.splice(index, 1)
+
+		callParticipantCollection._trigger('remove', [callParticipantModel])
+	}
+
+	/**
+	 * Asserts that the audio element with the given id has the expected
+	 * srcObject and muted value.
+	 *
+	 * @param {string} audioElementId the id of the audio element in the player.
+	 * @param {object|null} expectedSrcObject the expected srcObject in the
+	 *        element.
+	 * @param {boolean} expectedMuted the expected muted value in the element.
+	 */
+	function assertAudioElement(audioElementId, expectedSrcObject, expectedMuted) {
+		expect(callParticipantsAudioPlayer._audioElements.get(audioElementId)).not.toBe(null)
+		expect(callParticipantsAudioPlayer._audioElements.get(audioElementId).tagName.toLowerCase()).toBe('audio')
+		expect(callParticipantsAudioPlayer._audioElements.get(audioElementId).srcObject).toBe(expectedSrcObject)
+		expect(callParticipantsAudioPlayer._audioElements.get(audioElementId).muted).toBe(expectedMuted)
+	}
+
+	beforeEach(() => {
+		callParticipantCollection = new CallParticipantCollectionStub()
+
+		callParticipantsAudioPlayer = new CallParticipantsAudioPlayer(callParticipantCollection)
+	})
+
+	afterEach(() => {
+		jest.clearAllMocks()
+	})
+
+	describe('constructor', () => {
+		test('without participants', () => {
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(0)
+		})
+
+		test('with several participants', () => {
+			const callParticipantModel1 = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel1)
+
+			const callParticipantModel2 = new CallParticipantModelStub('peerId2')
+			addCallParticipantModel(callParticipantModel2)
+
+			callParticipantModel2.attributes.audioAvailable = false
+
+			const stream2 = new MediaStreamMock('stream2')
+			callParticipantModel2.set('stream', stream2)
+
+			const callParticipantModel3 = new CallParticipantModelStub('peerId3')
+			addCallParticipantModel(callParticipantModel3)
+
+			callParticipantModel3.attributes.audioAvailable = true
+
+			const stream3 = new MediaStreamMock('stream3')
+			callParticipantModel3.set('stream', stream3)
+			const screen3 = new MediaStreamMock('screen3')
+			callParticipantModel3.set('screen', screen3)
+
+			callParticipantsAudioPlayer = new CallParticipantsAudioPlayer(callParticipantCollection)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(3)
+			assertAudioElement('peerId2-stream', stream2, true)
+			assertAudioElement('peerId3-stream', stream3, false)
+			assertAudioElement('peerId3-screen', screen3, false)
+		})
+
+		test('add stream and screen', () => {
+			const callParticipantModel1 = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel1)
+
+			callParticipantsAudioPlayer = new CallParticipantsAudioPlayer(callParticipantCollection)
+
+			const stream1 = new MediaStreamMock('stream1')
+			callParticipantModel1.set('stream', stream1)
+			const screen1 = new MediaStreamMock('screen1')
+			callParticipantModel1.set('screen', screen1)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(2)
+			assertAudioElement('peerId1-stream', stream1, true)
+			assertAudioElement('peerId1-screen', screen1, false)
+		})
+
+		test('change audio available', () => {
+			const callParticipantModel1 = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel1)
+
+			callParticipantModel1.attributes.audioAvailable = false
+
+			const stream1 = new MediaStreamMock('stream1')
+			callParticipantModel1.set('stream', stream1)
+			const screen1 = new MediaStreamMock('screen1')
+			callParticipantModel1.set('screen', screen1)
+
+			callParticipantsAudioPlayer = new CallParticipantsAudioPlayer(callParticipantCollection)
+
+			callParticipantModel1.set('audioAvailable', true)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(2)
+			assertAudioElement('peerId1-stream', stream1, false)
+			assertAudioElement('peerId1-screen', screen1, false)
+		})
+
+		test('remove participant, stream and screen', () => {
+			const callParticipantModel1 = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel1)
+
+			const stream1 = new MediaStreamMock('stream1')
+			callParticipantModel1.set('stream', stream1)
+			const screen1 = new MediaStreamMock('screen1')
+			callParticipantModel1.set('screen', screen1)
+
+			const callParticipantModel2 = new CallParticipantModelStub('peerId2')
+			addCallParticipantModel(callParticipantModel2)
+
+			const stream2 = new MediaStreamMock('stream2')
+			callParticipantModel2.set('stream', stream2)
+			const screen2 = new MediaStreamMock('screen2')
+			callParticipantModel2.set('screen', screen2)
+
+			callParticipantsAudioPlayer = new CallParticipantsAudioPlayer(callParticipantCollection)
+
+			callParticipantModel1.set('stream', null)
+			callParticipantModel1.set('screen', null)
+
+			removeCallParticipantModel(callParticipantModel2)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(0)
+		})
+	})
+
+	describe('add stream and screen', () => {
+		test('stream with available audio', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.attributes.audioAvailable = true
+
+			const stream = new MediaStreamMock('stream1')
+			callParticipantModel.set('stream', stream)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(1)
+			assertAudioElement('peerId1-stream', stream, false)
+		})
+
+		test('stream without available audio', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.attributes.audioAvailable = false
+
+			const stream = new MediaStreamMock('stream1')
+			callParticipantModel.set('stream', stream)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(1)
+			assertAudioElement('peerId1-stream', stream, true)
+		})
+
+		test('screen', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			const screen = new MediaStreamMock('screen1')
+			callParticipantModel.set('screen', screen)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(1)
+			assertAudioElement('peerId1-screen', screen, false)
+		})
+
+		test('stream and screen', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.attributes.audioAvailable = false
+
+			const stream = new MediaStreamMock('stream1')
+			callParticipantModel.set('stream', stream)
+			const screen = new MediaStreamMock('screen1')
+			callParticipantModel.set('screen', screen)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(2)
+			assertAudioElement('peerId1-stream', stream, true)
+			assertAudioElement('peerId1-screen', screen, false)
+		})
+
+		// This should not happen (the stream and screen are expected to be set
+		// once the participant is already in the collection), but test it just
+		// in case.
+		test('participant', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+
+			callParticipantModel.attributes.audioAvailable = false
+
+			const stream = new MediaStreamMock('stream1')
+			callParticipantModel.set('stream', stream)
+			const screen = new MediaStreamMock('screen1')
+			callParticipantModel.set('screen', screen)
+
+			addCallParticipantModel(callParticipantModel)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(2)
+			assertAudioElement('peerId1-stream', stream, true)
+			assertAudioElement('peerId1-screen', screen, false)
+		})
+
+		test('several participants, streams and screens', () => {
+			const callParticipantModel1 = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel1)
+			const callParticipantModel2 = new CallParticipantModelStub('peerId2')
+			addCallParticipantModel(callParticipantModel2)
+
+			callParticipantModel1.attributes.audioAvailable = false
+
+			const screen1 = new MediaStreamMock('screen1')
+			callParticipantModel1.set('screen', screen1)
+			const stream1 = new MediaStreamMock('stream1')
+			callParticipantModel1.set('stream', stream1)
+
+			const screen2 = new MediaStreamMock('screen2')
+			callParticipantModel2.set('screen', screen2)
+
+			const callParticipantModel3 = new CallParticipantModelStub('peerId3')
+			addCallParticipantModel(callParticipantModel3)
+
+			callParticipantModel3.attributes.audioAvailable = true
+
+			const stream3 = new MediaStreamMock('stream3')
+			callParticipantModel3.set('stream', stream3)
+
+			const callParticipantModel4 = new CallParticipantModelStub('peerId4')
+
+			callParticipantModel4.attributes.audioAvailable = true
+
+			const stream4 = new MediaStreamMock('stream4')
+			callParticipantModel4.set('stream', stream4)
+
+			addCallParticipantModel(callParticipantModel4)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(5)
+			assertAudioElement('peerId1-stream', stream1, true)
+			assertAudioElement('peerId1-screen', screen1, false)
+			assertAudioElement('peerId2-screen', screen2, false)
+			assertAudioElement('peerId3-stream', stream3, false)
+			assertAudioElement('peerId4-stream', stream4, false)
+		})
+
+		// This should not happen (the previous stream or screen is expected to
+		// be removed first), but test it just in case.
+		test('replace stream and screen', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.attributes.audioAvailable = false
+
+			const stream1 = new MediaStreamMock('stream1')
+			callParticipantModel.set('stream', stream1)
+			const screen1 = new MediaStreamMock('screen1')
+			callParticipantModel.set('screen', screen1)
+
+			const audioElementStream1 = callParticipantsAudioPlayer._audioElements.get('peerId1-stream')
+			const audioElementScreen1 = callParticipantsAudioPlayer._audioElements.get('peerId1-screen')
+
+			callParticipantModel.attributes.audioAvailable = true
+
+			const stream2 = new MediaStreamMock('stream2')
+			callParticipantModel.set('stream', stream2)
+			const screen2 = new MediaStreamMock('screen2')
+			callParticipantModel.set('screen', screen2)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(2)
+			assertAudioElement('peerId1-stream', stream2, false)
+			assertAudioElement('peerId1-screen', screen2, false)
+			expect(audioElementStream1.srcObject).toBe(null)
+			expect(audioElementScreen1.srcObject).toBe(null)
+		})
+	})
+
+	describe('change audio available', () => {
+		test('not available with stream', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.attributes.audioAvailable = true
+
+			const stream = new MediaStreamMock('stream1')
+			callParticipantModel.set('stream', stream)
+
+			callParticipantModel.set('audioAvailable', false)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(1)
+			assertAudioElement('peerId1-stream', stream, true)
+		})
+
+		test('not available without stream', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.set('audioAvailable', false)
+
+			expect(callParticipantsAudioPlayer._audioElements.size).toBe(0)
+		})
+
+		test('not available with screen', () => {
+			const callParticipantModel = new CallParticipantModelStub('peerId1')
+			addCallParticipantModel(callParticipantModel)
+
+			callParticipantModel.attributes.audioAvailable = true
+
+			const screen = new MediaStreamMock('screen1')
+			callParticipantModel.set('screen', screen)
+
+			callParticipantModel.set('audioAvailable', false)
+
+			expe