add handler for participants speaking status changes

Signed-off-by: Maksim Sukharev <antreesy.web@gmail.com>

3 files changed, 197 insertions, 0 deletions

diff --git a/src/store/participantsStore.js b/src/store/participantsStore.js
index b63ad6d28..e8b96ffa6 100644
--- a/src/store/participantsStore.js
+++ b/src/store/participantsStore.js
@@ -57,6 +57,8 @@ const state = {
 	},
 	typing: {
 	},
+	speaking: {
+	},
 }
 
 const getters = {
@@ -322,6 +324,58 @@ const mutations = {
 	},
 
 	/**
+	 * Sets the speaking status of a participant in a conversation / call.
+	 *
+	 * Note that "updateParticipant" should not be called to add a "speaking"
+	 * property to an existing participant, as the participant would be reset
+	 * when the participants are purged whenever they are fetched again.
+	 * Similarly, "addParticipant" can not be called either to add a participant
+	 * if it was not fetched yet but the signaling reported it as being speaking,
+	 * as the attendeeId would be unknown.
+	 *
+	 * @param {object} state - current store state.
+	 * @param {object} data - the wrapping object.
+	 * @param {string} data.token - the conversation token participant is speaking in.
+	 * @param {string} data.sessionId - the Nextcloud session ID of the participant.
+	 * @param {boolean} data.speaking - whether the participant is speaking or not
+	 */
+	setSpeaking(state, { token, sessionId, speaking }) {
+		// create a dummy object for current call
+		if (!state.speaking[token]) {
+			Vue.set(state.speaking, token, {})
+		}
+		if (!state.speaking[token][sessionId]) {
+			Vue.set(state.speaking[token], sessionId, { speaking: null, lastTimestamp: 0, totalCountedTime: 0 })
+		}
+
+		const currentTimestamp = Date.now()
+		const currentSpeakingState = state.speaking[token][sessionId].speaking
+
+		// when speaking has stopped, update the total talking time
+		if (!speaking && state.speaking[token][sessionId].lastTimestamp) {
+			state.speaking[token][sessionId].totalCountedTime += (currentTimestamp - state.speaking[token][sessionId].lastTimestamp)
+		}
+
+		// don't change state for consecutive identical signals
+		if (currentSpeakingState !== speaking) {
+			state.speaking[token][sessionId].speaking = speaking
+			state.speaking[token][sessionId].lastTimestamp = currentTimestamp
+		}
+	},
+
+	/**
+	 * Purge the speaking information for recent call when local participant leaves call
+	 * (including cases when the call ends for everyone).
+	 *
+	 * @param {object} state - current store state.
+	 * @param {object} data - the wrapping object.
+	 * @param {string} data.token - the conversation token.
+	 */
+	purgeSpeakingStore(state, { token }) {
+		Vue.delete(state.speaking, token)
+	},
+
+	/**
 	 * Purge a given conversation from the previously added participants.
 	 *
 	 * @param {object} state - current store state.
@@ -750,6 +804,14 @@ const actions = {
 			context.commit('setTyping', { token, sessionId, typing: true, expirationTimeout })
 		}
 	},
+
+	setSpeaking(context, { token, sessionId, speaking }) {
+		context.commit('setSpeaking', { token, sessionId, speaking })
+	},
+
+	purgeSpeakingStore(context, { token }) {
+		context.commit('purgeSpeakingStore', { token })
+	},
 }
 
 export default { state, mutations, getters, actions }
diff --git a/src/utils/webrtc/SpeakingStatusHandler.js b/src/utils/webrtc/SpeakingStatusHandler.js
new file mode 100644
index 000000000..feb804b17
--- /dev/null
+++ b/src/utils/webrtc/SpeakingStatusHandler.js
@@ -0,0 +1,129 @@
+/**
+ *
+ * @copyright Copyright (c) 2023 Maksim Sukharev <antreesy.web@gmail.com>
+ *
+ * @author Maksim Sukharev <antreesy.web@gmail.com>
+ *
+ * @license AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/**
+ * Helper to handle speaking status changes notified by call models.
+ *
+ * The store is updated when local or remote participants change their speaking status.
+ * It is expected that the speaking status of participant will be
+ * modified only when the current conversation is joined and call is started.
+ */
+export default class SpeakingStatusHandler {
+
+	// Constants, properties
+	#store
+	#localMediaModel
+	#callParticipantCollection
+
+	// Methods (bound to have access to 'this')
+	#handleAddParticipantBound
+	#handleRemoveParticipantBound
+	#handleLocalSpeakingBound
+	#handleSpeakingBound
+
+	constructor(store, localMediaModel, callParticipantCollection) {
+		this.#store = store
+		this.#localMediaModel = localMediaModel
+		this.#callParticipantCollection = callParticipantCollection
+
+		this.#handleAddParticipantBound = this.#handleAddParticipant.bind(this)
+		this.#handleRemoveParticipantBound = this.#handleRemoveParticipant.bind(this)
+		this.#handleLocalSpeakingBound = this.#handleLocalSpeaking.bind(this)
+		this.#handleSpeakingBound = this.#handleSpeaking.bind(this)
+
+		this.#localMediaModel.on('change:speaking', this.#handleLocalSpeakingBound)
+		this.#localMediaModel.on('change:stoppedSpeaking', this.#handleLocalSpeakingBound)
+
+		this.#callParticipantCollection.on('add', this.#handleAddParticipantBound)
+		this.#callParticipantCollection.on('remove', this.#handleRemoveParticipantBound)
+	}
+
+	/**
+	 * Destroy a handler, remove all listeners, purge the speaking state from store
+	 */
+	destroy() {
+		this.#localMediaModel.off('change:speaking', this.#handleLocalSpeakingBound)
+		this.#localMediaModel.off('change:stoppedSpeaking', this.#handleLocalSpeakingBound)
+
+		this.#callParticipantCollection.off('add', this.#handleAddParticipantBound)
+		this.#callParticipantCollection.off('remove', this.#handleRemoveParticipantBound)
+
+		this.#callParticipantCollection.callParticipantModels.forEach(callParticipantModel => {
+			callParticipantModel.off('change:speaking', this.#handleSpeakingBound)
+			callParticipantModel.off('change:stoppedSpeaking', this.#handleSpeakingBound)
+		})
+
+		this.#store.dispatch('purgeSpeakingStore', { token: this.#store.getters.getToken() })
+	}
+
+	/**
+	 * Add listeners for speaking status changes on added participants model
+	 *
+	 * @param {object} callParticipantCollection the collection of external participant models
+	 * @param {object} callParticipantModel the added participant model
+	 */
+	#handleAddParticipant(callParticipantCollection, callParticipantModel) {
+		callParticipantModel.on('change:speaking', this.#handleSpeakingBound)
+		callParticipantModel.on('change:stoppedSpeaking', this.#handleSpeakingBound)
+	}
+
+	/**
+	 * Remove listeners for speaking status changes on removed participants model
+	 *
+	 * @param {object} callParticipantCollection the collection of external participant models
+	 * @param {object} callParticipantModel the removed participant model
+	 */
+	#handleRemoveParticipant(callParticipantCollection, callParticipantModel) {
+		callParticipantModel.off('change:speaking', this.#handleSpeakingBound)
+		callParticipantModel.off('change:stoppedSpeaking', this.#handleSpeakingBound)
+	}
+
+	/**
+	 * Dispatch speaking status of local participant to the store
+	 *
+	 * @param {object} localMediaModel the local media model
+	 * @param {boolean} speaking whether the participant is speaking or not
+	 */
+	#handleLocalSpeaking(localMediaModel, speaking) {
+		this.#store.dispatch('setSpeaking', {
+			token: this.#store.getters.getToken(),
+			sessionId: this.#store.getters.getSessionId(),
+			speaking,
+		})
+	}
+
+	/**
+	 * Dispatch speaking status of participant to the store
+	 *
+	 * @param {object} callParticipantModel the participant model
+	 * @param {boolean} speaking whether the participant is speaking or not
+	 */
+	#handleSpeaking(callParticipantModel, speaking) {
+		this.#store.dispatch('setSpeaking', {
+			token: this.#store.getters.getToken(),
+			sessionId: callParticipantModel.attributes.nextcloudSessionId,
+			speaking,
+		})
+	}
+
+}
diff --git a/src/utils/webrtc/index.js b/src/utils/webrtc/index.js
index 31b48be1c..6853fba53 100644
--- a/src/utils/webrtc/index.js
+++ b/src/utils/webrtc/index.js
@@ -37,6 +37,7 @@ import LocalMediaModel from './models/LocalMediaModel.js'
 import SentVideoQualityThrottler from './SentVideoQualityThrottler.js'
 import './shims/MediaStream.js'
 import './shims/MediaStreamTrack.js'
+import SpeakingStatusHandler from './SpeakingStatusHandler.js'
 import initWebRtc from './webrtc.js'
 
 let webRtc = null
@@ -46,6 +47,7 @@ const localMediaModel = new LocalMediaModel()
 const mediaDevicesManager = new MediaDevicesManager()
 let callAnalyzer = null
 let sentVideoQualityThrottler = null
+let speakingStatusHandler = null
 
 // This does not really belongs here, as it is unrelated to WebRTC, but it is
 // included here for the time being until signaling and WebRTC are split.
@@ -213,6 +215,7 @@ async function signalingJoinCall(token, flags, silent) {
 		setupWebRtc()
 
 		sentVideoQualityThrottler = new SentVideoQualityThrottler(localMediaModel, callParticipantCollection, webRtc.webrtc._videoTrackConstrainer)
+		speakingStatusHandler = new SpeakingStatusHandler(store, localMediaModel, callParticipantCollection)
 
 		if (signaling.hasFeature('mcu')) {
 			callAnalyzer = new CallAnalyzer(localMediaModel, localCallParticipantModel, callParticipantCollection)
@@ -416,6 +419,9 @@ async function signalingLeaveCall(token, all = false) {
 	sentVideoQualityThrottler.destroy()
 	sentVideoQualityThrottler = null
 
+	speakingStatusHandler.destroy()
+	speakingStatusHandler = null
+
 	callAnalyzer.destroy()
 	callAnalyzer = null