From 72d6da611b38105c5473ed64bbea0ba41a5a7d72 Mon Sep 17 00:00:00 2001 From: Horatiu Muresan Date: Fri, 7 Jul 2023 18:07:22 +0300 Subject: [PATCH] feat(external-api) add commands/event for remote screen capture and facing mode - added command for screencapture remotely - this will trigger a response caught in an event handler - added command for setting the camera facing mode remotely - enhanced toggleVideo command to optionally accept the facing mode --- modules/API/API.js | 53 ++++++-- modules/API/external/external_api.js | 5 +- react/features/base/media/constants.ts | 9 ++ react/features/base/tracks/actions.web.ts | 42 ++++++- react/features/base/tracks/constants.ts | 4 + react/features/base/tracks/middleware.web.ts | 31 +++++ .../{middleware.ts => middleware.any.ts} | 3 +- .../features/large-video/middleware.native.ts | 2 + react/features/large-video/middleware.web.ts | 116 ++++++++++++++++++ 9 files changed, 251 insertions(+), 14 deletions(-) create mode 100644 react/features/base/tracks/constants.ts rename react/features/large-video/{middleware.ts => middleware.any.ts} (98%) create mode 100644 react/features/large-video/middleware.native.ts create mode 100644 react/features/large-video/middleware.web.ts diff --git a/modules/API/API.js b/modules/API/API.js index cd12dc45297..fd3724d1ae5 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -30,7 +30,7 @@ import { toggleDialog } from '../../react/features/base/dialog/actions'; import { isSupportedBrowser } from '../../react/features/base/environment/environment'; import { parseJWTFromURLParams } from '../../react/features/base/jwt/functions'; import JitsiMeetJS, { JitsiRecordingConstants } from '../../react/features/base/lib-jitsi-meet'; -import { MEDIA_TYPE, VIDEO_TYPE } from '../../react/features/base/media/constants'; +import { CAPTURE_SCREENSHOT_MESSAGE, MEDIA_TYPE, VIDEO_TYPE } from '../../react/features/base/media/constants'; import { grantModerator, kickParticipant, @@ -50,8 +50,8 @@ import { } from '../../react/features/base/participants/functions'; import { updateSettings } from '../../react/features/base/settings/actions'; import { getDisplayName } from '../../react/features/base/settings/functions.web'; -import { toggleCamera } from '../../react/features/base/tracks/actions.any'; -import { isToggleCameraEnabled } from '../../react/features/base/tracks/functions'; +import { setCameraFacingMode } from '../../react/features/base/tracks/actions.web'; +import { CAMERA_FACING_MODE_MESSAGE } from '../../react/features/base/tracks/constants'; import { autoAssignToBreakoutRooms, closeBreakoutRoom, @@ -395,12 +395,8 @@ function initCommands() { sendAnalytics(createApiEvent('film.strip.resize')); APP.store.dispatch(resizeFilmStrip(options.width)); }, - 'toggle-camera': () => { - if (!isToggleCameraEnabled(APP.store.getState())) { - return; - } - - APP.store.dispatch(toggleCamera()); + 'toggle-camera': facingMode => { + APP.store.dispatch(setCameraFacingMode(facingMode)); }, 'toggle-camera-mirror': () => { const state = APP.store.getState(); @@ -529,6 +525,29 @@ function initCommands() { logger.error('Failed sending endpoint text message', err); } }, + 'send-capture-screenshot-message': to => { + if (!to || !isLocalParticipantModerator(APP.store.getState())) { + logger.warn('Participant id not set or participant not moderator'); + + return; + } + + APP.conference.sendEndpointMessage(to, { + name: CAPTURE_SCREENSHOT_MESSAGE + }); + }, + 'send-camera-facing-mode-message': (to, facingMode) => { + if (!to || !isLocalParticipantModerator(APP.store.getState())) { + logger.warn('Participant id not set or participant not moderator'); + + return; + } + + APP.conference.sendEndpointMessage(to, { + name: CAMERA_FACING_MODE_MESSAGE, + facingMode + }); + }, 'overwrite-names': participantList => { logger.debug('Overwrite names command received'); @@ -1775,6 +1794,22 @@ class API { }); } + /** + * Notify external application (if API is enabled) that a participant's screen + * capture has been received. + * + * @param {Object} senderInfo - The sender's info. + * @param {string} dataURL - The image data. + * @returns {void} + */ + notifyLargeVideoScreenshotReceived(senderInfo, dataURL) { + this._sendEvent({ + name: 'screen-capture-received', + senderInfo, + dataURL + }); + } + /** * Notify external application (if API is enabled) that the dominant speaker * has been turned on/off. diff --git a/modules/API/external/external_api.js b/modules/API/external/external_api.js index ef2c5d69b5a..acb6b761f16 100644 --- a/modules/API/external/external_api.js +++ b/modules/API/external/external_api.js @@ -54,6 +54,8 @@ const commands = { removeBreakoutRoom: 'remove-breakout-room', resizeFilmStrip: 'resize-film-strip', resizeLargeVideo: 'resize-large-video', + sendCameraFacingMode: 'send-camera-facing-mode-message', + sendCaptureScreenshot: 'send-capture-screenshot-message', sendChatMessage: 'send-chat-message', sendEndpointTextMessage: 'send-endpoint-text-message', sendParticipantToRoom: 'send-participant-to-room', @@ -111,6 +113,7 @@ const events = { 'data-channel-opened': 'dataChannelOpened', 'device-list-changed': 'deviceListChanged', 'display-name-change': 'displayNameChange', + 'dominant-speaker-changed': 'dominantSpeakerChanged', 'email-change': 'emailChange', 'error-occurred': 'errorOccurred', 'endpoint-text-message-received': 'endpointTextMessageReceived', @@ -151,8 +154,8 @@ const events = { 'video-availability-changed': 'videoAvailabilityChanged', 'video-mute-status-changed': 'videoMuteStatusChanged', 'video-quality-changed': 'videoQualityChanged', + 'screen-capture-received': 'screenCaptureReceived', 'screen-sharing-status-changed': 'screenSharingStatusChanged', - 'dominant-speaker-changed': 'dominantSpeakerChanged', 'subject-change': 'subjectChange', 'suspend-detected': 'suspendDetected', 'tile-view-changed': 'tileViewChanged', diff --git a/react/features/base/media/constants.ts b/react/features/base/media/constants.ts index 90497a2f459..d6b42d65ba8 100644 --- a/react/features/base/media/constants.ts +++ b/react/features/base/media/constants.ts @@ -8,6 +8,11 @@ export const CAMERA_FACING_MODE = { USER: 'user' }; +/** + * The payload name for the capture screenshot remotely message. + */ +export const CAPTURE_SCREENSHOT_MESSAGE = 'capture-screenshot-message'; + export type MediaType = 'audio' | 'video' | 'screenshare'; /** @@ -25,6 +30,10 @@ export const MEDIA_TYPE: { VIDEO: 'video' }; +/** + * The payload name for the received screenshot message. + */ +export const SEND_SCREENSHOT_MESSAGE = 'send-screenshot-message'; /* eslint-disable no-bitwise */ diff --git a/react/features/base/tracks/actions.web.ts b/react/features/base/tracks/actions.web.ts index ffe6915d1be..e57f11e9cfd 100644 --- a/react/features/base/tracks/actions.web.ts +++ b/react/features/base/tracks/actions.web.ts @@ -19,12 +19,15 @@ import { MEDIA_TYPE, VIDEO_TYPE } from '../media/constants'; import { addLocalTrack, - replaceLocalTrack + replaceLocalTrack, + toggleCamera } from './actions.any'; import { createLocalTracksF, getLocalDesktopTrack, - getLocalJitsiAudioTrack + getLocalJitsiAudioTrack, + getLocalVideoTrack, + isToggleCameraEnabled } from './functions'; import { IShareOptions, IToggleScreenSharingOptions } from './types'; @@ -263,3 +266,38 @@ async function _toggleScreenSharing( APP.API.notifyScreenSharingStatusChanged(enable, screensharingDetails); } } + +/** + * Sets the camera facing mode(environment/user). If facing mode not provided, it will do a toggle. + * + * @param {string | undefined} facingMode - The selected facing mode. + * @returns {void} + */ +export function setCameraFacingMode(facingMode: string | undefined) { + return async (dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + + if (!isToggleCameraEnabled(state)) { + return; + } + + if (!facingMode) { + dispatch(toggleCamera()); + + return; + } + + const tracks = state['features/base/tracks']; + const localVideoTrack = getLocalVideoTrack(tracks)?.jitsiTrack; + + if (!tracks || !localVideoTrack) { + return; + } + + const currentFacingMode = localVideoTrack.getCameraFacingMode(); + + if (currentFacingMode !== facingMode) { + dispatch(toggleCamera()); + } + }; +} diff --git a/react/features/base/tracks/constants.ts b/react/features/base/tracks/constants.ts new file mode 100644 index 00000000000..abbe4abc390 --- /dev/null +++ b/react/features/base/tracks/constants.ts @@ -0,0 +1,4 @@ +/** + * The payload name for remotely setting the camera facing mode message. + */ +export const CAMERA_FACING_MODE_MESSAGE = 'camera-facing-mode-message'; diff --git a/react/features/base/tracks/middleware.web.ts b/react/features/base/tracks/middleware.web.ts index cd6b2884eec..36668af777e 100644 --- a/react/features/base/tracks/middleware.web.ts +++ b/react/features/base/tracks/middleware.web.ts @@ -3,7 +3,10 @@ import { AnyAction } from 'redux'; import { IStore } from '../../app/types'; import { hideNotification } from '../../notifications/actions'; import { isPrejoinPageVisible } from '../../prejoin/functions'; +import { CONFERENCE_JOINED } from '../conference/actionTypes'; +import { IJitsiConference } from '../conference/reducer'; import { getAvailableDevices } from '../devices/actions.web'; +import { JitsiConferenceEvents } from '../lib-jitsi-meet'; import { setScreenshareMuted } from '../media/actions'; import { MEDIA_TYPE, @@ -20,10 +23,12 @@ import { TRACK_UPDATED } from './actionTypes'; import { + setCameraFacingMode, showNoDataFromSourceVideoError, toggleScreensharing, trackNoDataFromSourceNotificationInfoChanged } from './actions.web'; +import { CAMERA_FACING_MODE_MESSAGE } from './constants'; import { getTrackByJitsiTrack } from './functions.web'; @@ -121,11 +126,37 @@ MiddlewareRegistry.register(store => next => action => { return result; } + case CONFERENCE_JOINED: { + _addsetCameraFacingModeListener(action.conference); + break; + } } return next(action); }); +/** + * Registers listener for {@link JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED} that + * will perform various chat related activities. + * + * @param {IJitsiConference} conference - The conference. + * @returns {void} + */ +function _addsetCameraFacingModeListener(conference: IJitsiConference) { + conference.on( + JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + (...args: any) => { + if (args && args.length >= 2) { + const [ , eventData ] = args; + + if (eventData.name === CAMERA_FACING_MODE_MESSAGE) { + APP.store.dispatch(setCameraFacingMode(eventData.facingMode)); + } + } + } + ); +} + /** * Handles no data from source errors. * diff --git a/react/features/large-video/middleware.ts b/react/features/large-video/middleware.any.ts similarity index 98% rename from react/features/large-video/middleware.ts rename to react/features/large-video/middleware.any.ts index 3bff54732a1..24f6cd2289c 100644 --- a/react/features/large-video/middleware.ts +++ b/react/features/large-video/middleware.any.ts @@ -16,8 +16,6 @@ import { TOGGLE_DOCUMENT_EDITING } from '../etherpad/actionTypes'; import { selectParticipantInLargeVideo } from './actions'; import logger from './logger'; -import './subscriber'; - /** * Middleware that catches actions related to participants and tracks and * dispatches an action to select a participant depicted by LargeVideo. @@ -72,3 +70,4 @@ MiddlewareRegistry.register(store => next => action => { return result; }); + diff --git a/react/features/large-video/middleware.native.ts b/react/features/large-video/middleware.native.ts new file mode 100644 index 00000000000..71cdc16a427 --- /dev/null +++ b/react/features/large-video/middleware.native.ts @@ -0,0 +1,2 @@ +import './middleware.any'; +import './subscriber.native'; diff --git a/react/features/large-video/middleware.web.ts b/react/features/large-video/middleware.web.ts new file mode 100644 index 00000000000..03d2bbdafab --- /dev/null +++ b/react/features/large-video/middleware.web.ts @@ -0,0 +1,116 @@ +import { v4 as uuidv4 } from 'uuid'; + +import { CONFERENCE_JOINED } from '../base/conference/actionTypes'; +import { IJitsiConference } from '../base/conference/reducer'; +import { JitsiConferenceEvents } from '../base/lib-jitsi-meet'; +import { CAPTURE_SCREENSHOT_MESSAGE, SEND_SCREENSHOT_MESSAGE } from '../base/media/constants'; +import MiddlewareRegistry from '../base/redux/MiddlewareRegistry'; + +import { captureLargeVideoScreenshot } from './actions'; + +import './subscriber.web'; +import './middleware.any'; + +/** + * A {@code Map} temporary holding in transit screen capture chunks. + */ +const receivedScreencaptureMap = new Map(); + +/** + * The image chunk size in Bytes <=> 60 KB. + */ +const IMAGE_CHUNK_SIZE = 1024 * 60; + +/** + * Middleware that catches actions related to participants and tracks and + * dispatches an action to select a participant depicted by LargeVideo. + * + * @param {Store} store - Redux store. + * @returns {Function} + */ +MiddlewareRegistry.register(_store => next => action => { + switch (action.type) { + case CONFERENCE_JOINED: { + _addScreenCaptureListeners(action.conference); + break; + } + } + const result = next(action); + + return result; +}); + + +/** + * Registers listener for {@link JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED} that + * will perform various chat related activities. + * + * @param {IJitsiConference} conference - The conference. + * @returns {void} + */ +function _addScreenCaptureListeners(conference: IJitsiConference) { + conference.on( + JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED, + (...args: any) => { + if (args && args.length >= 2) { + const [ sender, eventData ] = args; + + if (eventData.name === CAPTURE_SCREENSHOT_MESSAGE) { + APP.store.dispatch(captureLargeVideoScreenshot()).then(dataURL => { + if (!dataURL) { + return; + } + + const uuId = uuidv4(); + const size = Math.ceil(dataURL.length / IMAGE_CHUNK_SIZE); + let currentIndex = 0; + let idx = 0; + + while (currentIndex < dataURL.length) { + const newIndex = currentIndex + IMAGE_CHUNK_SIZE; + + conference.sendEndpointMessage(sender._id, { + name: SEND_SCREENSHOT_MESSAGE, + id: uuId, + size, + idx, + chunk: dataURL.slice(currentIndex, newIndex) + }); + currentIndex = newIndex; + idx++; + } + }); + } + + if (eventData.name === SEND_SCREENSHOT_MESSAGE) { + if (eventData.id) { + if (!receivedScreencaptureMap.has(eventData.id)) { + receivedScreencaptureMap.set(eventData.id, new Array(eventData.size)); + } + + const arr = receivedScreencaptureMap.get(eventData.id); + const { id, idx, chunk, size } = eventData; + + arr[idx] = chunk; + if (idx === size - 1) { + const dataURL = arr.join(''); + + APP.API.notifyLargeVideoScreenshotReceived({ + jid: sender._jid, + id: sender._id + }, + dataURL); + receivedScreencaptureMap.delete(id); + } + } else { + APP.API.notifyLargeVideoScreenshotReceived({ + jid: sender._jid, + id: sender._id + }, + undefined); + } + } + } + } + ); +}