From d98ee1ecf67e16368f4ad285b686ab9874e809af Mon Sep 17 00:00:00 2001 From: Horatiu Muresan Date: Fri, 7 Jul 2023 14:13:47 +0300 Subject: [PATCH] feat(external-api) add commands/event for remote screen capture and facing mode - added command for screencapture remotely - this will trigger a response caught in an event handler - added command for setting the camera facing mode remotely - enhanced toggleVideo command to optionally accept the facing mode --- conference.js | 85 ++++++++++++++++++++++- modules/API/API.js | 53 +++++++++++--- modules/API/constants.js | 5 ++ modules/API/external/external_api.js | 5 +- react/features/base/media/constants.ts | 10 +++ react/features/base/tracks/actions.any.ts | 38 +++++++++- 6 files changed, 183 insertions(+), 13 deletions(-) diff --git a/conference.js b/conference.js index 950b3703ff90c..bbb2c650adb70 100644 --- a/conference.js +++ b/conference.js @@ -3,9 +3,10 @@ import { jitsiLocalStorage } from '@jitsi/js-utils'; import Logger from '@jitsi/logger'; import EventEmitter from 'events'; +import { v4 as uuidv4 } from 'uuid'; import { openConnection } from './connection'; -import { ENDPOINT_TEXT_MESSAGE_NAME } from './modules/API/constants'; +import { CAMERA_FACING_MODE_MESSAGE, ENDPOINT_TEXT_MESSAGE_NAME } from './modules/API/constants'; import { AUDIO_ONLY_SCREEN_SHARE_NO_TRACK } from './modules/UI/UIErrors'; import AuthHandler from './modules/UI/authentication/AuthHandler'; import UIUtil from './modules/UI/util/UIUtil'; @@ -95,7 +96,7 @@ import { setVideoMuted, setVideoUnmutePermissions } from './react/features/base/media/actions'; -import { MEDIA_TYPE } from './react/features/base/media/constants'; +import { CAPTURE_SCREENSHOT_MESSAGE, MEDIA_TYPE, SEND_SCREENSHOT_MESSAGE } from './react/features/base/media/constants'; import { getStartWithAudioMuted, getStartWithVideoMuted, @@ -129,6 +130,7 @@ import { trackAdded, trackRemoved } from './react/features/base/tracks/actions'; +import { setCameraFacingMode } from './react/features/base/tracks/actions.any'; import { createLocalTracksF, getLocalJitsiAudioTrack, @@ -143,6 +145,7 @@ import { showDesktopPicker } from './react/features/desktop-picker/actions'; import { appendSuffix } from './react/features/display-name/functions'; import { maybeOpenFeedbackDialog, submitFeedback } from './react/features/feedback/actions'; import { initKeyboardShortcuts } from './react/features/keyboard-shortcuts/actions'; +import { captureLargeVideoScreenshot } from './react/features/large-video/actions.web'; import { maybeSetLobbyChatMessageListener } from './react/features/lobby/actions.any'; import { setNoiseSuppressionEnabled } from './react/features/noise-suppression/actions'; import { hideNotification, showNotification, showWarningNotification } from './react/features/notifications/actions'; @@ -172,6 +175,13 @@ const logger = Logger.getLogger(__filename); const eventEmitter = new EventEmitter(); +const receivedScreencaptureMap = new Map(); + +/** + * The image chunk size in Bytes <=> 60 KB + */ +const IMAGE_CHUNK_SIZE = 1024 * 60; + let room; let connection; @@ -2024,6 +2034,77 @@ export default { eventData }); } + + if (eventData.name === CAPTURE_SCREENSHOT_MESSAGE) { + APP.store.dispatch(captureLargeVideoScreenshot()).then(dataURL => { + if (!dataURL) { + room.sendEndpointMessage(sender._id, { + name: SEND_SCREENSHOT_MESSAGE, + id: undefined + }); + + return; + } + + const splitImage = { + uuId: uuidv4(), + chunks: [] + }; + let currentIndex = 0; + + while (currentIndex < dataURL.length) { + const newIndex = currentIndex + IMAGE_CHUNK_SIZE; + + splitImage.chunks.push(dataURL.slice(currentIndex, newIndex)); + currentIndex = newIndex; + } + + const size = splitImage.chunks.length; + + for (const [ idx, chunk ] of splitImage.chunks.entries()) { + room.sendEndpointMessage(sender._id, { + name: SEND_SCREENSHOT_MESSAGE, + id: splitImage.uuId, + size, + idx, + chunk + }); + } + }); + } + + if (eventData.name === SEND_SCREENSHOT_MESSAGE) { + if (eventData.id) { + if (!receivedScreencaptureMap.has(eventData.id)) { + receivedScreencaptureMap.set(eventData.id, new Array(eventData.size)); + } + + const arr = receivedScreencaptureMap.get(eventData.id); + const { id, idx, chunk, size } = eventData; + + arr[idx] = chunk; + if (idx === size - 1) { + const dataURL = arr.join(''); + + APP.API.notifyLargeVideoScreenshotReceived({ + jid: sender._jid, + id: sender._id + }, + dataURL); + receivedScreencaptureMap.delete(id); + } + } else { + APP.API.notifyLargeVideoScreenshotReceived({ + jid: sender._jid, + id: sender._id + }, + undefined); + } + } + + if (eventData.name === CAMERA_FACING_MODE_MESSAGE) { + APP.store.dispatch(setCameraFacingMode(eventData.facingMode)); + } } }); diff --git a/modules/API/API.js b/modules/API/API.js index cd328c8ac0637..e53f5ca51a030 100644 --- a/modules/API/API.js +++ b/modules/API/API.js @@ -30,7 +30,7 @@ import { toggleDialog } from '../../react/features/base/dialog/actions'; import { isSupportedBrowser } from '../../react/features/base/environment/environment'; import { parseJWTFromURLParams } from '../../react/features/base/jwt/functions'; import JitsiMeetJS, { JitsiRecordingConstants } from '../../react/features/base/lib-jitsi-meet'; -import { MEDIA_TYPE, VIDEO_TYPE } from '../../react/features/base/media/constants'; +import { CAPTURE_SCREENSHOT_MESSAGE, MEDIA_TYPE, VIDEO_TYPE } from '../../react/features/base/media/constants'; import { grantModerator, kickParticipant, @@ -50,8 +50,7 @@ import { } from '../../react/features/base/participants/functions'; import { updateSettings } from '../../react/features/base/settings/actions'; import { getDisplayName } from '../../react/features/base/settings/functions.web'; -import { toggleCamera } from '../../react/features/base/tracks/actions.any'; -import { isToggleCameraEnabled } from '../../react/features/base/tracks/functions'; +import { setCameraFacingMode } from '../../react/features/base/tracks/actions.any'; import { autoAssignToBreakoutRooms, closeBreakoutRoom, @@ -118,6 +117,7 @@ import { getJitsiMeetTransport } from '../transport'; import { API_ID, + CAMERA_FACING_MODE_MESSAGE, ENDPOINT_TEXT_MESSAGE_NAME } from './constants'; @@ -395,12 +395,8 @@ function initCommands() { sendAnalytics(createApiEvent('film.strip.resize')); APP.store.dispatch(resizeFilmStrip(options.width)); }, - 'toggle-camera': () => { - if (!isToggleCameraEnabled(APP.store.getState())) { - return; - } - - APP.store.dispatch(toggleCamera()); + 'toggle-camera': facingMode => { + APP.store.dispatch(setCameraFacingMode(facingMode)); }, 'toggle-camera-mirror': () => { const state = APP.store.getState(); @@ -529,6 +525,29 @@ function initCommands() { logger.error('Failed sending endpoint text message', err); } }, + 'send-capture-screenshot-message': to => { + if (!to || !isLocalParticipantModerator(APP.store.getState())) { + logger.warn('Participant id not set or participant not moderator'); + + return; + } + + APP.conference.sendEndpointMessage(to, { + name: CAPTURE_SCREENSHOT_MESSAGE + }); + }, + 'send-camera-facing-mode-message': (to, facingMode) => { + if (!to || !isLocalParticipantModerator(APP.store.getState())) { + logger.warn('Participant id not set or participant not moderator'); + + return; + } + + APP.conference.sendEndpointMessage(to, { + name: CAMERA_FACING_MODE_MESSAGE, + facingMode + }); + }, 'overwrite-names': participantList => { logger.debug('Overwrite names command received'); @@ -1775,6 +1794,22 @@ class API { }); } + /** + * Notify external application (if API is enabled) that a participant's screen + * capture has been received. + * + * @param {Object} senderInfo - The sender's info. + * @param {string} dataURL - The image data. + * @returns {void} + */ + notifyLargeVideoScreenshotReceived(senderInfo, dataURL) { + this._sendEvent({ + name: 'screen-capture-received', + senderInfo, + dataURL + }); + } + /** * Notify external application (if API is enabled) that the dominant speaker * has been turned on/off. diff --git a/modules/API/constants.js b/modules/API/constants.js index 6330c723b689a..e77bead2df881 100644 --- a/modules/API/constants.js +++ b/modules/API/constants.js @@ -22,3 +22,8 @@ export const ENDPOINT_TEXT_MESSAGE_NAME = 'endpoint-text-message'; * but rather allowing the estimations to take place. */ export const MIN_ASSUMED_BANDWIDTH_BPS = -1; + +/** + * The payload name for remotely setting the camera facing mode message. + */ +export const CAMERA_FACING_MODE_MESSAGE = 'camera-facing-mode-message'; diff --git a/modules/API/external/external_api.js b/modules/API/external/external_api.js index ef2c5d69b5af7..acb6b761f16cf 100644 --- a/modules/API/external/external_api.js +++ b/modules/API/external/external_api.js @@ -54,6 +54,8 @@ const commands = { removeBreakoutRoom: 'remove-breakout-room', resizeFilmStrip: 'resize-film-strip', resizeLargeVideo: 'resize-large-video', + sendCameraFacingMode: 'send-camera-facing-mode-message', + sendCaptureScreenshot: 'send-capture-screenshot-message', sendChatMessage: 'send-chat-message', sendEndpointTextMessage: 'send-endpoint-text-message', sendParticipantToRoom: 'send-participant-to-room', @@ -111,6 +113,7 @@ const events = { 'data-channel-opened': 'dataChannelOpened', 'device-list-changed': 'deviceListChanged', 'display-name-change': 'displayNameChange', + 'dominant-speaker-changed': 'dominantSpeakerChanged', 'email-change': 'emailChange', 'error-occurred': 'errorOccurred', 'endpoint-text-message-received': 'endpointTextMessageReceived', @@ -151,8 +154,8 @@ const events = { 'video-availability-changed': 'videoAvailabilityChanged', 'video-mute-status-changed': 'videoMuteStatusChanged', 'video-quality-changed': 'videoQualityChanged', + 'screen-capture-received': 'screenCaptureReceived', 'screen-sharing-status-changed': 'screenSharingStatusChanged', - 'dominant-speaker-changed': 'dominantSpeakerChanged', 'subject-change': 'subjectChange', 'suspend-detected': 'suspendDetected', 'tile-view-changed': 'tileViewChanged', diff --git a/react/features/base/media/constants.ts b/react/features/base/media/constants.ts index 90497a2f4592a..10977d0e1d750 100644 --- a/react/features/base/media/constants.ts +++ b/react/features/base/media/constants.ts @@ -64,3 +64,13 @@ export const VIDEO_TYPE: { [key: string]: VideoType; } = { }; export type VideoType = 'camera' | 'desktop'; + +/** + * The payload name for the capture screenshot remotely message. + */ +export const CAPTURE_SCREENSHOT_MESSAGE = 'capture-screenshot-message'; + +/** + * The payload name for the received screenshot message. + */ +export const SEND_SCREENSHOT_MESSAGE = 'send-screenshot-message'; diff --git a/react/features/base/tracks/actions.any.ts b/react/features/base/tracks/actions.any.ts index 7a9c7679e7402..6b6c349ff188d 100644 --- a/react/features/base/tracks/actions.any.ts +++ b/react/features/base/tracks/actions.any.ts @@ -38,7 +38,8 @@ import { getLocalTrack, getLocalTracks, getLocalVideoTrack, - getTrackByJitsiTrack + getTrackByJitsiTrack, + isToggleCameraEnabled } from './functions'; import logger from './logger'; import { ITrackOptions } from './types'; @@ -864,3 +865,38 @@ export function toggleCamera() { await APP.conference.useVideoStream(newVideoTrack); }; } + +/** + * Sets the camera facing mode(environment/user). If facing mode not provided, it will do a toggle. + * + * @param {string | undefined} facingMode - The selected facing mode. + * @returns {void} + */ +export function setCameraFacingMode(facingMode: string | undefined) { + return async (dispatch: IStore['dispatch'], getState: IStore['getState']) => { + const state = getState(); + + if (!isToggleCameraEnabled(state)) { + return; + } + + if (!facingMode) { + dispatch(toggleCamera()); + + return; + } + + const tracks = state['features/base/tracks']; + const localVideoTrack = getLocalVideoTrack(tracks)?.jitsiTrack; + + if (!tracks || !localVideoTrack) { + return; + } + + const currentFacingMode = localVideoTrack.getCameraFacingMode(); + + if (currentFacingMode !== facingMode) { + dispatch(toggleCamera()); + } + }; +}