feat(external-api) add commands/event for remote screen capture and f…

…acing mode

- added command for screencapture remotely - this will trigger a response caught in an event handler
- added command for setting the camera facing mode remotely
- enhanced toggleVideo command to optionally accept the facing mode

conference.js

@@ -3,9 +3,10 @@
 import { jitsiLocalStorage } from '@jitsi/js-utils';
 import Logger from '@jitsi/logger';
 import EventEmitter from 'events';
+import { v4 as uuidv4 } from 'uuid';
 
 import { openConnection } from './connection';
-import { ENDPOINT_TEXT_MESSAGE_NAME } from './modules/API/constants';
+import { CAMERA_FACING_MODE_MESSAGE, ENDPOINT_TEXT_MESSAGE_NAME } from './modules/API/constants';
 import { AUDIO_ONLY_SCREEN_SHARE_NO_TRACK } from './modules/UI/UIErrors';
 import AuthHandler from './modules/UI/authentication/AuthHandler';
 import UIUtil from './modules/UI/util/UIUtil';
@@ -95,7 +96,7 @@ import {
     setVideoMuted,
     setVideoUnmutePermissions
 } from './react/features/base/media/actions';
-import { MEDIA_TYPE } from './react/features/base/media/constants';
+import { CAPTURE_SCREENSHOT_MESSAGE, MEDIA_TYPE, SEND_SCREENSHOT_MESSAGE } from './react/features/base/media/constants';
 import {
     getStartWithAudioMuted,
     getStartWithVideoMuted,
@@ -129,6 +130,7 @@ import {
     trackAdded,
     trackRemoved
 } from './react/features/base/tracks/actions';
+import { setCameraFacingMode } from './react/features/base/tracks/actions.any';
 import {
     createLocalTracksF,
     getLocalJitsiAudioTrack,
@@ -143,6 +145,7 @@ import { showDesktopPicker } from './react/features/desktop-picker/actions';
 import { appendSuffix } from './react/features/display-name/functions';
 import { maybeOpenFeedbackDialog, submitFeedback } from './react/features/feedback/actions';
 import { initKeyboardShortcuts } from './react/features/keyboard-shortcuts/actions';
+import { captureLargeVideoScreenshot } from './react/features/large-video/actions.web';
 import { maybeSetLobbyChatMessageListener } from './react/features/lobby/actions.any';
 import { setNoiseSuppressionEnabled } from './react/features/noise-suppression/actions';
 import { hideNotification, showNotification, showWarningNotification } from './react/features/notifications/actions';
@@ -172,6 +175,13 @@ const logger = Logger.getLogger(__filename);
 
 const eventEmitter = new EventEmitter();
 
+const receivedScreencaptureMap = new Map();
+
+/**
+ * The image chunk size in Bytes <=> 60 KB
+ */
+const IMAGE_CHUNK_SIZE = 1024 * 60;
+
 let room;
 let connection;
 
@@ -2024,6 +2034,77 @@ export default {
                             eventData
                         });
                     }
+
+                    if (eventData.name === CAPTURE_SCREENSHOT_MESSAGE) {
+                        APP.store.dispatch(captureLargeVideoScreenshot()).then(dataURL => {
+                            if (!dataURL) {
+                                room.sendEndpointMessage(sender._id, {
+                                    name: SEND_SCREENSHOT_MESSAGE,
+                                    id: undefined
+                                });
+
+                                return;
+                            }
+
+                            const splitImage = {
+                                uuId: uuidv4(),
+                                chunks: []
+                            };
+                            let currentIndex = 0;
+
+                            while (currentIndex < dataURL.length) {
+                                const newIndex = currentIndex + IMAGE_CHUNK_SIZE;
+
+                                splitImage.chunks.push(dataURL.slice(currentIndex, newIndex));
+                                currentIndex = newIndex;
+                            }
+
+                            const size = splitImage.chunks.length;
+
+                            for (const [ idx, chunk ] of splitImage.chunks.entries()) {
+                                room.sendEndpointMessage(sender._id, {
+                                    name: SEND_SCREENSHOT_MESSAGE,
+                                    id: splitImage.uuId,
+                                    size,
+                                    idx,
+                                    chunk
+                                });
+                            }
+                        });
+                    }
+
+                    if (eventData.name === SEND_SCREENSHOT_MESSAGE) {
+                        if (eventData.id) {
+                            if (!receivedScreencaptureMap.has(eventData.id)) {
+                                receivedScreencaptureMap.set(eventData.id, new Array(eventData.size));
+                            }
+
+                            const arr = receivedScreencaptureMap.get(eventData.id);
+                            const { id, idx, chunk, size } = eventData;
+
+                            arr[idx] = chunk;
+                            if (idx === size - 1) {
+                                const dataURL = arr.join('');
+
+                                APP.API.notifyLargeVideoScreenshotReceived({
+                                    jid: sender._jid,
+                                    id: sender._id
+                                },
+                                dataURL);
+                                receivedScreencaptureMap.delete(id);
+                            }
+                        } else {
+                            APP.API.notifyLargeVideoScreenshotReceived({
+                                jid: sender._jid,
+                                id: sender._id
+                            },
+                            undefined);
+                        }
+                    }
+
+                    if (eventData.name === CAMERA_FACING_MODE_MESSAGE) {
+                        APP.store.dispatch(setCameraFacingMode(eventData.facingMode));
+                    }
                 }
             });
 

modules/API/API.js

@@ -30,7 +30,7 @@ import { toggleDialog } from '../../react/features/base/dialog/actions';
 import { isSupportedBrowser } from '../../react/features/base/environment/environment';
 import { parseJWTFromURLParams } from '../../react/features/base/jwt/functions';
 import JitsiMeetJS, { JitsiRecordingConstants } from '../../react/features/base/lib-jitsi-meet';
-import { MEDIA_TYPE, VIDEO_TYPE } from '../../react/features/base/media/constants';
+import { CAPTURE_SCREENSHOT_MESSAGE, MEDIA_TYPE, VIDEO_TYPE } from '../../react/features/base/media/constants';
 import {
     grantModerator,
     kickParticipant,
@@ -50,8 +50,7 @@ import {
 } from '../../react/features/base/participants/functions';
 import { updateSettings } from '../../react/features/base/settings/actions';
 import { getDisplayName } from '../../react/features/base/settings/functions.web';
-import { toggleCamera } from '../../react/features/base/tracks/actions.any';
-import { isToggleCameraEnabled } from '../../react/features/base/tracks/functions';
+import { setCameraFacingMode } from '../../react/features/base/tracks/actions.any';
 import {
     autoAssignToBreakoutRooms,
     closeBreakoutRoom,
@@ -118,6 +117,7 @@ import { getJitsiMeetTransport } from '../transport';
 
 import {
     API_ID,
+    CAMERA_FACING_MODE_MESSAGE,
     ENDPOINT_TEXT_MESSAGE_NAME
 } from './constants';
 
@@ -395,12 +395,8 @@ function initCommands() {
             sendAnalytics(createApiEvent('film.strip.resize'));
             APP.store.dispatch(resizeFilmStrip(options.width));
         },
-        'toggle-camera': () => {
-            if (!isToggleCameraEnabled(APP.store.getState())) {
-                return;
-            }
-
-            APP.store.dispatch(toggleCamera());
+        'toggle-camera': facingMode => {
+            APP.store.dispatch(setCameraFacingMode(facingMode));
         },
         'toggle-camera-mirror': () => {
             const state = APP.store.getState();
@@ -529,6 +525,29 @@ function initCommands() {
                 logger.error('Failed sending endpoint text message', err);
             }
         },
+        'send-capture-screenshot-message': to => {
+            if (!to || !isLocalParticipantModerator(APP.store.getState())) {
+                logger.warn('Participant id not set or participant not moderator');
+
+                return;
+            }
+
+            APP.conference.sendEndpointMessage(to, {
+                name: CAPTURE_SCREENSHOT_MESSAGE
+            });
+        },
+        'send-camera-facing-mode-message': (to, facingMode) => {
+            if (!to || !isLocalParticipantModerator(APP.store.getState())) {
+                logger.warn('Participant id not set or participant not moderator');
+
+                return;
+            }
+
+            APP.conference.sendEndpointMessage(to, {
+                name: CAMERA_FACING_MODE_MESSAGE,
+                facingMode
+            });
+        },
         'overwrite-names': participantList => {
             logger.debug('Overwrite names command received');
 
@@ -1775,6 +1794,22 @@ class API {
         });
     }
 
+    /**
+     * Notify external application (if API is enabled) that a participant's screen
+     * capture has been received.
+     *
+     * @param {Object} senderInfo - The sender's info.
+     * @param {string} dataURL - The image data.
+     * @returns {void}
+     */
+    notifyLargeVideoScreenshotReceived(senderInfo, dataURL) {
+        this._sendEvent({
+            name: 'screen-capture-received',
+            senderInfo,
+            dataURL
+        });
+    }
+
     /**
      * Notify external application (if API is enabled) that the dominant speaker
      * has been turned on/off.

modules/API/constants.js

@@ -22,3 +22,8 @@ export const ENDPOINT_TEXT_MESSAGE_NAME = 'endpoint-text-message';
  * but rather allowing the estimations to take place.
  */
 export const MIN_ASSUMED_BANDWIDTH_BPS = -1;
+
+/**
+ * The payload name for remotely setting the camera facing mode message.
+ */
+export const CAMERA_FACING_MODE_MESSAGE = 'camera-facing-mode-message';

modules/API/external/external_api.js

@@ -54,6 +54,8 @@ const commands = {
     removeBreakoutRoom: 'remove-breakout-room',
     resizeFilmStrip: 'resize-film-strip',
     resizeLargeVideo: 'resize-large-video',
+    sendCameraFacingMode: 'send-camera-facing-mode-message',
+    sendCaptureScreenshot: 'send-capture-screenshot-message',
     sendChatMessage: 'send-chat-message',
     sendEndpointTextMessage: 'send-endpoint-text-message',
     sendParticipantToRoom: 'send-participant-to-room',
@@ -111,6 +113,7 @@ const events = {
     'data-channel-opened': 'dataChannelOpened',
     'device-list-changed': 'deviceListChanged',
     'display-name-change': 'displayNameChange',
+    'dominant-speaker-changed': 'dominantSpeakerChanged',
     'email-change': 'emailChange',
     'error-occurred': 'errorOccurred',
     'endpoint-text-message-received': 'endpointTextMessageReceived',
@@ -151,8 +154,8 @@ const events = {
     'video-availability-changed': 'videoAvailabilityChanged',
     'video-mute-status-changed': 'videoMuteStatusChanged',
     'video-quality-changed': 'videoQualityChanged',
+    'screen-capture-received': 'screenCaptureReceived',
     'screen-sharing-status-changed': 'screenSharingStatusChanged',
-    'dominant-speaker-changed': 'dominantSpeakerChanged',
     'subject-change': 'subjectChange',
     'suspend-detected': 'suspendDetected',
     'tile-view-changed': 'tileViewChanged',

react/features/base/media/constants.ts

@@ -64,3 +64,13 @@ export const VIDEO_TYPE: { [key: string]: VideoType; } = {
 };
 
 export type VideoType = 'camera' | 'desktop';
+
+/**
+ * The payload name for the capture screenshot remotely message.
+ */
+export const CAPTURE_SCREENSHOT_MESSAGE = 'capture-screenshot-message';
+
+/**
+ * The payload name for the received screenshot message.
+ */
+export const SEND_SCREENSHOT_MESSAGE = 'send-screenshot-message';

react/features/base/tracks/actions.any.ts

@@ -38,7 +38,8 @@ import {
     getLocalTrack,
     getLocalTracks,
     getLocalVideoTrack,
-    getTrackByJitsiTrack
+    getTrackByJitsiTrack,
+    isToggleCameraEnabled
 } from './functions';
 import logger from './logger';
 import { ITrackOptions } from './types';
@@ -864,3 +865,38 @@ export function toggleCamera() {
         await APP.conference.useVideoStream(newVideoTrack);
     };
 }
+
+/**
+ * Sets the camera facing mode(environment/user). If facing mode not provided, it will do a toggle.
+ *
+ * @param {string | undefined} facingMode - The selected facing mode.
+ * @returns {void}
+ */
+export function setCameraFacingMode(facingMode: string | undefined) {
+    return async (dispatch: IStore['dispatch'], getState: IStore['getState']) => {
+        const state = getState();
+
+        if (!isToggleCameraEnabled(state)) {
+            return;
+        }
+
+        if (!facingMode) {
+            dispatch(toggleCamera());
+
+            return;
+        }
+
+        const tracks = state['features/base/tracks'];
+        const localVideoTrack = getLocalVideoTrack(tracks)?.jitsiTrack;
+
+        if (!tracks || !localVideoTrack) {
+            return;
+        }
+
+        const currentFacingMode = localVideoTrack.getCameraFacingMode();
+
+        if (currentFacingMode !== facingMode) {
+            dispatch(toggleCamera());
+        }
+    };
+}