Together mode client state and subscriber implementation (#5152)

* Together mode client state and subscriber implementation --------- Co-authored-by: Donald McEachern <[email protected]>
Azure · Oct 3, 2024 · 3175367 · 3175367
1 parent bb39ff5
commit 3175367
Show file tree

Hide file tree

Showing 19 changed files with 247 additions and 4 deletions.
diff --git a/change-beta/@azure-communication-react-367d1747-182e-425c-a897-cbcdfbc06ecd.json b/change-beta/@azure-communication-react-367d1747-182e-425c-a897-cbcdfbc06ecd.json
@@ -0,0 +1,9 @@
+{
+  "type": "prerelease",
+  "area": "feature",
+  "workstream": "Together Mode",
+  "comment": "Together mode client state and subscriber implementation",
+  "packageName": "@azure/communication-react",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}
diff --git a/change-beta/@azure-communication-react-43b808a4-0c26-4119-a26b-7d31efbb2b93.json b/change-beta/@azure-communication-react-43b808a4-0c26-4119-a26b-7d31efbb2b93.json
@@ -0,0 +1,9 @@
+{
+  "type": "prerelease",
+  "area": "feature",
+  "workstream": "Together Mode",
+  "comment": "Together mode client state and subscriber implementation",
+  "packageName": "@azure/communication-react",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}
diff --git a/change-beta/@azure-communication-react-839f12bb-f1d6-43dc-9ba8-41fc6b17baaf.json b/change-beta/@azure-communication-react-839f12bb-f1d6-43dc-9ba8-41fc6b17baaf.json
@@ -0,0 +1,9 @@
+{
+  "type": "prerelease",
+  "area": "feature",
+  "workstream": "Together Mode",
+  "comment": "Together mode client state and subscriber implementation",
+  "packageName": "@azure/communication-react",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}
diff --git a/change-beta/@azure-communication-react-a9fb84d9-5a0a-43a1-b407-decf5c80312c.json b/change-beta/@azure-communication-react-a9fb84d9-5a0a-43a1-b407-decf5c80312c.json
@@ -0,0 +1,9 @@
+{
+  "type": "prerelease",
+  "area": "feature",
+  "workstream": "Together Mode",
+  "comment": "Together mode client state and subscriber implementation",
+  "packageName": "@azure/communication-react",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}
diff --git a/change-beta/@azure-communication-react-c5b508a6-370f-414d-9663-905d1ef3f69b.json b/change-beta/@azure-communication-react-c5b508a6-370f-414d-9663-905d1ef3f69b.json
@@ -0,0 +1,9 @@
+{
+  "type": "prerelease",
+  "area": "feature",
+  "workstream": "Together Mode",
+  "comment": "This PR contains implementation of together mode client state changes and the event listener for together mode stream updates",
+  "packageName": "@azure/communication-react",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}
diff --git a/change-beta/@azure-communication-react-ebad70bc-6036-4b67-b5f9-438a20702e0e.json b/change-beta/@azure-communication-react-ebad70bc-6036-4b67-b5f9-438a20702e0e.json
@@ -0,0 +1,9 @@
+{
+  "type": "prerelease",
+  "area": "feature",
+  "workstream": "TogetherMode",
+  "comment": "Together mode client state and subscriber implementation",
+  "packageName": "@azure/communication-react",
+  "email": "[email protected]",
+  "dependentChangeType": "patch"
+}
diff --git a/packages/calling-stateful-client/src/CallClientState.ts b/packages/calling-stateful-client/src/CallClientState.ts
@@ -274,6 +274,49 @@ export interface RaiseHandCallFeatureState {
   localParticipantRaisedHand?: RaisedHandState;
 }
 
+/* @conditional-compile-remove(together-mode) */
+/**
+ * State only version of {@link @azure/communication-calling#TogetherModeCallFeature}. {@link StatefulCallClient} will
+ * automatically listen for raised hands on the call and update the state exposed by {@link StatefulCallClient} accordingly.
+ * @alpha
+ */
+export interface TogetherModeCallFeatureState {
+  /**
+   * Proxy of {@link @azure/communication-calling#TogetherModeCallFeature.togetherModeStream}.
+   */
+  stream: TogetherModeStreamState[];
+}
+
+/* @conditional-compile-remove(together-mode) */
+/**
+ * State only version of {@link @azure/communication-calling#TogetherModeVideoStream}.
+ * @alpha
+ */
+export interface TogetherModeStreamState {
+  /**
+   * Proxy of {@link @azure/communication-calling#TogetherModeVideoStream.id}.
+   */
+  id: number;
+  /**
+   * Proxy of {@link @azure/communication-calling#TogetherModeVideoStream.mediaStreamType}.
+   */
+  mediaStreamType: MediaStreamType;
+  /**
+   * Proxy of {@link @azure/communication-calling#TogetherModeVideoStream.isReceiving}.
+   * @public
+   */
+  isReceiving: boolean;
+  /**
+   * {@link VideoStreamRendererView} that is managed by createView/disposeView in {@link StatefulCallClient}
+   * API. This can be undefined if the stream has not yet been rendered and defined after createView creates the view.
+   */
+  view?: VideoStreamRendererViewState;
+  /**
+   * Proxy of {@link @azure/communication-calling#RemoteVideoStream.size}.
+   */
+  streamSize?: { width: number; height: number };
+}
+
 /**
  * State only version of {@link @azure/communication-calling#PPTLiveCallFeature}. {@link StatefulCallClient} will
  * automatically listen for pptLive on the call and update the state exposed by {@link StatefulCallClient} accordingly.
@@ -579,6 +622,11 @@ export interface CallState {
    * Proxy of {@link @azure/communication-calling#RaiseHandCallFeature}.
    */
   raiseHand: RaiseHandCallFeatureState;
+  /* @conditional-compile-remove(together-mode) */
+  /**
+   * Proxy of {@link @azure/communication-calling#TogetherModeCallFeature}.
+   */
+  togetherMode: TogetherModeCallFeatureState;
   /**
    * Proxy of {@link @azure/communication-calling#Call.ReactionMessage} with
    * UI helper props receivedOn which indicates the timestamp when the message was received.

diff --git a/packages/calling-stateful-client/src/CallContext.ts b/packages/calling-stateful-client/src/CallContext.ts
@@ -23,6 +23,8 @@ import { TeamsCaptionsInfo } from '@azure/communication-calling';
 import { CaptionsKind, CaptionsInfo as AcsCaptionsInfo } from '@azure/communication-calling';
 /* @conditional-compile-remove(unsupported-browser) */
 import { EnvironmentInfo } from '@azure/communication-calling';
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeVideoStream } from '@azure/communication-calling';
 import { AzureLogger, createClientLogger, getLogLevel } from '@azure/logger';
 import { EventEmitter } from 'events';
 import { enableMapSet, enablePatches, Patch, produce } from 'immer';
@@ -451,6 +453,31 @@ export class CallContext {
     });
   }
 
+  /* @conditional-compile-remove(together-mode) */
+  public setTogetherModeVideoStream(callId: string, addedStream: TogetherModeVideoStream[]): void {
+    this.modifyState((draft: CallClientState) => {
+      const call = draft.calls[this._callIdHistory.latestCallId(callId)];
+      if (call) {
+        call.togetherMode = { stream: addedStream };
+      }
+    });
+  }
+
+  /* @conditional-compile-remove(together-mode) */
+  public removeTogetherModeVideoStream(callId: string, removedStream: TogetherModeVideoStream[]): void {
+    this.modifyState((draft: CallClientState) => {
+      const call = draft.calls[this._callIdHistory.latestCallId(callId)];
+      if (call) {
+        for (const stream of removedStream) {
+          if (stream.mediaStreamType in call.togetherMode.stream) {
+            // Temporary lint fix: Remove the stream from the list
+            call.togetherMode.stream = [];
+          }
+        }
+      }
+    });
+  }
+
   public setCallRaisedHands(callId: string, raisedHands: RaisedHand[]): void {
     this.modifyState((draft: CallClientState) => {
       const call = draft.calls[this._callIdHistory.latestCallId(callId)];

diff --git a/packages/calling-stateful-client/src/CallSubscriber.ts b/packages/calling-stateful-client/src/CallSubscriber.ts
@@ -30,6 +30,8 @@ import { SpotlightSubscriber } from './SpotlightSubscriber';
 import { LocalRecordingSubscriber } from './LocalRecordingSubscriber';
 /* @conditional-compile-remove(breakout-rooms) */
 import { BreakoutRoomsSubscriber } from './BreakoutRoomsSubscriber';
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeSubscriber } from './TogetherModeSubscriber';
 
 /**
  * Keeps track of the listeners assigned to a particular call because when we get an event from SDK, it doesn't tell us
@@ -60,6 +62,8 @@ export class CallSubscriber {
   private _spotlightSubscriber: SpotlightSubscriber;
   /* @conditional-compile-remove(breakout-rooms) */
   private _breakoutRoomsSubscriber: BreakoutRoomsSubscriber;
+  /* @conditional-compile-remove(together-mode) */
+  private _togetherModeSubscriber: TogetherModeSubscriber;
 
   constructor(call: CallCommon, context: CallContext, internalContext: InternalCallContext) {
     this._call = call;
@@ -119,6 +123,12 @@ export class CallSubscriber {
       this._context,
       this._call.feature(Features.BreakoutRooms)
     );
+    /* @conditional-compile-remove(together-mode) */
+    this._togetherModeSubscriber = new TogetherModeSubscriber(
+      this._callIdRef,
+      this._context,
+      this._call.feature(Features.TogetherMode)
+    );
 
     this.subscribe();
   }
@@ -228,6 +238,8 @@ export class CallSubscriber {
     this._spotlightSubscriber.unsubscribe();
     /* @conditional-compile-remove(breakout-rooms) */
     this._breakoutRoomsSubscriber.unsubscribe();
+    /* @conditional-compile-remove(together-mode) */
+    this._togetherModeSubscriber.unsubscribe();
   };
 
   // This is a helper function to safely call subscriber functions. This is needed in order to prevent events

diff --git a/packages/calling-stateful-client/src/Converter.ts b/packages/calling-stateful-client/src/Converter.ts
@@ -156,6 +156,8 @@ export function convertSdkCallToDeclarativeCall(call: CallCommon): CallState {
     localRecording: { isLocalRecordingActive: false },
     pptLive: { isActive: false },
     raiseHand: { raisedHands: [] },
+    /* @conditional-compile-remove(together-mode) */
+    togetherMode: { stream: [] },
     localParticipantReaction: undefined,
     transcription: { isTranscriptionActive: false },
     screenShareRemoteParticipant: undefined,

diff --git a/packages/calling-stateful-client/src/StatefulCallClient.ts b/packages/calling-stateful-client/src/StatefulCallClient.ts
@@ -6,6 +6,8 @@ import { CallClient, CallClientOptions, CreateViewOptions, DeviceManager } from
 /* @conditional-compile-remove(unsupported-browser) */
 import { Features } from '@azure/communication-calling';
 import { CallClientState, LocalVideoStreamState, RemoteVideoStreamState } from './CallClientState';
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeStreamState } from './CallClientState';
 import { CallContext } from './CallContext';
 import { callAgentDeclaratify, DeclarativeCallAgent } from './CallAgentDeclarative';
 import { InternalCallContext } from './InternalCallContext';
@@ -111,7 +113,10 @@ export interface StatefulCallClient extends CallClient {
   createView(
     callId: string | undefined,
     participantId: CommunicationIdentifier | undefined,
-    stream: LocalVideoStreamState | RemoteVideoStreamState,
+    stream:
+      | LocalVideoStreamState
+      | RemoteVideoStreamState
+      | /* @conditional-compile-remove(together-mode) */ TogetherModeStreamState,
     options?: CreateViewOptions
   ): Promise<CreateViewResult | undefined>;
   /**

diff --git a/packages/calling-stateful-client/src/StreamUtils.test.ts b/packages/calling-stateful-client/src/StreamUtils.test.ts
@@ -89,6 +89,8 @@ function createMockCall(mockCallId: string): CallState {
     /* @conditional-compile-remove(local-recording-notification) */
     localRecording: { isLocalRecordingActive: false },
     raiseHand: { raisedHands: [] },
+    /* @conditional-compile-remove(together-mode) */
+    togetherMode: { stream: [] },
     localParticipantReaction: undefined,
     transcription: { isTranscriptionActive: false },
     screenShareRemoteParticipant: undefined,

diff --git a/packages/calling-stateful-client/src/StreamUtils.ts b/packages/calling-stateful-client/src/StreamUtils.ts
@@ -10,6 +10,8 @@ import {
 } from '@azure/communication-calling';
 import { CommunicationIdentifierKind } from '@azure/communication-common';
 import { LocalVideoStreamState, RemoteVideoStreamState } from './CallClientState';
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeStreamState } from './CallClientState';
 import { CallContext } from './CallContext';
 import {
   convertSdkLocalStreamToDeclarativeLocalStream,
@@ -35,7 +37,10 @@ async function createViewVideo(
   context: CallContext,
   internalContext: InternalCallContext,
   callId: string,
-  stream?: RemoteVideoStreamState | LocalVideoStreamState,
+  stream?:
+    | RemoteVideoStreamState
+    | LocalVideoStreamState
+    | /* @conditional-compile-remove(together-mode) */ TogetherModeStreamState,
   participantId?: CommunicationIdentifierKind | string,
   options?: CreateViewOptions
 ): Promise<CreateViewResult | undefined> {
@@ -488,7 +493,10 @@ export function createView(
   internalContext: InternalCallContext,
   callId: string | undefined,
   participantId: CommunicationIdentifierKind | string | undefined,
-  stream: LocalVideoStreamState | RemoteVideoStreamState,
+  stream:
+    | LocalVideoStreamState
+    | RemoteVideoStreamState
+    | /* @conditional-compile-remove(together-mode) */ TogetherModeStreamState,
   options?: CreateViewOptions
 ): Promise<CreateViewResult | undefined> {
   const streamType = stream.mediaStreamType;

diff --git a/packages/calling-stateful-client/src/TogetherModeSubscriber.ts b/packages/calling-stateful-client/src/TogetherModeSubscriber.ts
@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/* @conditional-compile-remove(together-mode) */
+import { TogetherModeCallFeature, TogetherModeVideoStream } from '@azure/communication-calling';
+/* @conditional-compile-remove(together-mode) */
+import { CallContext } from './CallContext';
+/* @conditional-compile-remove(together-mode) */
+import { CallIdRef } from './CallIdRef';
+/**
+ * @private
+ */
+
+/* @conditional-compile-remove(together-mode) */
+/**
+ * TogetherModeSubscriber is responsible for subscribing to together mode events and updating the call context accordingly.
+ */
+export class TogetherModeSubscriber {
+  private _callIdRef: CallIdRef;
+  private _context: CallContext;
+  private _togetherMode: TogetherModeCallFeature;
+
+  constructor(callIdRef: CallIdRef, context: CallContext, togetherMode: TogetherModeCallFeature) {
+    this._callIdRef = callIdRef;
+    this._context = context;
+    this._togetherMode = togetherMode;
+
+    this.subscribe();
+  }
+
+  private subscribe = (): void => {
+    this._togetherMode.on('togetherModeStreamsUpdated', this.onTogetherModeStreamUpdated);
+  };
+
+  public unsubscribe = (): void => {
+    this._togetherMode.off('togetherModeStreamsUpdated', this.onTogetherModeStreamUpdated);
+  };
+
+  private onTogetherModeStreamUpdated = (args: {
+    added: TogetherModeVideoStream[];
+    removed: TogetherModeVideoStream[];
+  }): void => {
+    if (args.added) {
+      this._context.setTogetherModeVideoStream(this._callIdRef.callId, args.added);
+    }
+    if (args.removed) {
+      this._context.removeTogetherModeVideoStream(this._callIdRef.callId, args.removed);
+    }
+  };
+}
diff --git a/packages/calling-stateful-client/src/index-public.ts b/packages/calling-stateful-client/src/index-public.ts
@@ -31,6 +31,11 @@ export type { TeamsIncomingCallState } from './CallClientState';
 export type { RemoteDiagnosticState } from './CallClientState';
 export type { CreateViewResult } from './StreamUtils';
 export type { RaiseHandCallFeatureState as RaiseHandCallFeature } from './CallClientState';
+/* @conditional-compile-remove(together-mode) */
+export type { TogetherModeCallFeatureState as TogetherModeCallFeature } from './CallClientState';
+/* @conditional-compile-remove(together-mode) */
+export type { TogetherModeStreamState } from './CallClientState';
+
 export type { RaisedHandState } from './CallClientState';
 export type { DeclarativeCallAgent, IncomingCallManagement } from './CallAgentDeclarative';
 /* @conditional-compile-remove(teams-identity-support) */

diff --git a/packages/communication-react/review/beta/communication-react.api.md b/packages/communication-react/review/beta/communication-react.api.md
@@ -1159,6 +1159,7 @@ export interface CallState {
     spotlight?: SpotlightCallFeatureState;
     startTime: Date;
     state: CallState_2;
+    togetherMode: TogetherModeCallFeature;
     totalParticipantCount?: number;
     transcription: TranscriptionCallFeature;
     transfer: TransferFeature;
@@ -4634,7 +4635,7 @@ export type StartTeamsCallIdentifier = MicrosoftTeamsUserIdentifier | PhoneNumbe
 export interface StatefulCallClient extends CallClient {
     createCallAgent(...args: Parameters<CallClient['createCallAgent']>): Promise<DeclarativeCallAgent>;
     createTeamsCallAgent(...args: Parameters<CallClient['createTeamsCallAgent']>): Promise<DeclarativeTeamsCallAgent>;
-    createView(callId: string | undefined, participantId: CommunicationIdentifier | undefined, stream: LocalVideoStreamState | RemoteVideoStreamState, options?: CreateViewOptions): Promise<CreateViewResult | undefined>;
+    createView(callId: string | undefined, participantId: CommunicationIdentifier | undefined, stream: LocalVideoStreamState | RemoteVideoStreamState | /* @conditional-compile-remove(together-mode) */ TogetherModeStreamState, options?: CreateViewOptions): Promise<CreateViewResult | undefined>;
     disposeView(callId: string | undefined, participantId: CommunicationIdentifier | undefined, stream: LocalVideoStreamState | RemoteVideoStreamState): void;
     getState(): CallClientState;
     offStateChange(handler: (state: CallClientState) => void): void;
@@ -4820,6 +4821,24 @@ export type TeamsOutboundCallAdapterArgs = TeamsCallAdapterArgsCommon & {
 // @public
 export const toFlatCommunicationIdentifier: (identifier: CommunicationIdentifier) => string;
 
+// @alpha
+export interface TogetherModeCallFeature {
+    stream: TogetherModeStreamState[];
+}
+
+// @alpha
+export interface TogetherModeStreamState {
+    id: number;
+    // @public
+    isReceiving: boolean;
+    mediaStreamType: MediaStreamType;
+    streamSize?: {
+        width: number;
+        height: number;
+    };
+    view?: VideoStreamRendererViewState;
+}
+
 // @public
 export type TopicChangedListener = (event: {
     topic: string;

diff --git a/packages/react-composites/src/composites/CallComposite/MockCallAdapter.ts b/packages/react-composites/src/composites/CallComposite/MockCallAdapter.ts
@@ -110,6 +110,9 @@ export class _MockCallAdapter implements CallAdapter {
   createStreamView(): Promise<void> {
     throw Error('createStreamView not implemented');
   }
+  startTogetherMode(): Promise<void> {
+    throw Error('startTogetherMode not implemented');
+  }
   disposeStreamView(): Promise<void> {
     return Promise.resolve();
   }
@@ -255,6 +258,8 @@ const createDefaultCallAdapterState = (role?: ParticipantRole): CallAdapterState
       remoteParticipants: {},
       remoteParticipantsEnded: {},
       raiseHand: { raisedHands: [] },
+      /* @conditional-compile-remove(together-mode) */
+      togetherMode: { stream: [] },
       pptLive: { isActive: false },
       localParticipantReaction: undefined,
       role,