diff --git a/CHANGELOG.md b/CHANGELOG.md index 99d5f79..1223c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,3 +5,9 @@ ## 0.0.2 * Add support for experimental messages. + +## 0.0.3 + +* Add mute/unmute support. +* Break apart state into separate notifiers. +* Update example app to take advantage of both (plus sendText). diff --git a/README.md b/README.md index 91291bc..82526a0 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,9 @@ dependencies: ```dart final session = UltravoxSession.create(); -final state = await session.joinCall(joinUrl); -state.addListener(myListener); +await session.joinCall(joinUrl); +session.statusNotifier.addListener(myListener); +await session.leaveCall(); ``` See the included example app for a more complete example. To get a `joinUrl`, you'll want to integrate your server with the [Ultravox REST API](https://fixie-ai.github.io/ultradox/). diff --git a/example/lib/main.dart b/example/lib/main.dart index f123c2d..0a336a4 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -40,20 +40,25 @@ class MyHomePage extends StatefulWidget { class _MyHomePageState extends State { UltravoxSession? _session; - bool debug = false; + bool _debug = false; + bool _connected = false; @override void dispose() { if (_session != null) { - _session!.state.removeListener(_onStateChange); + _session!.statusNotifier.removeListener(_onStatusChange); unawaited(_session!.leaveCall()); } super.dispose(); } - void _onStateChange() { - // Refresh the UI when the session state changes. - setState(() {}); + void _onStatusChange() { + if (_session?.status.live != _connected) { + // Refresh the UI when we connect and disconnect. + setState(() { + _connected = _session?.status.live ?? false; + }); + } } Future _startCall(String joinUrl) async { @@ -62,9 +67,9 @@ class _MyHomePageState extends State { } setState(() { _session = - UltravoxSession.create(experimentalMessages: debug ? {"debug"} : {}); + UltravoxSession.create(experimentalMessages: _debug ? {"debug"} : {}); }); - _session!.state.addListener(_onStateChange); + _session!.statusNotifier.addListener(_onStatusChange); await _session!.joinCall(joinUrl); } @@ -72,7 +77,7 @@ class _MyHomePageState extends State { if (_session == null) { return; } - _session!.state.removeListener(_onStateChange); + _session!.statusNotifier.removeListener(_onStatusChange); await _session!.leaveCall(); setState(() { _session = null; @@ -104,20 +109,21 @@ class _MyHomePageState extends State { text: 'Debug', style: TextStyle(fontWeight: FontWeight.bold))), Switch( - value: debug, - onChanged: (value) => setState(() => debug = value), + value: _debug, + onChanged: (value) => setState(() => _debug = value), ), const Spacer(), - ElevatedButton( + ElevatedButton.icon( + icon: const Icon(Icons.call), onPressed: () => _startCall(textController.text), - child: const Text('Start Call'), + label: const Text('Start Call'), ), ], ) ], ), )); - } else if (!_session!.state.status.live) { + } else if (!_connected) { mainBodyChildren.add(const Center( child: Column( mainAxisAlignment: MainAxisAlignment.center, @@ -129,31 +135,105 @@ class _MyHomePageState extends State { } else { mainBodyChildren.add( Container( - constraints: const BoxConstraints(maxHeight: 200), - child: ListView( - reverse: true, // Fill from bottom, clip at top. - children: [ - for (final transcript in _session!.state.transcripts.reversed) - TranscriptWidget(transcript: transcript), - ]), - ), + constraints: const BoxConstraints(maxHeight: 200), + child: ListenableBuilder( + listenable: _session!.transcriptsNotifier, + builder: (BuildContext context, Widget? child) { + return ListView( + reverse: true, // Fill from bottom, clip at top. + children: [ + for (final transcript in _session!.transcripts.reversed) + TranscriptWidget(transcript: transcript), + ]); + })), ); - mainBodyChildren.add( - ElevatedButton( - onPressed: _endCall, - child: const Text('End Call'), + final textController = TextEditingController(); + final textInput = TextField( + decoration: const InputDecoration( + border: OutlineInputBorder(), ), + controller: textController, ); - if (debug) { + mainBodyChildren.add(Row( + mainAxisAlignment: MainAxisAlignment.start, + children: [ + Expanded(child: textInput), + ElevatedButton.icon( + icon: const Icon(Icons.send), + onPressed: () { + _session!.sendText(textController.text); + textController.clear(); + }, + label: const Text('Send'), + ), + ], + )); + mainBodyChildren.add(const SizedBox(height: 20)); + mainBodyChildren.add(Row( + mainAxisAlignment: MainAxisAlignment.start, + children: [ + ListenableBuilder( + listenable: _session!.userMutedNotifier, + builder: (BuildContext context, Widget? child) { + return ElevatedButton.icon( + icon: _session!.userMuted + ? const Icon(Icons.mic_off) + : const Icon(Icons.mic), + onPressed: () { + if (_session!.userMuted) { + _session!.unmute({Role.user}); + } else { + _session!.mute({Role.user}); + } + }, + label: _session!.userMuted + ? const Text('Unmute') + : const Text('Mute'), + ); + }), + ListenableBuilder( + listenable: _session!.agentMutedNotifier, + builder: (BuildContext context, Widget? child) { + return ElevatedButton.icon( + icon: _session!.agentMuted + ? const Icon(Icons.volume_off) + : const Icon(Icons.volume_up), + onPressed: () { + if (_session!.agentMuted) { + _session!.unmute({Role.agent}); + } else { + _session!.mute({Role.agent}); + } + }, + label: _session!.agentMuted + ? const Text('Unmute Agent') + : const Text('Mute Agent'), + ); + }), + ElevatedButton.icon( + icon: const Icon(Icons.call_end), + onPressed: _endCall, + label: const Text('End Call'), + ), + ], + )); + if (_debug) { mainBodyChildren.add(const SizedBox(height: 20)); mainBodyChildren.add(const Text.rich(TextSpan( text: 'Last Debug Message:', style: TextStyle(fontWeight: FontWeight.w700)))); - if (_session!.state.lastExperimentalMessage != null) { - mainBodyChildren.add(DebugMessageWidget( - message: _session!.state.lastExperimentalMessage!)); - } + mainBodyChildren.add(ListenableBuilder( + listenable: _session!.experimentalMessageNotifier, + builder: (BuildContext context, Widget? child) { + final message = _session!.lastExperimentalMessage; + if (message.containsKey("type") && message["type"] == "debug") { + return DebugMessageWidget(message: message); + } else { + return const SizedBox(height: 20); + } + }, + )); } } return Scaffold( diff --git a/example/pubspec.lock b/example/pubspec.lock index 1d543ed..686b46c 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -85,10 +85,10 @@ packages: dependency: transitive description: name: dart_webrtc - sha256: ac7ef077084b3e54004716f1d736fcd839e1b60bc3f21f4122a35a9bb5ca2e47 + sha256: c664ad88d5646735753add421ee2118486c100febef5e92b7f59cdbabf6a51f6 url: "https://pub.dev" source: hosted - version: "1.4.8" + version: "1.4.9" dbus: dependency: transitive description: @@ -172,10 +172,10 @@ packages: dependency: transitive description: name: flutter_webrtc - sha256: "67faa07cf49392b50b1aa14590a83caa64d2109345fabd29899dcd8da8538348" + sha256: f6800cc2af79018c12e955ddf8ad007891fdfbb8199b0ce3dccd0977ed2add9c url: "https://pub.dev" source: hosted - version: "0.11.6+hotfix.1" + version: "0.11.7" http: dependency: transitive description: @@ -236,10 +236,10 @@ packages: dependency: transitive description: name: livekit_client - sha256: fc86a8b65b74b41faef646cc671c1892a457c24fd69910e25f7a50dc8cdd3155 + sha256: "5df9b6f153b5f2c59fbf116b41e54597dfe8b2340b6630f7d8869887a9e58f44" url: "https://pub.dev" source: hosted - version: "2.2.4" + version: "2.2.5" logging: dependency: transitive description: @@ -441,10 +441,10 @@ packages: dependency: transitive description: name: synchronized - sha256: a824e842b8a054f91a728b783c177c1e4731f6b124f9192468457a8913371255 + sha256: "51b08572b9f091f8c3eb4d9d4be253f196ff0075d5ec9b10a884026d5b55d7bc" url: "https://pub.dev" source: hosted - version: "3.2.0" + version: "3.3.0+2" term_glyph: dependency: transitive description: @@ -475,7 +475,7 @@ packages: path: ".." relative: true source: path - version: "0.0.1" + version: "0.0.3" uuid: dependency: transitive description: @@ -504,10 +504,10 @@ packages: dependency: transitive description: name: web - sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27" + sha256: d43c1d6b787bf0afad444700ae7f4db8827f701bc61c255ac8d328c6f4d52062 url: "https://pub.dev" source: hosted - version: "0.5.1" + version: "1.0.0" web_socket: dependency: transitive description: @@ -544,10 +544,10 @@ packages: dependency: transitive description: name: win32_registry - sha256: "723b7f851e5724c55409bb3d5a32b203b3afe8587eaf5dafb93a5fed8ecda0d6" + sha256: "21ec76dfc731550fd3e2ce7a33a9ea90b828fdf19a5c3bcf556fa992cfa99852" url: "https://pub.dev" source: hosted - version: "1.1.4" + version: "1.1.5" xdg_directories: dependency: transitive description: diff --git a/lib/src/session.dart b/lib/src/session.dart index 294943f..cc57e14 100644 --- a/lib/src/session.dart +++ b/lib/src/session.dart @@ -6,6 +6,7 @@ import 'dart:convert'; /// The current status of an [UltravoxSession]. enum UltravoxSessionStatus { /// The voice session is not connected and not attempting to connect. + /// /// This is the initial state of a voice session. disconnected(live: false), @@ -22,10 +23,12 @@ enum UltravoxSessionStatus { listening(live: true), /// The client is connected and the server is considering its response. + /// /// The user can still interrupt. thinking(live: true), /// The client is connected and the server is playing response audio. + /// /// The user can interrupt as needed. speaking(live: true); @@ -66,24 +69,16 @@ class Transcript { }); } -/// A state object for an [UltravoxSession]. -/// [UltravoxSessionState] is a [ChangeNotifier] that manages the state of a -/// single session and notifies listeners when the state changes. -class UltravoxSessionState extends ChangeNotifier { +/// A collection of [Transcript]s for an [UltravoxSession]. +/// +/// [Transcripts] is a [ChangeNotifier] that notifies listeners when +/// transcripts are updated or new transcripts are added. +class Transcripts extends ChangeNotifier { final _transcripts = []; - var _status = UltravoxSessionStatus.disconnected; - Map? _lastExperimentalMessage; - UltravoxSessionStatus get status => _status; List get transcripts => List.unmodifiable(_transcripts); - Map? get lastExperimentalMessage => _lastExperimentalMessage; - set status(UltravoxSessionStatus value) { - _status = value; - notifyListeners(); - } - - void addOrUpdateTranscript(Transcript transcript) { + void _addOrUpdateTranscript(Transcript transcript) { if (_transcripts.isNotEmpty && !_transcripts.last.isFinal && _transcripts.last.speaker == transcript.speaker) { @@ -94,16 +89,63 @@ class UltravoxSessionState extends ChangeNotifier { } notifyListeners(); } - - set lastExperimentalMessage(Map? value) { - _lastExperimentalMessage = value; - notifyListeners(); - } } /// Manages a single session with Ultravox. +/// +/// In addition to providing methods to manage a call, [UltravoxSession] exposes +/// several notifiers that allow UI elements to listen for specific state +/// changes. class UltravoxSession { - final _state = UltravoxSessionState(); + /// A [ValueNotifier] that emits events when the session status changes. + final statusNotifier = ValueNotifier( + UltravoxSessionStatus.disconnected, + ); + + /// A quick accessor for the session's current status. + /// + /// Listen to [statusNotifier] to receive updates when this changes. + UltravoxSessionStatus get status => statusNotifier.value; + + /// A [ChangeNotifier] that emits events when new transcripts are available. + final transcriptsNotifier = Transcripts(); + + /// A quick accessor for the session's current transcripts. + /// + /// Listen to [transcriptsNotifier] to receive updates on transcript changes. + List get transcripts => transcriptsNotifier.transcripts; + + /// A [ValueNotifier] that emits events when new experimental messages are + /// received. + /// + /// Experimental messages are messages that are not part of the released + /// Ultravox API but may be selected for testing new features or debugging. + /// The messages received depend on the `experimentalMessages` provided to + /// [UltravoxSession.create]. + final experimentalMessageNotifier = ValueNotifier>({}); + + /// A quick accessor for the last experimental message received. + /// + /// Listen to [experimentalMessageNotifier] to receive updates. + Map get lastExperimentalMessage => + experimentalMessageNotifier.value; + + /// A [ValueNotifier] that emits events when the user is muted or unmuted. + final userMutedNotifier = ValueNotifier(false); + + /// A quick accessor for the user's current mute status. + /// + /// Listen to [userMutedNotifier] to receive updates. + bool get userMuted => userMutedNotifier.value; + + /// A [ValueNotifier] that emits events when the agent is muted or unmuted. + final agentMutedNotifier = ValueNotifier(false); + + /// A quick accessor for the agent's current mute status. + /// + /// Listen to [agentMutedNotifier] to receive updates. + bool get agentMuted => agentMutedNotifier.value; + final Set _experimentalMessages; final lk.Room _room; final lk.EventsListener _listener; @@ -115,14 +157,12 @@ class UltravoxSession { UltravoxSession.create({Set? experimentalMessages}) : this(lk.Room(), experimentalMessages ?? {}); - UltravoxSessionState get state => _state; - - /// Connects to call using the given [joinUrl]. - Future joinCall(String joinUrl) async { - if (_state.status != UltravoxSessionStatus.disconnected) { + /// Connects to a call using the given [joinUrl]. + Future joinCall(String joinUrl) async { + if (status != UltravoxSessionStatus.disconnected) { throw Exception('Cannot join a new call while already in a call'); } - _changeStatus(UltravoxSessionStatus.connecting); + statusNotifier.value = UltravoxSessionStatus.connecting; var url = Uri.parse(joinUrl); if (_experimentalMessages.isNotEmpty) { final queryParameters = Map.from(url.queryParameters) @@ -136,7 +176,52 @@ class UltravoxSession { _wsChannel.stream.listen((event) async { await _handleSocketMessage(event); }); - return _state; + } + + /// Mutes the user, the agent, or both. + /// + /// If a given [Role] is already muted, this method does nothing for that + /// role. + void mute(Set roles) { + if (roles.contains(Role.user)) { + if (!userMuted) { + _room.localParticipant?.setMicrophoneEnabled(false); + } + userMutedNotifier.value = true; + } + if (roles.contains(Role.agent)) { + if (!agentMuted) { + for (final participant in _room.remoteParticipants.values) { + for (final publication in participant.audioTrackPublications) { + publication.track?.disable(); + } + } + } + agentMutedNotifier.value = true; + } + } + + /// Unmutes the user, the agent, or both. + /// + /// If a given [Role] is not currently muted, this method does nothing for + /// that role. + void unmute(Set roles) { + if (roles.contains(Role.user)) { + if (userMuted) { + _room.localParticipant?.setMicrophoneEnabled(true); + } + userMutedNotifier.value = false; + } + if (roles.contains(Role.agent)) { + if (agentMuted) { + for (final participant in _room.remoteParticipants.values) { + for (final publication in participant.audioTrackPublications) { + publication.track?.enable(); + } + } + } + agentMutedNotifier.value = false; + } } /// Leaves the current call (if any). @@ -146,28 +231,24 @@ class UltravoxSession { /// Sends a message via text. The agent will also respond via text. Future sendText(String text) async { - if (!_state.status.live) { + if (!status.live) { throw Exception( - 'Cannot send text while not connected. Current status: ${_state.status}'); + 'Cannot send text while not connected. Current status: $status'); } final message = jsonEncode({'type': 'input_text_message', 'text': text}); _room.localParticipant?.publishData(utf8.encode(message), reliable: true); } - void _changeStatus(UltravoxSessionStatus status) { - _state.status = status; - } - Future _disconnect() async { - if (_state.status == UltravoxSessionStatus.disconnected) { + if (status == UltravoxSessionStatus.disconnected) { return; } - _changeStatus(UltravoxSessionStatus.disconnecting); + statusNotifier.value = UltravoxSessionStatus.disconnecting; await Future.wait([ _room.disconnect(), _wsChannel.sink.close(), ]); - _changeStatus(UltravoxSessionStatus.disconnected); + statusNotifier.value = UltravoxSessionStatus.disconnected; } Future _handleSocketMessage(dynamic event) async { @@ -183,7 +264,7 @@ class UltravoxSession { _listener ..on(_handleTrackSubscribed) ..on(_handleDataMessage); - _changeStatus(UltravoxSessionStatus.idle); + statusNotifier.value = UltravoxSessionStatus.idle; break; default: // ignore @@ -200,13 +281,13 @@ class UltravoxSession { case 'state': switch (data['state']) { case 'listening': - _changeStatus(UltravoxSessionStatus.listening); + statusNotifier.value = UltravoxSessionStatus.listening; break; case 'thinking': - _changeStatus(UltravoxSessionStatus.thinking); + statusNotifier.value = UltravoxSessionStatus.thinking; break; case 'speaking': - _changeStatus(UltravoxSessionStatus.speaking); + statusNotifier.value = UltravoxSessionStatus.speaking; break; default: // ignore @@ -222,7 +303,7 @@ class UltravoxSession { speaker: Role.user, medium: medium, ); - _state.addOrUpdateTranscript(transcript); + transcriptsNotifier._addOrUpdateTranscript(transcript); break; case 'voice_synced_transcript': case 'agent_text_transcript': @@ -236,23 +317,23 @@ class UltravoxSession { speaker: Role.agent, medium: medium, ); - _state.addOrUpdateTranscript(transcript); + transcriptsNotifier._addOrUpdateTranscript(transcript); } else if (data['delta'] != null) { - final last = _state.transcripts.last; - if (last.speaker == Role.agent) { + final last = transcriptsNotifier._transcripts.lastOrNull; + if (last?.speaker == Role.agent) { final transcript = Transcript( - text: last.text + (data['delta'] as String), + text: last!.text + (data['delta'] as String), isFinal: data['final'] as bool, speaker: Role.agent, medium: medium, ); - _state.addOrUpdateTranscript(transcript); + transcriptsNotifier._addOrUpdateTranscript(transcript); } } break; default: if (_experimentalMessages.isNotEmpty) { - _state.lastExperimentalMessage = data as Map; + experimentalMessageNotifier.value = data as Map; } } } diff --git a/pubspec.yaml b/pubspec.yaml index a8a533e..103f00c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: ultravox_client description: "Flutter client SDK for Ultravox." -version: 0.0.2 +version: 0.0.3 homepage: https://ultravox.ai repository: https://github.com/fixie-ai/ultravox-client-sdk-flutter topics: diff --git a/test/ultravox_client_test.dart b/test/ultravox_client_test.dart index cd583ca..4c319ec 100644 --- a/test/ultravox_client_test.dart +++ b/test/ultravox_client_test.dart @@ -3,53 +3,23 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:ultravox_client/ultravox_client.dart'; void main() { - test('update transcript', () { - final state = UltravoxSessionState(); - final transcript1 = Transcript( - text: 'Hello', - isFinal: false, - speaker: Role.user, - medium: Medium.voice, - ); - final transcript2 = Transcript( - text: 'Hello world!', - isFinal: true, - speaker: Role.user, - medium: Medium.voice, - ); - state.addOrUpdateTranscript(transcript1); - - var fired = false; - state.addListener(() { - fired = true; - expect(state.transcripts, [transcript2]); - }); - state.addOrUpdateTranscript(transcript2); - expect(fired, true); - }); - - test('add transcript', () { - final state = UltravoxSessionState(); - final transcript1 = Transcript( - text: 'Hello world!', - isFinal: true, - speaker: Role.user, - medium: Medium.voice, - ); - final transcript2 = Transcript( - text: 'Something else', - isFinal: false, - speaker: Role.user, - medium: Medium.voice, - ); - state.addOrUpdateTranscript(transcript1); - - var fired = false; - state.addListener(() { - fired = true; - expect(state.transcripts, [transcript1, transcript2]); + test('mute', () { + final session = UltravoxSession.create(); + int muteCounter = 0; + session.userMutedNotifier.addListener(() { + muteCounter++; }); - state.addOrUpdateTranscript(transcript2); - expect(fired, true); + session.mute({Role.user}); + expect(muteCounter, 1); + session.mute({Role.user}); + expect(muteCounter, 1); + session.unmute({Role.user}); + expect(muteCounter, 2); + session.mute({Role.user, Role.agent}); + expect(muteCounter, 3); + session.unmute({}); + expect(muteCounter, 3); + session.unmute({Role.agent}); + expect(muteCounter, 3); }); }