diff --git a/CHANGELOG.md b/CHANGELOG.md index 490fdb5..d2eb6fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,3 +24,10 @@ # 0.0.6 * Add ability to set the output medium. + +# 0.0.7 + +* Start informing the server of the client version and API version. +* Use simplifed `transcript` messages. +* Expose `sendData` and `dataMessageNotifier` for bleeding edge use cases. +* Update dependencies. diff --git a/PUBLISHING.md b/PUBLISHING.md index c78235e..e31149f 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -2,7 +2,9 @@ The ultravox_client for Flutter is available on [pub.dev](https://pub.dev/packages/ultravox_client). To publish a new version: -1. **Version Bump** → Increment the version number in `pubspec.yaml`. +1. **Version Bump** → Increment the version number in `pubspec.yaml` and at the top of `lib/src/session.dart`. 1. **Change Log** → Add the new version number along with a brief summary of what's new to `CHANGELOG.md`. 1. **Error Check** → Run `dart pub publish --dry-run` and deal with any errors or unexpected includes. -1. **Publish** → Run `dart pub publish`. \ No newline at end of file +1. **Merge to main** → Open a PR in GitHub and get the changes merged. (This also runs tests, so please only publish from main!) +1. **Publish** → Run `dart pub publish`. +1. **Tag/Release** → Create a new tag and release in GitHub please. diff --git a/example/lib/main.dart b/example/lib/main.dart index c46b2bc..c6fb8b2 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:collection'; import 'dart:convert'; import 'package:flutter/material.dart'; @@ -42,12 +43,14 @@ class MyHomePage extends StatefulWidget { class _MyHomePageState extends State { UltravoxSession? _session; bool _debug = false; + final LinkedHashSet _transcriptArrivalOrder = LinkedHashSet(); bool _connected = false; @override void dispose() { if (_session != null) { _session!.statusNotifier.removeListener(_onStatusChange); + _session!.dataMessageNotifier.removeListener(_onDataMessage); unawaited(_session!.leaveCall()); } super.dispose(); @@ -62,6 +65,18 @@ class _MyHomePageState extends State { } } + void _onDataMessage() { + final message = _session!.lastDataMessage; + if (message["type"] == "transcript" && message.containsKey("ordinal")) { + final ordinal = message["ordinal"] as int; + if (!_transcriptArrivalOrder.contains(ordinal)) { + setState(() { + _transcriptArrivalOrder.add(ordinal); + }); + } + } + } + Future _startCall(String joinUrl) async { if (_session != null) { return; @@ -71,8 +86,9 @@ class _MyHomePageState extends State { UltravoxSession.create(experimentalMessages: _debug ? {"debug"} : {}); }); _session!.statusNotifier.addListener(_onStatusChange); + _session!.dataMessageNotifier.addListener(_onDataMessage); _session!.registerToolImplementation("getSecretMenu", _getSecretMenu); - await _session!.joinCall(joinUrl); + await _session!.joinCall(joinUrl, clientVersion: "UltravoxExampleApp"); } ClientToolResult _getSecretMenu(Object params) { @@ -92,10 +108,12 @@ class _MyHomePageState extends State { } Future _endCall() async { + _transcriptArrivalOrder.clear(); if (_session == null) { return; } _session!.statusNotifier.removeListener(_onStatusChange); + _session!.dataMessageNotifier.removeListener(_onDataMessage); await _session!.leaveCall(); setState(() { _session = null; @@ -244,6 +262,12 @@ class _MyHomePageState extends State { } }, )); + + mainBodyChildren.add(const SizedBox(height: 10)); + mainBodyChildren.add(const Text.rich(TextSpan( + text: 'Transcript Arrival Order:', + style: TextStyle(fontWeight: FontWeight.w700)))); + mainBodyChildren.add(Text(_transcriptArrivalOrder.join(", "))); } } return Scaffold( diff --git a/example/pubspec.lock b/example/pubspec.lock index 316f65d..735e593 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -5,10 +5,10 @@ packages: dependency: transitive description: name: args - sha256: "7cf60b9f0cc88203c5a190b4cd62a99feea42759a7fa695010eb5de1c0b2252a" + sha256: bf9f5caeea8d8fe6721a9c358dd8a5c1947b27f1cfaa18b39c301273594919e6 url: "https://pub.dev" source: hosted - version: "2.5.0" + version: "2.6.0" async: dependency: transitive description: @@ -53,10 +53,10 @@ packages: dependency: transitive description: name: connectivity_plus - sha256: "2056db5241f96cdc0126bd94459fc4cdc13876753768fc7a31c425e50a7177d0" + sha256: "876849631b0c7dc20f8b471a2a03142841b482438e3b707955464f5ffca3e4c3" url: "https://pub.dev" source: hosted - version: "6.0.5" + version: "6.1.0" connectivity_plus_platform_interface: dependency: transitive description: @@ -69,10 +69,10 @@ packages: dependency: transitive description: name: crypto - sha256: ec30d999af904f33454ba22ed9a86162b35e52b44ac4807d1d93c288041d7d27 + sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855" url: "https://pub.dev" source: hosted - version: "3.0.5" + version: "3.0.6" cupertino_icons: dependency: "direct main" description: @@ -101,10 +101,10 @@ packages: dependency: transitive description: name: device_info_plus - sha256: a7fd703482b391a87d60b6061d04dfdeab07826b96f9abd8f5ed98068acc0074 + sha256: f545ffbadee826f26f2e1a0f0cbd667ae9a6011cc0f77c0f8f00a969655e6e95 url: "https://pub.dev" source: hosted - version: "10.1.2" + version: "11.1.1" device_info_plus_platform_interface: dependency: transitive description: @@ -133,18 +133,18 @@ packages: dependency: transitive description: name: file - sha256: "5fc22d7c25582e38ad9a8515372cd9a93834027aacf1801cf01164dac0ffa08c" + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 url: "https://pub.dev" source: hosted - version: "7.0.0" + version: "7.0.1" fixnum: dependency: transitive description: name: fixnum - sha256: "25517a4deb0c03aa0f32fd12db525856438902d9c16536311e76cdc57b31d7d1" + sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be url: "https://pub.dev" source: hosted - version: "1.1.0" + version: "1.1.1" flutter: dependency: "direct main" description: flutter @@ -154,10 +154,10 @@ packages: dependency: "direct dev" description: name: flutter_lints - sha256: "3f41d009ba7172d5ff9be5f6e6e6abb4300e263aab8866d2a0842ed2a70f8f0c" + sha256: "5398f14efa795ffb7a33e9b6a08798b26a180edac4ad7db3f231e40f82ce11e1" url: "https://pub.dev" source: hosted - version: "4.0.0" + version: "5.0.0" flutter_test: dependency: "direct dev" description: flutter @@ -172,10 +172,10 @@ packages: dependency: transitive description: name: flutter_webrtc - sha256: f6800cc2af79018c12e955ddf8ad007891fdfbb8199b0ce3dccd0977ed2add9c + sha256: "0b69ecab98211504c10d40c1c4cb48eb387e03ea8e732079bd0d2665d8c20d3f" url: "https://pub.dev" source: hosted - version: "0.11.7" + version: "0.12.1+hotfix.1" http: dependency: transitive description: @@ -228,26 +228,26 @@ packages: dependency: transitive description: name: lints - sha256: "976c774dd944a42e83e2467f4cc670daef7eed6295b10b36ae8c85bcbf828235" + sha256: "3315600f3fb3b135be672bf4a178c55f274bebe368325ae18462c89ac1e3b413" url: "https://pub.dev" source: hosted - version: "4.0.0" + version: "5.0.0" livekit_client: dependency: transitive description: name: livekit_client - sha256: "5df9b6f153b5f2c59fbf116b41e54597dfe8b2340b6630f7d8869887a9e58f44" + sha256: ad55045435fbf1a106e2da4c9a8d523755ce834db47f6d967beaa58228b21a05 url: "https://pub.dev" source: hosted - version: "2.2.5" + version: "2.3.0" logging: dependency: transitive description: name: logging - sha256: "623a88c9594aa774443aa3eb2d41807a48486b5613e67599fb4c41c0ad47c340" + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 url: "https://pub.dev" source: hosted - version: "1.2.0" + version: "1.3.0" matcher: dependency: transitive description: @@ -292,18 +292,18 @@ packages: dependency: transitive description: name: path_provider - sha256: fec0d61223fba3154d87759e3cc27fe2c8dc498f6386c6d6fc80d1afdd1bf378 + sha256: "50c5dd5b6e1aaf6fb3a78b33f6aa3afca52bf903a8a5298f53101fdaee55bbcd" url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.1.5" path_provider_android: dependency: transitive description: name: path_provider_android - sha256: "6f01f8e37ec30b07bc424b4deabac37cacb1bc7e2e515ad74486039918a37eb7" + sha256: c464428172cb986b758c6d1724c603097febb8fb855aa265aeecc9280c294d4a url: "https://pub.dev" source: hosted - version: "2.2.10" + version: "2.2.12" path_provider_foundation: dependency: transitive description: @@ -348,10 +348,10 @@ packages: dependency: transitive description: name: platform - sha256: "9b71283fc13df574056616011fb138fd3b793ea47cc509c189a6c3fa5f8a1a65" + sha256: "5d6b1b0036a5f331ebc77c850ebc8506cbc1e9416c27e59b439f917a902a4984" url: "https://pub.dev" source: hosted - version: "3.1.5" + version: "3.1.6" platform_detect: dependency: transitive description: @@ -441,10 +441,10 @@ packages: dependency: transitive description: name: synchronized - sha256: "51b08572b9f091f8c3eb4d9d4be253f196ff0075d5ec9b10a884026d5b55d7bc" + sha256: "69fe30f3a8b04a0be0c15ae6490fc859a78ef4c43ae2dd5e8a623d45bfcf9225" url: "https://pub.dev" source: hosted - version: "3.3.0+2" + version: "3.3.0+3" term_glyph: dependency: transitive description: @@ -465,25 +465,25 @@ packages: dependency: transitive description: name: typed_data - sha256: facc8d6582f16042dd49f2463ff1bd6e2c9ef9f3d5da3d9b087e244a7b564b3c + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 url: "https://pub.dev" source: hosted - version: "1.3.2" + version: "1.4.0" ultravox_client: dependency: "direct main" description: path: ".." relative: true source: path - version: "0.0.4" + version: "0.0.7" uuid: dependency: transitive description: name: uuid - sha256: f33d6bb662f0e4f79dcd7ada2e6170f3b3a2530c28fc41f49a411ddedd576a77 + sha256: a5be9ef6618a7ac1e964353ef476418026db906c4facdedaa299b7a2e71690ff url: "https://pub.dev" source: hosted - version: "4.5.0" + version: "4.5.1" vector_math: dependency: transitive description: @@ -504,10 +504,10 @@ packages: dependency: transitive description: name: web - sha256: d43c1d6b787bf0afad444700ae7f4db8827f701bc61c255ac8d328c6f4d52062 + sha256: cd3543bd5798f6ad290ea73d210f423502e71900302dde696f8bff84bf89a1cb url: "https://pub.dev" source: hosted - version: "1.0.0" + version: "1.1.0" web_socket: dependency: transitive description: @@ -536,10 +536,10 @@ packages: dependency: transitive description: name: win32 - sha256: "68d1e89a91ed61ad9c370f9f8b6effed9ae5e0ede22a270bdfa6daf79fc2290a" + sha256: "84ba388638ed7a8cb3445a320c8273136ab2631cd5f2c57888335504ddab1bc2" url: "https://pub.dev" source: hosted - version: "5.5.4" + version: "5.8.0" win32_registry: dependency: transitive description: @@ -552,10 +552,10 @@ packages: dependency: transitive description: name: xdg_directories - sha256: faea9dee56b520b55a566385b84f2e8de55e7496104adada9962e0bd11bcff1d + sha256: "7a3f37b05d989967cdddcbb571f1ea834867ae2faa29725fd085180e0883aa15" url: "https://pub.dev" source: hosted - version: "1.0.4" + version: "1.1.0" xml: dependency: transitive description: @@ -566,4 +566,4 @@ packages: version: "6.5.0" sdks: dart: ">=3.5.1 <4.0.0" - flutter: ">=3.22.0" + flutter: ">=3.24.0" diff --git a/example/pubspec.yaml b/example/pubspec.yaml index 03e2bcf..38984dc 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -19,7 +19,7 @@ dependencies: dev_dependencies: flutter_test: sdk: flutter - flutter_lints: ^4.0.0 + flutter_lints: ^5.0.0 flutter: uses-material-design: true diff --git a/lib/src/session.dart b/lib/src/session.dart index 26e7bfa..c669312 100644 --- a/lib/src/session.dart +++ b/lib/src/session.dart @@ -5,6 +5,8 @@ import 'package:livekit_client/livekit_client.dart' as lk; import 'package:web_socket_channel/web_socket_channel.dart'; import 'dart:convert'; +const ultravoxSdkVersion = '0.0.7'; + /// The current status of an [UltravoxSession]. enum UltravoxSessionStatus { /// The voice session is not connected and not attempting to connect. @@ -76,18 +78,32 @@ class Transcript { /// [Transcripts] is a [ChangeNotifier] that notifies listeners when /// transcripts are updated or new transcripts are added. class Transcripts extends ChangeNotifier { - final _transcripts = []; + final _transcripts = []; - List get transcripts => List.unmodifiable(_transcripts); + List get transcripts => + List.unmodifiable(_transcripts.whereType()); - void _addOrUpdateTranscript(Transcript transcript) { - if (_transcripts.isNotEmpty && - !_transcripts.last.isFinal && - _transcripts.last.speaker == transcript.speaker) { - _transcripts.replaceRange( - _transcripts.length - 1, _transcripts.length, [transcript]); + void _addOrUpdateTranscript( + int ordinal, Medium medium, Role speaker, bool isFinal, + {String? text, String? delta}) { + while (_transcripts.length < ordinal) { + _transcripts.add(null); + } + if (_transcripts.length == ordinal) { + _transcripts.add(Transcript( + text: text ?? delta ?? '', + isFinal: isFinal, + speaker: speaker, + medium: medium, + )); } else { - _transcripts.add(transcript); + final priorText = _transcripts[ordinal]?.text ?? ''; + _transcripts[ordinal] = Transcript( + text: text ?? priorText + (delta ?? ''), + isFinal: isFinal, + speaker: speaker, + medium: medium, + ); } notifyListeners(); } @@ -160,6 +176,17 @@ class UltravoxSession { Map get lastExperimentalMessage => experimentalMessageNotifier.value; + /// A [ValueNotifier] that emits events when any new data messages are + /// received, including those typically handled by this SDK. + /// + /// See https://docs.ultravox.ai/datamessages for message types. + final dataMessageNotifier = ValueNotifier>({}); + + /// A quick accessor for the last data message received. + /// + /// Listen to [dataMessageNotifier] to receive updates. + Map get lastDataMessage => dataMessageNotifier.value; + /// A [ValueNotifier] that emits events when the user's mic is muted or unmuted. final micMutedNotifier = ValueNotifier(false); @@ -234,19 +261,24 @@ class UltravoxSession { } /// Connects to a call using the given [joinUrl]. - Future joinCall(String joinUrl) async { + Future joinCall(String joinUrl, {String? clientVersion}) async { if (status != UltravoxSessionStatus.disconnected) { throw Exception('Cannot join a new call while already in a call'); } statusNotifier.value = UltravoxSessionStatus.connecting; var url = Uri.parse(joinUrl); + final queryParams = Map.from(url.queryParameters); + var uvClientVersion = "flutter_$ultravoxSdkVersion"; + if (clientVersion != null) { + uvClientVersion += ":$clientVersion"; + } + queryParams.addAll({'clientVersion': uvClientVersion, 'apiVersion': '1'}); if (_experimentalMessages.isNotEmpty) { - final queryParameters = Map.from(url.queryParameters) - ..addAll({ - 'experimentalMessages': _experimentalMessages.join(','), - }); - url = url.replace(queryParameters: queryParameters); + queryParams.addAll({ + 'experimentalMessages': _experimentalMessages.join(','), + }); } + url = url.replace(queryParameters: queryParams); _wsChannel = WebSocketChannel.connect(url); await _wsChannel.ready; _wsChannel.stream.listen((event) async { @@ -268,7 +300,7 @@ class UltravoxSession { throw Exception( 'Cannot set speaker medium while not connected. Current status: $status'); } - await _sendData({'type': 'set_output_medium', 'medium': medium.name}); + await sendData({'type': 'set_output_medium', 'medium': medium.name}); } /// Sends a message via text. @@ -277,7 +309,19 @@ class UltravoxSession { throw Exception( 'Cannot send text while not connected. Current status: $status'); } - await _sendData({'type': 'input_text_message', 'text': text}); + await sendData({'type': 'input_text_message', 'text': text}); + } + + /// Sends an arbitrary data message to the server. + /// + /// See https://docs.ultravox.ai/datamessages for message types. + Future sendData(Map data) async { + if (!data.containsKey("type")) { + throw Exception("Data must contain a 'type' key"); + } + final message = jsonEncode(data); + await _room.localParticipant + ?.publishData(utf8.encode(message), reliable: true); } Future _disconnect() async { @@ -318,7 +362,8 @@ class UltravoxSession { } Future _handleDataMessage(lk.DataReceivedEvent event) async { - final data = jsonDecode(utf8.decode(event.data)); + final data = jsonDecode(utf8.decode(event.data)) as Map; + dataMessageNotifier.value = data; switch (data['type']) { case 'state': switch (data['state']) { @@ -336,41 +381,18 @@ class UltravoxSession { } break; case 'transcript': - final medium = data['transcript']['medium'] == 'voice' - ? Medium.voice - : Medium.text; - final transcript = Transcript( - text: data['transcript']['text'] as String, - isFinal: data['transcript']['final'] as bool, - speaker: Role.user, - medium: medium, - ); - transcriptsNotifier._addOrUpdateTranscript(transcript); - break; - case 'voice_synced_transcript': - case 'agent_text_transcript': - final medium = data['type'] == 'voice_synced_transcript' - ? Medium.voice - : Medium.text; + final medium = data['medium'] == 'voice' ? Medium.voice : Medium.text; + final role = data['role'] == 'agent' ? Role.agent : Role.user; + final ordinal = data['ordinal'] as int; + final isFinal = data['final'] as bool? ?? false; if (data['text'] != null) { - final transcript = Transcript( - text: data['text'] as String, - isFinal: data['final'] as bool, - speaker: Role.agent, - medium: medium, - ); - transcriptsNotifier._addOrUpdateTranscript(transcript); + transcriptsNotifier._addOrUpdateTranscript( + ordinal, medium, role, isFinal, + text: data['text'] as String); } else if (data['delta'] != null) { - final last = transcriptsNotifier._transcripts.lastOrNull; - if (last?.speaker == Role.agent) { - final transcript = Transcript( - text: last!.text + (data['delta'] as String), - isFinal: data['final'] as bool, - speaker: Role.agent, - medium: medium, - ); - transcriptsNotifier._addOrUpdateTranscript(transcript); - } + transcriptsNotifier._addOrUpdateTranscript( + ordinal, medium, role, isFinal, + delta: data['delta'] as String); } break; case 'client_tool_invocation': @@ -378,7 +400,7 @@ class UltravoxSession { data['invocationId'] as String, data['parameters'] as Object); default: if (_experimentalMessages.isNotEmpty) { - experimentalMessageNotifier.value = data as Map; + experimentalMessageNotifier.value = data; } } } @@ -387,7 +409,7 @@ class UltravoxSession { String toolName, String invocationId, Object parameters) async { final tool = _registeredTools[toolName]; if (tool == null) { - await _sendData({ + await sendData({ 'type': 'client_tool_result', 'invocationId': invocationId, 'errorType': 'undefined', @@ -406,9 +428,9 @@ class UltravoxSession { if (result.responseType != null) { data['responseType'] = result.responseType!; } - await _sendData(data); + await sendData(data); } catch (e) { - await _sendData({ + await sendData({ 'type': 'client_tool_result', 'invocationId': invocationId, 'errorType': 'implementation-error', @@ -416,10 +438,4 @@ class UltravoxSession { }); } } - - Future _sendData(Object data) async { - final message = jsonEncode(data); - await _room.localParticipant - ?.publishData(utf8.encode(message), reliable: true); - } } diff --git a/lib/ultravox_client.dart b/lib/ultravox_client.dart index 65b9daf..883f76c 100644 --- a/lib/ultravox_client.dart +++ b/lib/ultravox_client.dart @@ -1,3 +1,3 @@ -library ultravox_client; +library; export 'src/session.dart'; diff --git a/pubspec.yaml b/pubspec.yaml index 4fd9cd0..ebb2e41 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: ultravox_client description: "Flutter client SDK for Ultravox." -version: 0.0.6 +version: 0.0.7 homepage: https://ultravox.ai repository: https://github.com/fixie-ai/ultravox-client-sdk-flutter topics: @@ -23,13 +23,14 @@ environment: dependencies: flutter: sdk: flutter - livekit_client: ^2.2.4 + livekit_client: ^2.3.0 web_socket_channel: ^3.0.1 dev_dependencies: flutter_test: sdk: flutter - flutter_lints: ^4.0.0 + flutter_lints: ^5.0.0 mockito: ^5.4.4 build_runner: ^2.4.13 + pubspec_parse: ^1.3.0 diff --git a/test/check_version_test.dart b/test/check_version_test.dart new file mode 100644 index 0000000..f7fc452 --- /dev/null +++ b/test/check_version_test.dart @@ -0,0 +1,15 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:pubspec_parse/pubspec_parse.dart'; +import 'package:ultravox_client/ultravox_client.dart'; + +void main() { + test('pubspec version matches SDK constant', () async { + final file = File("./pubspec.yaml"); + final fileContent = await file.readAsString(); + final pubspec = Pubspec.parse(fileContent); + final expected = pubspec.version!.canonicalizedVersion; + expect(ultravoxSdkVersion, expected); + }); +}