From eda565dfa4fa18e80cf3fe15f93482299932f071 Mon Sep 17 00:00:00 2001 From: Rodion Mostovoi Date: Mon, 14 Aug 2023 04:40:54 +0600 Subject: [PATCH] Refactor ChatGPT translator to support economical batching This commit involves major refactoring of the `ChatGPTTranslatorServiceEconomical.cs` file. This change replaces single translation process with a more economical batching. The primary element Buffer has been replaced by Batch and changed the usage accordingly. This optimization allows multitudes of text to be translated in one session, thus saving resources and improving performance. Additional changes include the implementation of a new interface `IChatGPTTranslatorService` for translation services and some changes to method signatures. Tests were updated to imitate a batch translation scenario to ensure the successful functionality of the updates. --- .../ChatGPTTranslatorService.cs | 2 +- .../ChatGPTTranslatorServiceEconomical.cs | 72 +++++++++------ .../IChatGPTTranslatorService.cs | 34 ++++++++ .../ChatGptTranslatorServiceTests.cs | 87 ++++++++++++++++++- 4 files changed, 166 insertions(+), 29 deletions(-) create mode 100644 src/modules/OpenAI.ChatGpt.Modules.Translator/IChatGPTTranslatorService.cs diff --git a/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorService.cs b/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorService.cs index c8e275d..bd465c3 100644 --- a/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorService.cs +++ b/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorService.cs @@ -8,7 +8,7 @@ namespace OpenAI.ChatGpt.Modules.Translator; [Fody.ConfigureAwait(false)] // ReSharper disable once InconsistentNaming -public class ChatGPTTranslatorService : IDisposable +public class ChatGPTTranslatorService : IDisposable, IChatGPTTranslatorService { private readonly IOpenAiClient _client; private readonly string? _defaultSourceLanguage; diff --git a/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorServiceEconomical.cs b/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorServiceEconomical.cs index 72d986f..e2f27bd 100644 --- a/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorServiceEconomical.cs +++ b/src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorServiceEconomical.cs @@ -3,9 +3,12 @@ namespace OpenAI.ChatGpt.Modules.Translator; +/// +/// Provides a service for translating text using GPT models with economical batching. +/// public class ChatGPTTranslatorServiceEconomical : IAsyncDisposable { - private readonly ChatGPTTranslatorService _chatGptTranslatorService; + private readonly IChatGPTTranslatorService _chatGptTranslatorService; private readonly string _sourceLanguage; private readonly string _targetLanguage; private readonly int? _maxTokens; @@ -16,12 +19,24 @@ public class ChatGPTTranslatorServiceEconomical : IAsyncDisposable private readonly TimeSpan _sendRequestAfterInactivity; private static readonly TimeSpan DefaultSendRequestAfterInactivity = TimeSpan.FromMilliseconds(100); - private Buffer _buffer = new(); - private TaskCompletionSource _tcs = new(); + private Batch _batch = new(); + private TaskCompletionSource _tcs = new(); private readonly object _syncLock = new(); + /// + /// Initializes a new instance of the class. + /// + /// The GPT translation service to use. + /// The source language code. + /// The target language code. + /// The maximum number of tokens allowed. (Optional) + /// The model to use for translation. (Optional) + /// The creative temperature. (Optional) + /// The user ID. (Optional) + /// The timespan for sending requests after inactivity. (Optional) + /// The maximum tokens per request. (Optional) public ChatGPTTranslatorServiceEconomical( - ChatGPTTranslatorService chatGptTranslatorService, + IChatGPTTranslatorService chatGptTranslatorService, string sourceLanguage, string targetLanguage, int? maxTokens = null, @@ -54,11 +69,11 @@ public async ValueTask DisposeAsync() { lock (_syncLock) { - if (_buffer.Version > 0) + if (_batch.Version > 0) { if (CanBeRan(_tcs)) { - _ = SendRequestAndResetBuffer(_buffer, _tcs); + _ = SendRequestAndResetBatch(_batch, _tcs); } } } @@ -68,19 +83,22 @@ public async ValueTask DisposeAsync() } } + /// + /// Translates the given text. + /// + /// The text to translate. + /// A task representing the translated text. public async Task TranslateText(string text) { ArgumentNullException.ThrowIfNull(text); - int index; - - TaskCompletionSource tcs; + TaskCompletionSource tcs; lock (_syncLock) { - var buffer = GetBuffer(); + var batch = GetRelevantBatch(); tcs = _tcs; - index = buffer.Add(text); - _ = RunRequestSendingInactivityTimer(tcs, buffer); + index = batch.Add(text); + _ = RunRequestSendingInactivityTimer(tcs, batch); } var result = await tcs.Task; @@ -88,31 +106,31 @@ public async Task TranslateText(string text) } // Debouncer - private async Task RunRequestSendingInactivityTimer(TaskCompletionSource tcs, Buffer buffer) + private async Task RunRequestSendingInactivityTimer(TaskCompletionSource tcs, Batch batch) { await Task.Delay(_sendRequestAfterInactivity); lock (_syncLock) { - var isBufferRelevant = ReferenceEquals(_buffer, buffer) && buffer.Version == _buffer.Version; - if (isBufferRelevant && CanBeRan(tcs)) + var isBatchRelevant = ReferenceEquals(_batch, batch) && batch.Version == _batch.Version; + if (isBatchRelevant && CanBeRan(tcs)) { - _ = SendRequestAndResetBuffer(buffer, tcs); + _ = SendRequestAndResetBatch(batch, tcs); } } } - private Buffer GetBuffer() + private Batch GetRelevantBatch() { var tcs = _tcs; - if (_buffer.GetTotalLength() > _maxTokensPerRequest) + if (_batch.GetTotalLength() > _maxTokensPerRequest) { - _ = SendRequestAndResetBuffer(_buffer, tcs); + _ = SendRequestAndResetBatch(_batch, tcs); } - return _buffer; + return _batch; } - private async Task SendRequestAndResetBuffer(Buffer buffer, TaskCompletionSource tcs) + private async Task SendRequestAndResetBatch(Batch batch, TaskCompletionSource tcs) { if (!CanBeRan(tcs)) { @@ -122,13 +140,13 @@ private async Task SendRequestAndResetBuffer(Buffer buffer, TaskCompletionSource lock (_syncLock) { _tcs = new(); - _buffer = new(); + _batch = new(); } try { var response = await _chatGptTranslatorService.TranslateObject( - buffer, true, _sourceLanguage, _targetLanguage, _maxTokens, _model, _temperature, _user); + batch, true, _sourceLanguage, _targetLanguage, _maxTokens, _model, _temperature, _user); tcs.SetResult(response); } @@ -138,12 +156,12 @@ private async Task SendRequestAndResetBuffer(Buffer buffer, TaskCompletionSource } } - private static bool CanBeRan(TaskCompletionSource tcs) + private static bool CanBeRan(TaskCompletionSource tcs) { return tcs.Task.Status is TaskStatus.Created or TaskStatus.WaitingForActivation; } - private class Buffer + internal class Batch { public IReadOnlyDictionary Texts => _texts; @@ -154,12 +172,12 @@ private class Buffer private int _indexer; [JsonConstructor] - public Buffer(IReadOnlyDictionary texts) + public Batch(IReadOnlyDictionary texts) { _texts = new Dictionary(texts) ?? throw new ArgumentNullException(nameof(texts)); } - public Buffer() + public Batch() { _texts = new(); } diff --git a/src/modules/OpenAI.ChatGpt.Modules.Translator/IChatGPTTranslatorService.cs b/src/modules/OpenAI.ChatGpt.Modules.Translator/IChatGPTTranslatorService.cs new file mode 100644 index 0000000..3d82b09 --- /dev/null +++ b/src/modules/OpenAI.ChatGpt.Modules.Translator/IChatGPTTranslatorService.cs @@ -0,0 +1,34 @@ +using System.Text.Json; +using OpenAI.ChatGpt.Models.ChatCompletion; + +namespace OpenAI.ChatGpt.Modules.Translator; + +public interface IChatGPTTranslatorService +{ + Task TranslateText( + string text, + string? sourceLanguage = null, + string? targetLanguage = null, + int? maxTokens = null, + string? model = null, + float temperature = ChatCompletionTemperatures.Default, + string? user = null, + Action? requestModifier = null, + Action? rawResponseGetter = null, + CancellationToken cancellationToken = default); + + Task TranslateObject( + TObject objectToTranslate, + bool isBatch = false, + string? sourceLanguage = null, + string? targetLanguage = null, + int? maxTokens = null, + string? model = null, + float temperature = ChatCompletionTemperatures.Default, + string? user = null, + Action? requestModifier = null, + Action? rawResponseGetter = null, + JsonSerializerOptions? jsonSerializerOptions = null, + JsonSerializerOptions? jsonDeserializerOptions = null, + CancellationToken cancellationToken = default) where TObject : class; +} \ No newline at end of file diff --git a/tests/OpenAI.ChatGpt.UnitTests/ChatGptTranslatorServiceTests.cs b/tests/OpenAI.ChatGpt.UnitTests/ChatGptTranslatorServiceTests.cs index 555cfc8..b9ff6b0 100644 --- a/tests/OpenAI.ChatGpt.UnitTests/ChatGptTranslatorServiceTests.cs +++ b/tests/OpenAI.ChatGpt.UnitTests/ChatGptTranslatorServiceTests.cs @@ -69,5 +69,90 @@ public async Task Translate_without_source_and_target_languages_uses_default_lan Times.Once); } - + [Fact] + public async Task Translating_multiple_texts_should_be_batched_together() + { + // Arrange + var mockTranslatorService = new Mock(); + var batch = new ChatGPTTranslatorServiceEconomical.Batch(); + batch.Add("Text 1"); + batch.Add("Text 2"); + + mockTranslatorService.Setup(m + => m.TranslateObject(It.IsAny(), + true, + "en", + "fr", + It.IsAny(), + It.IsAny(), + It.IsAny(), + null, + default, + default, + default, + default, + default) + ).ReturnsAsync(batch) + .Verifiable(); + + var service = new ChatGPTTranslatorServiceEconomical( + mockTranslatorService.Object, "en", "fr", maxTokensPerRequest: 50 + ); + + // Act + var result1 = service.TranslateText("Text 1"); + var result2 = service.TranslateText("Text 2"); + + // Assert + await Task.WhenAll(result1, result2); + mockTranslatorService.Verify(m + => m.TranslateObject(It.IsAny(), + true, + "en", + "fr", + It.IsAny(), + It.IsAny(), + It.IsAny(), + null, + default, + default, + default, + default, + default), + Times.Once); + } + + [Fact] + public async Task Batch_is_processed_after_inactivity_period() + { + var mockTranslatorService = new Mock(); + + var service = new ChatGPTTranslatorServiceEconomical( + mockTranslatorService.Object, + "en", + "fr", + sendRequestAfterInactivity: TimeSpan.FromMilliseconds(100) + ); + + _ = service.TranslateText("Hello"); + + await Task.Delay(150); // Wait for the inactivity period + + mockTranslatorService.Verify(x => + x.TranslateObject(It.IsAny(), + true, + "en", + "fr", + It.IsAny(), + It.IsAny(), + It.IsAny(), + null, + default, + default, + default, + default, + default), + Times.Once); + } + } \ No newline at end of file