Refactor ChatGPT translator to support economical batching

This commit involves major refactoring of the `ChatGPTTranslatorServiceEconomical.cs` file. This change replaces single translation process with a more economical batching. The primary element Buffer has been replaced by Batch and changed the usage accordingly. This optimization allows multitudes of text to be translated in one session, thus saving resources and improving performance.

Additional changes include the implementation of a new interface `IChatGPTTranslatorService` for translation services and some changes to method signatures.
Tests were updated to imitate a batch translation scenario to ensure the successful functionality of the updates.

src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorService.cs

@@ -8,7 +8,7 @@ namespace OpenAI.ChatGpt.Modules.Translator;
 
 [Fody.ConfigureAwait(false)]
 // ReSharper disable once InconsistentNaming
-public class ChatGPTTranslatorService : IDisposable
+public class ChatGPTTranslatorService : IDisposable, IChatGPTTranslatorService
 {
     private readonly IOpenAiClient _client;
     private readonly string? _defaultSourceLanguage;

src/modules/OpenAI.ChatGpt.Modules.Translator/ChatGPTTranslatorServiceEconomical.cs

@@ -3,9 +3,12 @@
 
 namespace OpenAI.ChatGpt.Modules.Translator;
 
+/// <summary>
+/// Provides a service for translating text using GPT models with economical batching.
+/// </summary>
 public class ChatGPTTranslatorServiceEconomical : IAsyncDisposable
 {
-    private readonly ChatGPTTranslatorService _chatGptTranslatorService;
+    private readonly IChatGPTTranslatorService _chatGptTranslatorService;
     private readonly string _sourceLanguage;
     private readonly string _targetLanguage;
     private readonly int? _maxTokens;
@@ -16,12 +19,24 @@ public class ChatGPTTranslatorServiceEconomical : IAsyncDisposable
     private readonly TimeSpan _sendRequestAfterInactivity;
     private static readonly TimeSpan DefaultSendRequestAfterInactivity = TimeSpan.FromMilliseconds(100);
 
-    private Buffer _buffer = new();
-    private TaskCompletionSource<Buffer> _tcs = new();
+    private Batch _batch = new();
+    private TaskCompletionSource<Batch> _tcs = new();
     private readonly object _syncLock = new();
 
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ChatGPTTranslatorServiceEconomical"/> class.
+    /// </summary>
+    /// <param name="chatGptTranslatorService">The GPT translation service to use.</param>
+    /// <param name="sourceLanguage">The source language code.</param>
+    /// <param name="targetLanguage">The target language code.</param>
+    /// <param name="maxTokens">The maximum number of tokens allowed. (Optional)</param>
+    /// <param name="model">The model to use for translation. (Optional)</param>
+    /// <param name="temperature">The creative temperature. (Optional)</param>
+    /// <param name="user">The user ID. (Optional)</param>
+    /// <param name="sendRequestAfterInactivity">The timespan for sending requests after inactivity. (Optional)</param>
+    /// <param name="maxTokensPerRequest">The maximum tokens per request. (Optional)</param>
     public ChatGPTTranslatorServiceEconomical(
-        ChatGPTTranslatorService chatGptTranslatorService,
+        IChatGPTTranslatorService chatGptTranslatorService,
         string sourceLanguage,
         string targetLanguage,
         int? maxTokens = null,
@@ -54,11 +69,11 @@ public async ValueTask DisposeAsync()
     {
         lock (_syncLock)
         {
-            if (_buffer.Version > 0)
+            if (_batch.Version > 0)
             {
                 if (CanBeRan(_tcs))
                 {
-                    _ = SendRequestAndResetBuffer(_buffer, _tcs);
+                    _ = SendRequestAndResetBatch(_batch, _tcs);
                 }
             }
         }
@@ -68,51 +83,54 @@ public async ValueTask DisposeAsync()
         }
     }
 
+    /// <summary>
+    /// Translates the given text.
+    /// </summary>
+    /// <param name="text">The text to translate.</param>
+    /// <returns>A task representing the translated text.</returns>
     public async Task<string> TranslateText(string text)
     {
         ArgumentNullException.ThrowIfNull(text);
-
         int index;
-
-        TaskCompletionSource<Buffer> tcs;
+        TaskCompletionSource<Batch> tcs;
         lock (_syncLock)
         {
-            var buffer = GetBuffer();
+            var batch = GetRelevantBatch();
             tcs = _tcs;
-            index = buffer.Add(text);
-            _ = RunRequestSendingInactivityTimer(tcs, buffer);
+            index = batch.Add(text);
+            _ = RunRequestSendingInactivityTimer(tcs, batch);
         }
 
         var result = await tcs.Task;
         return result.Texts[index];
     }
 
     // Debouncer
-    private async Task RunRequestSendingInactivityTimer(TaskCompletionSource<Buffer> tcs, Buffer buffer)
+    private async Task RunRequestSendingInactivityTimer(TaskCompletionSource<Batch> tcs, Batch batch)
     {
         await Task.Delay(_sendRequestAfterInactivity);
         lock (_syncLock)
         {
-            var isBufferRelevant = ReferenceEquals(_buffer, buffer) && buffer.Version == _buffer.Version;
-            if (isBufferRelevant && CanBeRan(tcs))
+            var isBatchRelevant = ReferenceEquals(_batch, batch) && batch.Version == _batch.Version;
+            if (isBatchRelevant && CanBeRan(tcs))
             {
-                _ = SendRequestAndResetBuffer(buffer, tcs);
+                _ = SendRequestAndResetBatch(batch, tcs);
             }
         }
     }
 
-    private Buffer GetBuffer()
+    private Batch GetRelevantBatch()
     {
         var tcs = _tcs;
-        if (_buffer.GetTotalLength() > _maxTokensPerRequest)
+        if (_batch.GetTotalLength() > _maxTokensPerRequest)
         {
-            _ = SendRequestAndResetBuffer(_buffer, tcs);
+            _ = SendRequestAndResetBatch(_batch, tcs);
         }
 
-        return _buffer;
+        return _batch;
     }
 
-    private async Task SendRequestAndResetBuffer(Buffer buffer, TaskCompletionSource<Buffer> tcs)
+    private async Task SendRequestAndResetBatch(Batch batch, TaskCompletionSource<Batch> tcs)
     {
         if (!CanBeRan(tcs))
         {
@@ -122,13 +140,13 @@ private async Task SendRequestAndResetBuffer(Buffer buffer, TaskCompletionSource
         lock (_syncLock)
         {
             _tcs = new();
-            _buffer = new();
+            _batch = new();
         }
 
         try
         {
             var response = await _chatGptTranslatorService.TranslateObject(
-                buffer, true, _sourceLanguage, _targetLanguage, _maxTokens, _model, _temperature, _user);
+                batch, true, _sourceLanguage, _targetLanguage, _maxTokens, _model, _temperature, _user);
 
             tcs.SetResult(response);
         }
@@ -138,12 +156,12 @@ private async Task SendRequestAndResetBuffer(Buffer buffer, TaskCompletionSource
         }
     }
 
-    private static bool CanBeRan(TaskCompletionSource<Buffer> tcs)
+    private static bool CanBeRan(TaskCompletionSource<Batch> tcs)
     {
         return tcs.Task.Status is TaskStatus.Created or TaskStatus.WaitingForActivation;
     }
 
-    private class Buffer
+    internal class Batch
     {
         public IReadOnlyDictionary<int, string> Texts => _texts;
 
@@ -154,12 +172,12 @@ private class Buffer
         private int _indexer;
 
         [JsonConstructor]
-        public Buffer(IReadOnlyDictionary<int, string> texts)
+        public Batch(IReadOnlyDictionary<int, string> texts)
         {
             _texts = new Dictionary<int, string>(texts) ?? throw new ArgumentNullException(nameof(texts));
         }
 
-        public Buffer()
+        public Batch()
         {
             _texts = new();
         }

src/modules/OpenAI.ChatGpt.Modules.Translator/IChatGPTTranslatorService.cs

@@ -0,0 +1,34 @@
+using System.Text.Json;
+using OpenAI.ChatGpt.Models.ChatCompletion;
+
+namespace OpenAI.ChatGpt.Modules.Translator;
+
+public interface IChatGPTTranslatorService
+{
+    Task<string> TranslateText(
+        string text,
+        string? sourceLanguage = null,
+        string? targetLanguage = null,
+        int? maxTokens = null,
+        string? model = null,
+        float temperature = ChatCompletionTemperatures.Default,
+        string? user = null,
+        Action<ChatCompletionRequest>? requestModifier = null,
+        Action<ChatCompletionResponse>? rawResponseGetter = null,
+        CancellationToken cancellationToken = default);
+
+    Task<TObject> TranslateObject<TObject>(
+        TObject objectToTranslate,
+        bool isBatch = false,
+        string? sourceLanguage = null,
+        string? targetLanguage = null,
+        int? maxTokens = null,
+        string? model = null,
+        float temperature = ChatCompletionTemperatures.Default,
+        string? user = null,
+        Action<ChatCompletionRequest>? requestModifier = null,
+        Action<ChatCompletionResponse>? rawResponseGetter = null,
+        JsonSerializerOptions? jsonSerializerOptions = null,
+        JsonSerializerOptions? jsonDeserializerOptions = null,
+        CancellationToken cancellationToken = default) where TObject : class;
+}

tests/OpenAI.ChatGpt.UnitTests/ChatGptTranslatorServiceTests.cs

@@ -69,5 +69,90 @@ public async Task Translate_without_source_and_target_languages_uses_default_lan
             Times.Once);
     }
 
-
+    [Fact]
+    public async Task Translating_multiple_texts_should_be_batched_together()
+    {
+        // Arrange
+        var mockTranslatorService = new Mock<IChatGPTTranslatorService>();
+        var batch = new ChatGPTTranslatorServiceEconomical.Batch();
+        batch.Add("Text 1");
+        batch.Add("Text 2");
+
+        mockTranslatorService.Setup(m 
+                => m.TranslateObject(It.IsAny<ChatGPTTranslatorServiceEconomical.Batch>(),
+                    true,
+                    "en",
+                    "fr",
+                    It.IsAny<int?>(),
+                    It.IsAny<string>(),
+                    It.IsAny<float>(),
+                    null,
+                    default,
+                    default,
+                    default,
+                    default,
+                    default)
+            ).ReturnsAsync(batch)
+            .Verifiable();
+
+        var service = new ChatGPTTranslatorServiceEconomical(
+            mockTranslatorService.Object, "en", "fr", maxTokensPerRequest: 50
+        );
+
+        // Act
+        var result1 = service.TranslateText("Text 1");
+        var result2 = service.TranslateText("Text 2");
+
+        // Assert
+        await Task.WhenAll(result1, result2);
+        mockTranslatorService.Verify(m 
+                => m.TranslateObject(It.IsAny<ChatGPTTranslatorServiceEconomical.Batch>(),
+                    true,
+                    "en",
+                    "fr",
+                    It.IsAny<int?>(),
+                    It.IsAny<string>(),
+                    It.IsAny<float>(),
+                    null,
+                    default,
+                    default,
+                    default,
+                    default,
+                    default),
+            Times.Once);
+    }
+
+    [Fact]
+    public async Task Batch_is_processed_after_inactivity_period()
+    {
+        var mockTranslatorService = new Mock<IChatGPTTranslatorService>();
+
+        var service = new ChatGPTTranslatorServiceEconomical(
+            mockTranslatorService.Object,
+            "en",
+            "fr",
+            sendRequestAfterInactivity: TimeSpan.FromMilliseconds(100)
+        );
+
+        _ = service.TranslateText("Hello");
+
+        await Task.Delay(150); // Wait for the inactivity period
+
+        mockTranslatorService.Verify(x => 
+            x.TranslateObject(It.IsAny<ChatGPTTranslatorServiceEconomical.Batch>(),
+                true,
+                "en",
+                "fr",
+                It.IsAny<int?>(),
+                It.IsAny<string>(),
+                It.IsAny<float>(),
+                null,
+                default,
+                default,
+                default,
+                default,
+                default),
+            Times.Once);
+    }
+
 }