Add support for stream_options (#161)

marcominerva · Sep 5, 2024 · ebdc1e6 · ebdc1e6
2 parents ffe9b12 + 8df90d3
commit ebdc1e6
Show file tree

Hide file tree

Showing 17 changed files with 95 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -61,7 +61,8 @@ builder.Services.AddChatGpt(options =>
   - 2024-04-01-preview
   - 2024-05-01-preview
   - 2024-02-01
-  - 2024-06-01 (default)
+  - 2024-06-01
+  - 2024-07-01-preview (default)
 - _AuthenticationType_: it specifies if the key is an actual API Key or an [Azure Active Directory token](https://learn.microsoft.com/azure/cognitive-services/openai/how-to/managed-identity) (optional, default: "ApiKey").
 
 ### DefaultModel and DefaultEmbeddingModel

diff --git a/samples/ChatGptApi/ChatGptApi.csproj b/samples/ChatGptApi/ChatGptApi.csproj
@@ -7,10 +7,10 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.7" />
-        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
-        <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
-        <PackageReference Include="TinyHelpers.AspNetCore" Version="3.1.6" />
+        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.8" />
+        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.9.0" />
+        <PackageReference Include="Swashbuckle.AspNetCore" Version="6.7.3" />
+        <PackageReference Include="TinyHelpers.AspNetCore" Version="3.1.9" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/samples/ChatGptApi/appsettings.json b/samples/ChatGptApi/appsettings.json
@@ -4,7 +4,7 @@
         "ApiKey": "", // Required
         //"Organization": "", // Optional, used only by OpenAI
         "ResourceName": "", // Required when using Azure OpenAI Service
-        "ApiVersion": "2024-06-01", // Optional, used only by Azure OpenAI Service (default: 2024-06-01)
+        "ApiVersion": "2024-07-01-preview", // Optional, used only by Azure OpenAI Service (default: 2024-07-01-preview)
         "AuthenticationType": "ApiKey", // Optional, used only by Azure OpenAI Service. Allowed values: ApiKey (default) or ActiveDirectory
 
         "DefaultModel": "my-model",

diff --git a/samples/ChatGptBlazor.Wasm/ChatGptBlazor.Wasm.csproj b/samples/ChatGptBlazor.Wasm/ChatGptBlazor.Wasm.csproj
@@ -9,9 +9,9 @@
 
     <ItemGroup>
         <PackageReference Include="Markdig" Version="0.37.0" />
-        <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly" Version="8.0.7" />
-        <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="8.0.7" PrivateAssets="all" />
-        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
+        <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly" Version="8.0.8" />
+        <PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.DevServer" Version="8.0.8" PrivateAssets="all" />
+        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.9.0" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/samples/ChatGptConsole/ChatGptConsole.csproj b/samples/ChatGptConsole/ChatGptConsole.csproj
@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
+        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.9.0" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/samples/ChatGptConsole/appsettings.json b/samples/ChatGptConsole/appsettings.json
@@ -4,7 +4,7 @@
         "ApiKey": "", // Required
         //"Organization": "", // Optional, used only by OpenAI
         "ResourceName": "", // Required when using Azure OpenAI Service
-        "ApiVersion": "2024-06-01", // Optional, used only by Azure OpenAI Service (default: 2024-06-01)
+        "ApiVersion": "2024-07-01-preview", // Optional, used only by Azure OpenAI Service (default: 2024-07-01-preview)
         "AuthenticationType": "ApiKey", // Optional, used only by Azure OpenAI Service. Allowed values: ApiKey (default) or ActiveDirectory
 
         "DefaultModel": "my-model",

diff --git a/samples/ChatGptFunctionCallingConsole/ChatGptFunctionCallingConsole.csproj b/samples/ChatGptFunctionCallingConsole/ChatGptFunctionCallingConsole.csproj
@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
+        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.9.0" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/samples/ChatGptFunctionCallingConsole/appsettings.json b/samples/ChatGptFunctionCallingConsole/appsettings.json
@@ -4,7 +4,7 @@
         "ApiKey": "", // Required
         //"Organization": "", // Optional, used only by OpenAI
         "ResourceName": "", // Required when using Azure OpenAI Service
-        "ApiVersion": "2024-06-01", // Optional, used only by Azure OpenAI Service (default: 2024-06-01)
+        "ApiVersion": "2024-07-01-preview", // Optional, used only by Azure OpenAI Service (default: 2024-07-01-preview)
         "AuthenticationType": "ApiKey", // Optional, used only by Azure OpenAI Service. Allowed values: ApiKey (default) or ActiveDirectory
 
         "DefaultModel": "my-model",

diff --git a/samples/ChatGptStreamConsole/ChatGptStreamConsole.csproj b/samples/ChatGptStreamConsole/ChatGptStreamConsole.csproj
@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.7.0" />
+        <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.9.0" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/samples/ChatGptStreamConsole/appsettings.json b/samples/ChatGptStreamConsole/appsettings.json
@@ -1,35 +1,35 @@
-{
-    "ChatGPT": {
-        "Provider": "OpenAI", // Optional. Allowed values: OpenAI (default) or Azure
-        "ApiKey": "", // Required
-        //"Organization": "", // Optional, used only by OpenAI
-        "ResourceName": "", // Required when using Azure OpenAI Service
-        "ApiVersion": "2024-06-01", // Optional, used only by Azure OpenAI Service (default: 2024-06-01)
-        "AuthenticationType": "ApiKey", // Optional, used only by Azure OpenAI Service. Allowed values: ApiKey (default) or ActiveDirectory
-
-        "DefaultModel": "my-model",
-        "DefaultEmbeddingModel": "text-embedding-ada-002", // Optional, set it if you want to use embeddings
-        "MessageLimit": 20,
-        "MessageExpiration": "00:30:00",
-        "ThrowExceptionOnError": true // Optional, default: true
-        //"User": "UserName",
-        //"DefaultParameters": {
-        //    "Temperature": 0.8,
-        //    "TopP": 1,
-        //    "MaxTokens": 500,
-        //    "PresencePenalty": 0,
-        //    "FrequencyPenalty": 0,
-        //    "ResponseFormat": { "Type": "text" }, // Allowed values for Type: text (default) or json_object
-        //    "Seed": 42                            // Optional (any integer value)
-        //},
-        //"DefaultEmbeddingParameters": {
-        //    "Dimensions": 1536
-        //}
-    },
-    "Logging": {
-        "LogLevel": {
-            "Default": "Error",
-            "Microsoft": "Warning"
-        }
-    }
-}
+{
+    "ChatGPT": {
+        "Provider": "OpenAI", // Optional. Allowed values: OpenAI (default) or Azure
+        "ApiKey": "", // Required
+        //"Organization": "", // Optional, used only by OpenAI
+        "ResourceName": "", // Required when using Azure OpenAI Service
+        "ApiVersion": "2024-07-01-preview", // Optional, used only by Azure OpenAI Service (default: 2024-07-01-preview)
+        "AuthenticationType": "ApiKey", // Optional, used only by Azure OpenAI Service. Allowed values: ApiKey (default) or ActiveDirectory
+
+        "DefaultModel": "my-model",
+        "DefaultEmbeddingModel": "text-embedding-ada-002", // Optional, set it if you want to use embeddings
+        "MessageLimit": 20,
+        "MessageExpiration": "00:30:00",
+        "ThrowExceptionOnError": true // Optional, default: true
+        //"User": "UserName",
+        //"DefaultParameters": {
+        //    "Temperature": 0.8,
+        //    "TopP": 1,
+        //    "MaxTokens": 500,
+        //    "PresencePenalty": 0,
+        //    "FrequencyPenalty": 0,
+        //    "ResponseFormat": { "Type": "text" }, // Allowed values for Type: text (default) or json_object
+        //    "Seed": 42                            // Optional (any integer value)
+        //},
+        //"DefaultEmbeddingParameters": {
+        //    "Dimensions": 1536
+        //}
+    },
+    "Logging": {
+        "LogLevel": {
+            "Default": "Error",
+            "Microsoft": "Warning"
+        }
+    }
+}
diff --git a/src/ChatGptNet/ChatGptClient.cs b/src/ChatGptNet/ChatGptClient.cs
@@ -158,6 +158,12 @@ public async IAsyncEnumerable<ChatGptResponse> AskStreamAsync(Guid conversationI
                                 }
                             }
                         }
+                        else if (response.Usage is not null)
+                        {
+                            // The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array.
+                            // All other chunks will also include a usage field, but with a null value.
+                            yield return response;
+                        }
                     }
                     else if (line.StartsWith("data: [DONE]"))
                     {
@@ -350,6 +356,8 @@ private ChatGptRequest CreateChatGptRequest(IEnumerable<ChatGptMessage> messages
             },
 
             Stream = stream,
+            StreamOptions = stream ? new() { IncludeUsage = true } : null,
+
             Seed = parameters?.Seed ?? options.DefaultParameters.Seed,
             Temperature = parameters?.Temperature ?? options.DefaultParameters.Temperature,
             TopP = parameters?.TopP ?? options.DefaultParameters.TopP,

diff --git a/src/ChatGptNet/Models/ChatGptRequest.cs b/src/ChatGptNet/Models/ChatGptRequest.cs
@@ -3,7 +3,7 @@
 namespace ChatGptNet.Models;
 
 /// <summary>
-/// Represents a request for a chat completions.
+/// Represents a request for chat completions.
 /// </summary>
 /// <remarks>
 /// See <see href="https://platform.openai.com/docs/api-reference/chat/create">Create chat completion (OpenAI)</see> or <see href="https://learn.microsoft.com/azure/cognitive-services/openai/reference#chat-completions">Chat Completions (Azure)</see> for more information.
@@ -32,7 +32,7 @@ internal class ChatGptRequest
     /// Controls which (if any) function is called by the model.
     /// </summary>
     /// <remarks>
-    /// <list type = "bullet" >
+    /// <list type="bullet">
     ///   <item>
     ///     <term><see cref="ChatGptToolChoices.None"/></term>
     ///     <description>Model will not call a function and instead generates a message.</description>
@@ -46,7 +46,7 @@ internal class ChatGptRequest
     ///     <description>Specifying a particular function name forces the model to call that function.</description>
     ///   </item>
     /// </list>
-    /// <see cref="ChatGptToolChoices.None"/> is the default when no functions are present. <see cref="ChatGptToolChoices.None"/> is the default if functions are present.
+    /// <para><see cref="ChatGptToolChoices.None"/> is the default when no functions are present. <see cref="ChatGptToolChoices.None"/> is the default if functions are present.</para>
     /// </remarks>
     /// <seealso cref="ChatGptFunction"/>
     [JsonPropertyName("function_call")]
@@ -64,7 +64,7 @@ internal class ChatGptRequest
     /// Controls which (if any) function is called by the model.
     /// </summary>
     /// <remarks>
-    /// <list type = "bullet" >
+    /// <list type="bullet">
     ///   <item>
     ///     <term><see cref="ChatGptToolChoices.None"/></term>
     ///     <description>Model will not call a function and instead generates a message.</description>
@@ -78,17 +78,23 @@ internal class ChatGptRequest
     ///     <description>Specifying a particular function name forces the model to call that function.</description>
     ///   </item>
     /// </list>
-    /// <see cref="ChatGptToolChoices.None"/> is the default when no functions are present. <see cref="ChatGptToolChoices.None"/> is the default if functions are present.
+    /// <para><see cref="ChatGptToolChoices.None"/> is the default when no functions are present. <see cref="ChatGptToolChoices.None"/> is the default if functions are present.</para>
     /// </remarks>
     /// <seealso cref="ChatGptFunction"/>
     [JsonPropertyName("tool_choice")]
     public object? ToolChoice { get; set; }
 
     /// <summary>
-    /// Gets or sets a value that specify if response will be sent in streaming as partial message deltas.
+    /// Gets or sets a value that specifies if the response will be sent in streaming as partial message deltas.
     /// </summary>
     public bool Stream { get; set; }
 
+    /// <summary>
+    /// Gets or sets the stream options for chat completions.
+    /// </summary>
+    [JsonPropertyName("stream_options")]
+    public ChatGptStreamOptions? StreamOptions { get; set; }
+
     /// <summary>
     /// Gets or sets a value such that, if specified, the system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result.
     /// </summary>
@@ -99,10 +105,10 @@ internal class ChatGptRequest
     public int? Seed { get; set; }
 
     /// <summary>
-    /// Gets or sets what sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic (default: 1).
+    /// Gets or sets the sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic (default: 1).
     /// </summary>
     /// <remarks>
-    /// It is generally recommend altering this value or <see cref="TopP"/> but not both.
+    /// It is generally recommended to alter this value or <see cref="TopP"/> but not both.
     /// </remarks>
     /// <seealso cref="TopP"/>
     public double? Temperature { get; set; }
@@ -111,7 +117,7 @@ internal class ChatGptRequest
     /// Gets or sets an alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with <see cref="TopP"/> probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered (default: 1).
     /// </summary>
     /// <remarks>
-    /// It is generally recommend altering this value or <see cref="Temperature"/> but not both.
+    /// It is generally recommended to alter this value or <see cref="Temperature"/> but not both.
     /// </remarks>
     /// <seealso cref="Temperature"/>
     [JsonPropertyName("top_p")]
@@ -124,7 +130,7 @@ internal class ChatGptRequest
     public int? MaxTokens { get; set; }
 
     /// <summary>
-    /// Gets or sets the presence penalties for chat completion. Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics (default: 0).
+    /// Gets or sets the presence penalties for chat completion. A number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics (default: 0).
     /// </summary>
     /// <remarks>
     /// See <see href="https://platform.openai.com/docs/api-reference/parameter-details">Parameter details</see> for more information.
@@ -133,7 +139,7 @@ internal class ChatGptRequest
     public double? PresencePenalty { get; set; }
 
     /// <summary>
-    /// Gets or sets the frequency penalties for chat completion. Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim (default: 0).
+    /// Gets or sets the frequency penalties for chat completion. A number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim (default: 0).
     /// </summary>
     /// <remarks>
     /// See <see href="https://platform.openai.com/docs/api-reference/parameter-details">Parameter details</see> for more information.
@@ -149,7 +155,7 @@ internal class ChatGptRequest
     public ChatGptResponseFormat? ResponseFormat { get; set; }
 
     /// <summary>
-    /// Gets or set a value that determines whether to return log probabilities of the output tokens or not. If <see langword="true"/>, returns the log probabilities of each output token returned in the content of message (default: <see langword="false"/>).
+    /// Gets or sets a value that determines whether to return log probabilities of the output tokens or not. If <see langword="true"/>, returns the log probabilities of each output token returned in the content of the message (default: <see langword="false"/>).
     /// </summary>
     /// <seealso cref="TopLogProbabilities"/>
     [JsonPropertyName("logprobs")]
@@ -159,7 +165,7 @@ internal class ChatGptRequest
     /// Gets or sets a value between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability.
     /// </summary>
     /// <remarks>
-    /// <see cref="LogProbabilities"/>must be set to <see langword="true"/> if this parameter is used.
+    /// <see cref="LogProbabilities"/> must be set to <see langword="true"/> if this parameter is used.
     /// </remarks>
     /// <seealso cref="LogProbabilities"/>
     [JsonPropertyName("top_logprobs")]
@@ -172,4 +178,4 @@ internal class ChatGptRequest
     /// See <see href="https://platform.openai.com/docs/guides/safety-best-practices/end-user-ids">Safety best practices</see> for more information.
     /// </remarks>
     public string? User { get; set; }
-}
+}
diff --git a/src/ChatGptNet/Models/ChatGptStreamOptions.cs b/src/ChatGptNet/Models/ChatGptStreamOptions.cs
@@ -0,0 +1,9 @@
+using System.Text.Json.Serialization;
+
+namespace ChatGptNet.Models;
+
+internal class ChatGptStreamOptions
+{
+    [JsonPropertyName("include_usage")]
+    public bool IncludeUsage { get; set; } = true;
+}
diff --git a/src/ChatGptNet/Models/Common/Response.cs b/src/ChatGptNet/Models/Common/Response.cs
@@ -20,9 +20,6 @@ public abstract class Response
     /// <summary>
     /// Gets or sets information about token usage.
     /// </summary>
-    /// <remarks>
-    /// The <see cref="Usage"/> property is always <see langword="null"/> when requesting response streaming with <see cref="ChatGptClient.AskStreamAsync(Guid, string, ChatGptParameters?, string?, bool, CancellationToken)"/>.
-    /// </remarks>
     public ChatGptUsage? Usage { get; set; }
 
     /// <summary>

diff --git a/src/ChatGptNet/ServiceConfigurations/AzureChatGptServiceConfiguration.cs b/src/ChatGptNet/ServiceConfigurations/AzureChatGptServiceConfiguration.cs
@@ -10,15 +10,15 @@ internal class AzureChatGptServiceConfiguration : ChatGptServiceConfiguration
     /// <summary>
     /// The default API version for Azure OpenAI service.
     /// </summary>
-    public const string DefaultApiVersion = "2024-06-01";
+    public const string DefaultApiVersion = "2024-07-01-preview";
 
     /// <summary>
     /// Gets or sets the name of the Azure OpenAI Resource.
     /// </summary>
     public string? ResourceName { get; set; }
 
     /// <summary>
-    /// Gets or sets the API version of the Azure OpenAI service (Default: 2024-06-01).
+    /// Gets or sets the API version of the Azure OpenAI service (Default: 2024-07-01-preview).
     /// </summary>
     /// <remarks>
     /// Currently supported versions are:
@@ -59,7 +59,10 @@ internal class AzureChatGptServiceConfiguration : ChatGptServiceConfiguration
     ///     <term>2024-06-01</term>
     ///     <description><see href="https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cognitiveservices/data-plane/AzureOpenAI/inference/stable/2024-06-01/inference.json">Swagger spec</see></description>
     ///   </item>
-    /// </list>
+    ///   <item>
+    ///     <term>2024-07-01-preview</term>
+    ///     <description><see href="https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cognitiveservices/data-plane/AzureOpenAI/inference/preview/2024-07-01-preview/inference.json">Swagger spec</see></description>
+    ///   </item>/// </list>
     /// </remarks>
     public string ApiVersion { get; set; } = DefaultApiVersion;
 

diff --git a/src/ChatGptNet/version.json b/src/ChatGptNet/version.json
@@ -1,6 +1,6 @@
 {
   "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json",
-  "version": "3.2",
+  "version": "3.3",
   "publicReleaseRefSpec": [
     "^refs/heads/master$" // we release out of master
   ],

diff --git a/src/Directory.Build.props b/src/Directory.Build.props
@@ -9,7 +9,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Nerdbank.GitVersioning" Version="3.6.139"  PrivateAssets="All" />
+        <PackageReference Include="Nerdbank.GitVersioning" Version="3.6.143"  PrivateAssets="All" />
     </ItemGroup>
 
 </Project>