diff --git a/src/core.ts b/src/core.ts index 592f014..c2814ba 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,4 +1,5 @@ import { VERSION } from './version'; +import { Stream } from './lib/streaming'; import { GroqError, APIError, @@ -38,6 +39,19 @@ type APIResponseProps = { async function defaultParseResponse(props: APIResponseProps): Promise { const { response } = props; + if (props.options.stream) { + debug('response', response.status, response.url, response.headers, response.body); + + // Note: there is an invariant here that isn't represented in the type system + // that if you set `stream: true` the response type must also be `Stream` + + if (props.options.__streamClass) { + return props.options.__streamClass.fromSSEResponse(response, props.controller) as any; + } + + return Stream.fromSSEResponse(response, props.controller) as any; + } + // fetch refuses to read the body when the status code is 204. if (response.status === 204) { return null as T; @@ -736,6 +750,7 @@ export type RequestOptions | Readable> = idempotencyKey?: string; __binaryResponse?: boolean | undefined; + __streamClass?: typeof Stream; }; // This is required so that we can determine if a given object matches the RequestOptions @@ -756,6 +771,7 @@ const requestOptionsKeys: KeysEnum = { idempotencyKey: true, __binaryResponse: true, + __streamClass: true, }; export const isRequestOptions = (obj: unknown): obj is RequestOptions => { diff --git a/src/lib/chat_completions_ext.ts b/src/lib/chat_completions_ext.ts deleted file mode 100644 index 01af661..0000000 --- a/src/lib/chat_completions_ext.ts +++ /dev/null @@ -1,98 +0,0 @@ -// Manually curated models for streaming chat completions. -import { ChatCompletion } from 'groq-sdk/resources/chat/index'; - -export interface ChatCompletionChunk { - id: string; - - choices: Array; - - created: number; - - model: string; - - object: 'chat.completion.chunk'; - - system_fingerprint?: string; - - x_groq?: ChatCompletionChunk.XGroq; -} - -export namespace ChatCompletionChunk { - export interface Choice { - delta: Choice.Delta; - - finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | 'function_call' | null; - - index: number; - - logprobs?: Choice.Logprobs | null; - } - - export namespace Choice { - export interface Delta { - content?: string | null; - - function_call?: Delta.FunctionCall; - - role?: 'system' | 'user' | 'assistant' | 'tool'; - - tool_calls?: Array; - } - - export namespace Delta { - export interface FunctionCall { - arguments?: string; - - name?: string; - } - - export interface ToolCall { - index: number; - - id?: string; - - function?: ToolCall.Function; - - type?: 'function'; - } - - export namespace ToolCall { - export interface Function { - arguments?: string; - - name?: string; - } - } - } - - export interface Logprobs { - content: Array | null; - } - } - - export type XGroq = { - id?: string; - usage?: ChatCompletion.Usage; - error?: string; - }; -} - -export interface ChatCompletionTokenLogprob { - token: string; - - bytes: Array | null; - - logprob: number; - - top_logprobs: Array; -} - -export namespace ChatCompletionTokenLogprob { - export interface TopLogprob { - token: string; - - bytes: Array | null; - - logprob: number; - } -} diff --git a/src/lib/streaming.ts b/src/lib/streaming.ts index ed760b5..7c9b17d 100644 --- a/src/lib/streaming.ts +++ b/src/lib/streaming.ts @@ -1,7 +1,7 @@ -import { ReadableStream, type Response } from 'groq-sdk/_shims/index'; -import { GroqError } from 'groq-sdk/error'; +import { ReadableStream, type Response } from '../_shims/index'; +import { GroqError } from '../error'; -import { APIError } from 'groq-sdk/error'; +import { APIError } from '../error'; type Bytes = string | ArrayBuffer | Uint8Array | Buffer | null | undefined; diff --git a/src/resources/chat/completions.ts b/src/resources/chat/completions.ts index 8507e35..bbc6d09 100644 --- a/src/resources/chat/completions.ts +++ b/src/resources/chat/completions.ts @@ -5,13 +5,33 @@ import { APIResource } from '../../resource'; import * as ChatCompletionsAPI from './completions'; import * as CompletionsAPI from '../completions'; import * as Shared from '../shared'; +import { Stream } from '../../lib/streaming'; export class Completions extends APIResource { /** * Creates a model response for the given chat conversation. */ - create(body: CompletionCreateParams, options?: Core.RequestOptions): Core.APIPromise { - return this._client.post('/openai/v1/chat/completions', { body, ...options }); + create( + body: ChatCompletionCreateParamsNonStreaming, + options?: Core.RequestOptions, + ): Core.APIPromise; + create( + body: ChatCompletionCreateParamsStreaming, + options?: Core.RequestOptions, + ): Core.APIPromise>; + create( + body: ChatCompletionCreateParamsBase, + options?: Core.RequestOptions, + ): Core.APIPromise | ChatCompletion>; + create( + body: ChatCompletionCreateParams, + options?: Core.RequestOptions, + ): Core.APIPromise | Core.APIPromise> { + return this._client.post('/openai/v1/chat/completions', { + body, + ...options, + stream: body.stream ?? false, + }) as Core.APIPromise | Core.APIPromise>; } } @@ -205,7 +225,7 @@ export namespace ChatCompletionChunk { * number of tokens specified in the request was reached, `tool_calls` if the model * called a tool, or `function_call` (deprecated) if the model called a function. */ - finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | 'function_call' | null; + finish_reason: 'stop' | 'length' | 'tool_calls' | 'function_call' | null; /** * The index of the choice in the list of choices. @@ -640,7 +660,11 @@ export interface ChatCompletionUserMessageParam { name?: string; } -export interface CompletionCreateParams { +export type ChatCompletionCreateParams = + | ChatCompletionCreateParamsNonStreaming + | ChatCompletionCreateParamsStreaming; + +export interface ChatCompletionCreateParamsBase { /** * A list of messages comprising the conversation so far. */ @@ -843,6 +867,26 @@ export namespace CompletionCreateParams { } } +export interface ChatCompletionCreateParamsNonStreaming extends ChatCompletionCreateParamsBase { + /** + * If set, partial message deltas will be sent. Tokens will be sent as data-only + * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) + * as they become available, with the stream terminated by a `data: [DONE]` + * message. [Example code](/docs/text-chat#streaming-a-chat-completion). + */ + stream?: false | null; +} + +export interface ChatCompletionCreateParamsStreaming extends ChatCompletionCreateParamsBase { + /** + * If set, partial message deltas will be sent. Tokens will be sent as data-only + * [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) + * as they become available, with the stream terminated by a `data: [DONE]` + * message. [Example code](/docs/text-chat#streaming-a-chat-completion). + */ + stream: true; +} + export namespace Completions { export import ChatCompletion = ChatCompletionsAPI.ChatCompletion; export import ChatCompletionAssistantMessageParam = ChatCompletionsAPI.ChatCompletionAssistantMessageParam;