diff --git a/docs/core_docs/docs/integrations/chat/anthropic.mdx b/docs/core_docs/docs/integrations/chat/anthropic.mdx index 8bbc5049be11..3b34c3c1b840 100644 --- a/docs/core_docs/docs/integrations/chat/anthropic.mdx +++ b/docs/core_docs/docs/integrations/chat/anthropic.mdx @@ -48,3 +48,37 @@ You can pass custom headers in your requests like this: import AnthropicCustomHeaders from "@examples/models/chat/integration_anthropic_custom_headers.ts"; {AnthropicCustomHeaders} + +## Tools + +The Anthropic API supports tool calling, along with multi-tool calling. The following examples demonstrate how to call tools: + +### Single Tool + +import AnthropicSingleTool from "@examples/models/chat/integration_anthropic_single_tool.ts"; + +{AnthropicSingleTool} + +:::tip +See the LangSmith trace [here](https://smith.langchain.com/public/90c03ed0-154b-4a50-afbf-83dcbf302647/r) +::: + +### Multi-Tool + +import AnthropicMultiTool from "@examples/models/chat/integration_anthropic_multi_tool.ts"; + +{AnthropicMultiTool} + +:::tip +See the LangSmith trace [here](https://smith.langchain.com/public/1349bb57-df1b-48b5-89c2-e6d5bd8f694a/r) +::: + +### `withStructuredOutput` + +import AnthropicWSA from "@examples/models/chat/integration_anthropic_wsa.ts"; + +{AnthropicWSA} + +:::tip +See the LangSmith trace [here](https://smith.langchain.com/public/efbd11c5-886e-4e07-be1a-951690fa8a27/r) +::: diff --git a/docs/core_docs/docs/integrations/chat/anthropic_tools.mdx b/docs/core_docs/docs/integrations/chat/anthropic_tools.mdx index 8d0ee4e6a42b..0188971ec339 100644 --- a/docs/core_docs/docs/integrations/chat/anthropic_tools.mdx +++ b/docs/core_docs/docs/integrations/chat/anthropic_tools.mdx @@ -1,7 +1,12 @@ --- sidebar_label: Anthropic Tools +sidebar_class_name: hidden --- +:::warning +This API is deprecated as Anthropic now officially supports tools. [Click here to read the documentation](/docs/integrations/chat/anthropic#tools). +::: + # Anthropic Tools LangChain offers an experimental wrapper around Anthropic that gives it the same API as OpenAI Functions. diff --git a/examples/src/models/chat/integration_anthropic_multi_tool.ts b/examples/src/models/chat/integration_anthropic_multi_tool.ts new file mode 100644 index 000000000000..1ad34e7b63d3 --- /dev/null +++ b/examples/src/models/chat/integration_anthropic_multi_tool.ts @@ -0,0 +1,100 @@ +import { ChatAnthropic } from "@langchain/anthropic"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; + +const calculatorSchema = z.object({ + operation: z + .enum(["add", "subtract", "multiply", "divide", "average"]) + .describe("The type of operation to execute."), + numbers: z.array(z.number()).describe("The numbers to operate on."), +}); + +const weatherSchema = z + .object({ + location: z.string().describe("The name of city to get the weather for."), + }) + .describe( + "Get the weather of a specific location and return the temperature in Celsius." + ); + +const tools = [ + { + name: "calculator", + description: "A simple calculator tool.", + input_schema: zodToJsonSchema(calculatorSchema), + }, + { + name: "get_weather", + description: "Get the weather of a location", + input_schema: zodToJsonSchema(weatherSchema), + }, +]; + +const model = new ChatAnthropic({ + anthropicApiKey: process.env.ANTHROPIC_API_KEY, + modelName: "claude-3-opus-20240229", +}).bind({ + tools, +}); + +const prompt = ChatPromptTemplate.fromMessages([ + [ + "system", + "You are a helpful assistant who always uses tools to ensure you provide accurate, up to date information.", + ], + ["human", "{input}"], +]); + +// Chain your prompt and model together +const chain = prompt.pipe(model); + +const response = await chain.invoke({ + input: + "What is the current weather in new york, and san francisco? Also, what is the average of these numbers: 2273,7192,272,92737?", +}); +console.log(JSON.stringify(response, null, 2)); +/* +{ + "kwargs": { + "content": "\nTo answer this query, there are two relevant tools:\n\n1. get_weather - This can be used to get the current weather for New York and San Francisco. It requires a \"location\" parameter. Since the user provided \"new york\" and \"san francisco\" as locations, we have the necessary information to call this tool twice - once for each city.\n\n2. calculator - This can be used to calculate the average of the provided numbers. It requires a \"numbers\" parameter which is an array of numbers, and an \"operation\" parameter. The user provided the numbers \"2273,7192,272,92737\" which we can split into an array, and they asked for the \"average\", so we have the necessary information to call this tool.\n\nSince we have the required parameters for both relevant tools, we can proceed with the function calls.\n", + "additional_kwargs": { + "id": "msg_013AgVS83LU6fWRHbykfvbYS", + "type": "message", + "role": "assistant", + "model": "claude-3-opus-20240229", + "stop_reason": "tool_use", + "usage": { + "input_tokens": 714, + "output_tokens": 336 + }, + "tool_calls": [ + { + "id": "toolu_01NHY2v7kZx8WqAvGzBuCu4h", + "type": "function", + "function": { + "arguments": "{\"location\":\"new york\"}", + "name": "get_weather" + } + }, + { + "id": "toolu_01PVCofvgkbnD4NfWfvXdsPC", + "type": "function", + "function": { + "arguments": "{\"location\":\"san francisco\"}", + "name": "get_weather" + } + }, + { + "id": "toolu_019AVVNUyCYnvsVdpkGKVDdv", + "type": "function", + "function": { + "arguments": "{\"operation\":\"average\",\"numbers\":[2273,7192,272,92737]}", + "name": "calculator" + } + } + ] + }, + } +} +*/ diff --git a/examples/src/models/chat/integration_anthropic_single_tool.ts b/examples/src/models/chat/integration_anthropic_single_tool.ts new file mode 100644 index 000000000000..7571fe8b8956 --- /dev/null +++ b/examples/src/models/chat/integration_anthropic_single_tool.ts @@ -0,0 +1,64 @@ +import { ChatAnthropic } from "@langchain/anthropic"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; + +const calculatorSchema = z.object({ + operation: z + .enum(["add", "subtract", "multiply", "divide"]) + .describe("The type of operation to execute."), + number1: z.number().describe("The first number to operate on."), + number2: z.number().describe("The second number to operate on."), +}); + +const tool = { + name: "calculator", + description: "A simple calculator tool", + input_schema: zodToJsonSchema(calculatorSchema), +}; + +const model = new ChatAnthropic({ + anthropicApiKey: process.env.ANTHROPIC_API_KEY, + modelName: "claude-3-haiku-20240307", +}).bind({ + tools: [tool], +}); + +const prompt = ChatPromptTemplate.fromMessages([ + [ + "system", + "You are a helpful assistant who always needs to use a calculator.", + ], + ["human", "{input}"], +]); + +// Chain your prompt and model together +const chain = prompt.pipe(model); + +const response = await chain.invoke({ + input: "What is 2 + 2?", +}); +console.log(JSON.stringify(response, null, 2)); +/* +{ + "kwargs": { + "content": "Okay, let's calculate that using the calculator tool:", + "additional_kwargs": { + "id": "msg_01YcT1KFV8qH7xG6T6C4EpGq", + "role": "assistant", + "model": "claude-3-haiku-20240307", + "tool_calls": [ + { + "id": "toolu_01UiqGsTTH45MUveRQfzf7KH", + "type": "function", + "function": { + "arguments": "{\"number1\":2,\"number2\":2,\"operation\":\"add\"}", + "name": "calculator" + } + } + ] + }, + "response_metadata": {} + } +} +*/ diff --git a/examples/src/models/chat/integration_anthropic_wsa.ts b/examples/src/models/chat/integration_anthropic_wsa.ts new file mode 100644 index 000000000000..0d24c0e6e775 --- /dev/null +++ b/examples/src/models/chat/integration_anthropic_wsa.ts @@ -0,0 +1,90 @@ +import { ChatAnthropic } from "@langchain/anthropic"; +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { z } from "zod"; + +const calculatorSchema = z + .object({ + operation: z + .enum(["add", "subtract", "multiply", "divide"]) + .describe("The type of operation to execute."), + number1: z.number().describe("The first number to operate on."), + number2: z.number().describe("The second number to operate on."), + }) + .describe("A simple calculator tool"); + +const model = new ChatAnthropic({ + anthropicApiKey: process.env.ANTHROPIC_API_KEY, + modelName: "claude-3-haiku-20240307", +}); + +// Pass the schema and tool name to the withStructuredOutput method +const modelWithTool = model.withStructuredOutput(calculatorSchema); + +const prompt = ChatPromptTemplate.fromMessages([ + [ + "system", + "You are a helpful assistant who always needs to use a calculator.", + ], + ["human", "{input}"], +]); + +// Chain your prompt and model together +const chain = prompt.pipe(modelWithTool); + +const response = await chain.invoke({ + input: "What is 2 + 2?", +}); +console.log(response); +/* + { operation: 'add', number1: 2, number2: 2 } +*/ + +/** + * You can supply a "name" field to give the LLM additional context + * around what you are trying to generate. You can also pass + * 'includeRaw' to get the raw message back from the model too. + */ +const includeRawModel = model.withStructuredOutput(calculatorSchema, { + name: "calculator", + includeRaw: true, +}); +const includeRawChain = prompt.pipe(includeRawModel); + +const includeRawResponse = await includeRawChain.invoke({ + input: "What is 2 + 2?", +}); +console.log(JSON.stringify(includeRawResponse, null, 2)); +/* +{ + "raw": { + "kwargs": { + "content": "Okay, let me use the calculator tool to find the result of 2 + 2:", + "additional_kwargs": { + "id": "msg_01HYwRhJoeqwr5LkSCHHks5t", + "type": "message", + "role": "assistant", + "model": "claude-3-haiku-20240307", + "usage": { + "input_tokens": 458, + "output_tokens": 109 + }, + "tool_calls": [ + { + "id": "toolu_01LDJpdtEQrq6pXSqSgEHErC", + "type": "function", + "function": { + "arguments": "{\"number1\":2,\"number2\":2,\"operation\":\"add\"}", + "name": "calculator" + } + } + ] + }, + } + }, + "parsed": { + "operation": "add", + "number1": 2, + "number2": 2 + } +} +*/ diff --git a/langchain-core/src/messages/index.ts b/langchain-core/src/messages/index.ts index d91598690bd7..18e39776a81d 100644 --- a/langchain-core/src/messages/index.ts +++ b/langchain-core/src/messages/index.ts @@ -49,7 +49,13 @@ export type MessageContentImageUrl = { image_url: string | { url: string; detail?: ImageDetail }; }; -export type MessageContentComplex = MessageContentText | MessageContentImageUrl; +export type MessageContentComplex = + | MessageContentText + | MessageContentImageUrl + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | (Record & { type?: "text" | "image_url" | string }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | (Record & { type?: never }); export type MessageContent = string | MessageContentComplex[]; diff --git a/langchain-core/src/output_parsers/string.ts b/langchain-core/src/output_parsers/string.ts index 5aaa1301e886..59476cd9baa7 100644 --- a/langchain-core/src/output_parsers/string.ts +++ b/langchain-core/src/output_parsers/string.ts @@ -63,15 +63,25 @@ export class StringOutputParser extends BaseTransformOutputParser { ): string { switch (content.type) { case "text": - return this._textContentToString(content); + if ("text" in content) { + // Type guard for MessageContentText + return this._textContentToString(content as MessageContentText); + } + break; case "image_url": - return this._imageUrlContentToString(content); + if ("image_url" in content) { + // Type guard for MessageContentImageUrl + return this._imageUrlContentToString( + content as MessageContentImageUrl + ); + } + break; default: throw new Error( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - `Cannot coerce "${(content as any).type}" message part into a string.` + `Cannot coerce "${content.type}" message part into a string.` ); } + throw new Error(`Invalid content type: ${content.type}`); } protected _baseMessageContentToString( diff --git a/langchain-core/src/prompts/chat.ts b/langchain-core/src/prompts/chat.ts index 90c0fc6aca28..45defd2ce07d 100644 --- a/langchain-core/src/prompts/chat.ts +++ b/langchain-core/src/prompts/chat.ts @@ -704,12 +704,31 @@ function _coerceMessagePromptTemplateLike( return new MessagesPlaceholder({ variableName, optional: true }); } const message = coerceMessageLikeToMessage(messagePromptTemplateLike); + let templateData: + | string + | (string | _TextTemplateParam | _ImageTemplateParam)[]; + + if (typeof message.content === "string") { + templateData = message.content; + } else { + // Assuming message.content is an array of complex objects, transform it. + templateData = message.content.map((item) => { + if ("text" in item) { + return { text: item.text }; + } else if ("image_url" in item) { + return { image_url: item.image_url }; + } else { + throw new Error("Invalid message content"); + } + }); + } + if (message._getType() === "human") { - return HumanMessagePromptTemplate.fromTemplate(message.content); + return HumanMessagePromptTemplate.fromTemplate(templateData); } else if (message._getType() === "ai") { - return AIMessagePromptTemplate.fromTemplate(message.content); + return AIMessagePromptTemplate.fromTemplate(templateData); } else if (message._getType() === "system") { - return SystemMessagePromptTemplate.fromTemplate(message.content); + return SystemMessagePromptTemplate.fromTemplate(templateData); } else if (ChatMessage.isInstance(message)) { return ChatMessagePromptTemplate.fromTemplate( message.content as string, diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index 6603b9c65cbd..d20a4ff7331e 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -7,19 +7,55 @@ import { AIMessageChunk, type BaseMessage, } from "@langchain/core/messages"; -import { ChatGenerationChunk, type ChatResult } from "@langchain/core/outputs"; +import { + ChatGeneration, + ChatGenerationChunk, + type ChatResult, +} from "@langchain/core/outputs"; import { getEnvironmentVariable } from "@langchain/core/utils/env"; import { BaseChatModel, type BaseChatModelParams, } from "@langchain/core/language_models/chat_models"; -import { type BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; +import { + StructuredOutputMethodOptions, + type BaseLanguageModelCallOptions, + BaseLanguageModelInput, +} from "@langchain/core/language_models/base"; +import { StructuredToolInterface } from "@langchain/core/tools"; +import { zodToJsonSchema } from "zod-to-json-schema"; +import { BaseLLMOutputParser } from "@langchain/core/output_parsers"; +import { + Runnable, + RunnablePassthrough, + RunnableSequence, +} from "@langchain/core/runnables"; +import { isZodSchema } from "@langchain/core/utils/types"; +import { z } from "zod"; +import { AnthropicToolsOutputParser } from "./output_parsers.js"; +import { AnthropicToolResponse } from "./types.js"; + +type AnthropicTool = { + name: string; + description: string; + /** + * JSON schema. + */ + input_schema: Record; +}; type AnthropicMessage = Anthropic.MessageParam; type AnthropicMessageCreateParams = Anthropic.MessageCreateParamsNonStreaming; type AnthropicStreamingMessageCreateParams = Anthropic.MessageCreateParamsStreaming; type AnthropicMessageStreamEvent = Anthropic.MessageStreamEvent; +type AnthropicRequestOptions = Anthropic.RequestOptions; + +interface ChatAnthropicCallOptions extends BaseLanguageModelCallOptions { + tools?: StructuredToolInterface[] | AnthropicTool[]; +} + +type AnthropicMessageResponse = Anthropic.ContentBlock | AnthropicToolResponse; function _formatImage(imageUrl: string) { const regex = /^data:(image\/.+);base64,(.+)$/; @@ -39,6 +75,39 @@ function _formatImage(imageUrl: string) { // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; } + +function anthropicResponseToChatMessages( + messages: AnthropicMessageResponse[], + additionalKwargs: Record +): ChatGeneration[] { + if (messages.length === 1 && messages[0].type === "text") { + return [ + { + text: messages[0].text, + message: new AIMessage(messages[0].text, additionalKwargs), + }, + ]; + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const castMessage = messages as any; + const generations: ChatGeneration[] = [ + { + text: "", + message: new AIMessage({ + content: castMessage, + additional_kwargs: additionalKwargs, + }), + }, + ]; + return generations; + } +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function isAnthropicTool(tool: any): tool is AnthropicTool { + return "input_schema" in tool; +} + /** * Input to AnthropicChat class. */ @@ -134,7 +203,7 @@ type Kwargs = Record; * ``` */ export class ChatAnthropicMessages< - CallOptions extends BaseLanguageModelCallOptions = BaseLanguageModelCallOptions + CallOptions extends ChatAnthropicCallOptions = ChatAnthropicCallOptions > extends BaseChatModel implements AnthropicInput @@ -211,6 +280,40 @@ export class ChatAnthropicMessages< this.clientOptions = fields?.clientOptions ?? {}; } + /** + * Formats LangChain StructuredTools to AnthropicTools. + * + * @param {ChatAnthropicCallOptions["tools"]} tools The tools to format + * @returns {AnthropicTool[] | undefined} The formatted tools, or undefined if none are passed. + * @throws {Error} If a mix of AnthropicTools and StructuredTools are passed. + */ + formatStructuredToolToAnthropic( + tools: ChatAnthropicCallOptions["tools"] + ): AnthropicTool[] | undefined { + if (!tools || !tools.length) { + return undefined; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((tools as any[]).every((tool) => isAnthropicTool(tool))) { + // If the tool is already an anthropic tool, return it + return tools as AnthropicTool[]; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((tools as any[]).some((tool) => isAnthropicTool(tool))) { + throw new Error( + `Can not pass in a mix of AnthropicTools and StructuredTools` + ); + } + + return (tools as StructuredToolInterface[]).map((tool) => ({ + name: tool.name, + description: tool.description, + input_schema: zodToJsonSchema(tool.schema), + })); + } + /** * Get the parameters used to invoke the model */ @@ -233,6 +336,35 @@ export class ChatAnthropicMessages< }; } + invocationOptions( + request: Omit< + AnthropicMessageCreateParams | AnthropicStreamingMessageCreateParams, + "messages" + > & + Kwargs, + options: this["ParsedCallOptions"] + ): AnthropicRequestOptions { + const toolUseBetaHeader = { + "anthropic-beta": "tools-2024-04-04", + }; + const tools = this.formatStructuredToolToAnthropic(options?.tools); + // If tools are present, populate the body with the message request params. + // This is because Anthropic overwrites the message request params if a body + // is passed. + const body = tools + ? { + ...request, + tools, + } + : undefined; + const headers = tools ? toolUseBetaHeader : undefined; + return { + signal: options.signal, + ...(body ? { body } : {}), + ...(headers ? { headers } : {}), + }; + } + /** @ignore */ _identifyingParams() { return { @@ -257,67 +389,102 @@ export class ChatAnthropicMessages< runManager?: CallbackManagerForLLMRun ): AsyncGenerator { const params = this.invocationParams(options); - const stream = await this.createStreamWithRetry({ - ...params, - ...this.formatMessagesForAnthropic(messages), - stream: true, - }); - let usageData = { input_tokens: 0, output_tokens: 0 }; - for await (const data of stream) { - if (options.signal?.aborted) { - stream.controller.abort(); - throw new Error("AbortError: User aborted the request."); - } - if (data.type === "message_start") { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const { content, usage, ...additionalKwargs } = data.message; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const filteredAdditionalKwargs: Record = {}; - for (const [key, value] of Object.entries(additionalKwargs)) { - if (value !== undefined && value !== null) { - filteredAdditionalKwargs[key] = value; - } - } - usageData = usage; - yield new ChatGenerationChunk({ - message: new AIMessageChunk({ - content: "", - additional_kwargs: filteredAdditionalKwargs, - }), - text: "", - }); - } else if (data.type === "message_delta") { - yield new ChatGenerationChunk({ - message: new AIMessageChunk({ - content: "", - additional_kwargs: { ...data.delta }, - }), - text: "", - }); - if (data?.usage !== undefined) { - usageData.output_tokens += data.usage.output_tokens; + const requestOptions = this.invocationOptions( + { + ...params, + stream: false, + ...this.formatMessagesForAnthropic(messages), + }, + options + ); + if (options.tools !== undefined && options.tools.length > 0) { + const requestOptions = this.invocationOptions( + { + ...params, + stream: false, + ...this.formatMessagesForAnthropic(messages), + }, + options + ); + const generations = await this._generateNonStreaming( + messages, + params, + requestOptions + ); + + yield new ChatGenerationChunk({ + message: new AIMessageChunk({ + content: generations[0].message.content, + additional_kwargs: generations[0].message.additional_kwargs, + }), + text: generations[0].text, + }); + } else { + const stream = await this.createStreamWithRetry( + { + ...params, + ...this.formatMessagesForAnthropic(messages), + stream: true, + }, + requestOptions + ); + let usageData = { input_tokens: 0, output_tokens: 0 }; + for await (const data of stream) { + if (options.signal?.aborted) { + stream.controller.abort(); + throw new Error("AbortError: User aborted the request."); } - } else if (data.type === "content_block_delta") { - const content = data.delta?.text; - if (content !== undefined) { + if (data.type === "message_start") { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { content, usage, ...additionalKwargs } = data.message; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const filteredAdditionalKwargs: Record = {}; + for (const [key, value] of Object.entries(additionalKwargs)) { + if (value !== undefined && value !== null) { + filteredAdditionalKwargs[key] = value; + } + } + usageData = usage; + yield new ChatGenerationChunk({ + message: new AIMessageChunk({ + content: "", + additional_kwargs: filteredAdditionalKwargs, + }), + text: "", + }); + } else if (data.type === "message_delta") { yield new ChatGenerationChunk({ message: new AIMessageChunk({ - content, - additional_kwargs: {}, + content: "", + additional_kwargs: { ...data.delta }, }), - text: content, + text: "", }); - await runManager?.handleLLMNewToken(content); + if (data?.usage !== undefined) { + usageData.output_tokens += data.usage.output_tokens; + } + } else if (data.type === "content_block_delta") { + const content = data.delta?.text; + if (content !== undefined) { + yield new ChatGenerationChunk({ + message: new AIMessageChunk({ + content, + additional_kwargs: {}, + }), + text: content, + }); + await runManager?.handleLLMNewToken(content); + } } } + yield new ChatGenerationChunk({ + message: new AIMessageChunk({ + content: "", + additional_kwargs: { usage: usageData }, + }), + text: "", + }); } - yield new ChatGenerationChunk({ - message: new AIMessageChunk({ - content: "", - additional_kwargs: { usage: usageData }, - }), - text: "", - }); } /** @@ -344,6 +511,8 @@ export class ChatAnthropicMessages< role = "user" as const; } else if (message._getType() === "ai") { role = "assistant" as const; + } else if (message._getType() === "tool") { + role = "user" as const; } else if (message._getType() === "system") { throw new Error( "System messages are only permitted as the first passed message." @@ -358,26 +527,35 @@ export class ChatAnthropicMessages< role, content: message.content, }; - } else { - return { - role, - content: message.content.map((contentPart) => { - if (contentPart.type === "image_url") { - let source; - if (typeof contentPart.image_url === "string") { - source = _formatImage(contentPart.image_url); - } else { - source = _formatImage(contentPart.image_url.url); - } - return { - type: "image" as const, - source, - }; + } else if ("type" in message.content) { + const contentBlocks = message.content.map((contentPart) => { + if (contentPart.type === "image_url") { + let source; + if (typeof contentPart.image_url === "string") { + source = _formatImage(contentPart.image_url); } else { - return contentPart; + source = _formatImage(contentPart.image_url.url); } - }), + return { + type: "image" as const, // Explicitly setting the type as "image" + source, + }; + } else if (contentPart.type === "text") { + // Assuming contentPart is of type MessageContentText here + return { + type: "text" as const, // Explicitly setting the type as "text" + text: contentPart.text, + }; + } else { + throw new Error("Unsupported message content format"); + } + }); + return { + role, + content: contentBlocks, }; + } else { + throw new Error("Unsupported message content format"); } }); return { @@ -386,6 +564,35 @@ export class ChatAnthropicMessages< }; } + /** @ignore */ + async _generateNonStreaming( + messages: BaseMessage[], + params: Omit< + | Anthropic.Messages.MessageCreateParamsNonStreaming + | Anthropic.Messages.MessageCreateParamsStreaming, + "messages" + > & + Kwargs, + requestOptions: AnthropicRequestOptions + ) { + const response = await this.completionWithRetry( + { + ...params, + stream: false, + ...this.formatMessagesForAnthropic(messages), + }, + requestOptions + ); + + const { content, ...additionalKwargs } = response; + + const generations = anthropicResponseToChatMessages( + content, + additionalKwargs + ); + return generations; + } + /** @ignore */ async _generate( messages: BaseMessage[], @@ -401,11 +608,7 @@ export class ChatAnthropicMessages< const params = this.invocationParams(options); if (params.stream) { let finalChunk: ChatGenerationChunk | undefined; - const stream = await this._streamResponseChunks( - messages, - options, - runManager - ); + const stream = this._streamResponseChunks(messages, options, runManager); for await (const chunk of stream) { if (finalChunk === undefined) { finalChunk = chunk; @@ -425,34 +628,21 @@ export class ChatAnthropicMessages< ], }; } else { - const response = await this.completionWithRetry( + const requestOptions = this.invocationOptions( { ...params, stream: false, ...this.formatMessagesForAnthropic(messages), }, - { signal: options.signal } + options + ); + const generations = await this._generateNonStreaming( + messages, + params, + requestOptions ); - - const { content, ...additionalKwargs } = response; - - if (!Array.isArray(content) || content.length !== 1) { - console.log(content); - throw new Error( - "Received multiple content parts in Anthropic response. Only single part messages are currently supported." - ); - } - return { - generations: [ - { - text: content[0].text, - message: new AIMessage({ - content: content[0].text, - additional_kwargs: additionalKwargs, - }), - }, - ], + generations, }; } } @@ -463,31 +653,35 @@ export class ChatAnthropicMessages< * @returns A streaming request. */ protected async createStreamWithRetry( - request: AnthropicStreamingMessageCreateParams & Kwargs + request: AnthropicStreamingMessageCreateParams & Kwargs, + options?: AnthropicRequestOptions ): Promise> { if (!this.streamingClient) { - const options = this.apiUrl ? { baseURL: this.apiUrl } : undefined; + const options_ = this.apiUrl ? { baseURL: this.apiUrl } : undefined; this.streamingClient = new Anthropic({ ...this.clientOptions, - ...options, + ...options_, apiKey: this.anthropicApiKey, // Prefer LangChain built-in retries maxRetries: 0, }); } const makeCompletionRequest = async () => - this.streamingClient.messages.create({ - ...request, - ...this.invocationKwargs, - stream: true, - } as AnthropicStreamingMessageCreateParams); + this.streamingClient.messages.create( + { + ...request, + ...this.invocationKwargs, + stream: true, + } as AnthropicStreamingMessageCreateParams, + options + ); return this.caller.call(makeCompletionRequest); } /** @ignore */ protected async completionWithRetry( request: AnthropicMessageCreateParams & Kwargs, - options: { signal?: AbortSignal } + options: AnthropicRequestOptions ): Promise { if (!this.anthropicApiKey) { throw new Error("Missing Anthropic API key."); @@ -502,12 +696,15 @@ export class ChatAnthropicMessages< }); } const makeCompletionRequest = async () => - this.batchClient.messages.create({ - ...request, - ...this.invocationKwargs, - } as AnthropicMessageCreateParams); + this.batchClient.messages.create( + { + ...request, + ...this.invocationKwargs, + } as AnthropicMessageCreateParams, + options + ); return this.caller.callWithOptions( - { signal: options.signal }, + { signal: options.signal ?? undefined }, makeCompletionRequest ); } @@ -515,6 +712,126 @@ export class ChatAnthropicMessages< _llmType() { return "anthropic"; } + + withStructuredOutput< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunOutput extends Record = Record + >( + outputSchema: + | z.ZodType + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | Record, + config?: StructuredOutputMethodOptions + ): Runnable; + + withStructuredOutput< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunOutput extends Record = Record + >( + outputSchema: + | z.ZodType + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | Record, + config?: StructuredOutputMethodOptions + ): Runnable; + + withStructuredOutput< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunOutput extends Record = Record + >( + outputSchema: + | z.ZodType + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | Record, + config?: StructuredOutputMethodOptions + ): + | Runnable + | Runnable< + BaseLanguageModelInput, + { raw: BaseMessage; parsed: RunOutput } + > { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const schema: z.ZodType | Record = outputSchema; + const name = config?.name; + const method = config?.method; + const includeRaw = config?.includeRaw; + if (method === "jsonMode") { + throw new Error(`Anthropic only supports "functionCalling" as a method.`); + } + + let functionName = name ?? "extract"; + let outputParser: BaseLLMOutputParser; + let tools: AnthropicTool[]; + if (isZodSchema(schema)) { + const jsonSchema = zodToJsonSchema(schema); + tools = [ + { + name: functionName, + description: + jsonSchema.description ?? "A function available to call.", + input_schema: jsonSchema, + }, + ]; + outputParser = new AnthropicToolsOutputParser({ + returnSingle: true, + keyName: functionName, + zodSchema: schema, + }); + } else { + let anthropicTools: AnthropicTool; + if ( + typeof schema.name === "string" && + typeof schema.description === "string" && + typeof schema.input_schema === "object" && + schema.input_schema != null + ) { + anthropicTools = schema as AnthropicTool; + functionName = schema.name; + } else { + anthropicTools = { + name: functionName, + description: schema.description ?? "", + input_schema: schema, + }; + } + tools = [anthropicTools]; + outputParser = new AnthropicToolsOutputParser({ + returnSingle: true, + keyName: functionName, + }); + } + const llm = this.bind({ + tools, + } as Partial); + + if (!includeRaw) { + return llm.pipe(outputParser).withConfig({ + runName: "ChatAnthropicStructuredOutput", + }) as Runnable; + } + + const parserAssign = RunnablePassthrough.assign({ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + parsed: (input: any, config) => outputParser.invoke(input.raw, config), + }); + const parserNone = RunnablePassthrough.assign({ + parsed: () => null, + }); + const parsedWithFallback = parserAssign.withFallbacks({ + fallbacks: [parserNone], + }); + return RunnableSequence.from< + BaseLanguageModelInput, + { raw: BaseMessage; parsed: RunOutput } + >([ + { + raw: llm, + }, + parsedWithFallback, + ]).withConfig({ + runName: "StructuredOutputRunnable", + }); + } } export class ChatAnthropic extends ChatAnthropicMessages {} diff --git a/libs/langchain-anthropic/src/experimental/tool_calling.ts b/libs/langchain-anthropic/src/experimental/tool_calling.ts index 48aff6898310..8ba90731850d 100644 --- a/libs/langchain-anthropic/src/experimental/tool_calling.ts +++ b/libs/langchain-anthropic/src/experimental/tool_calling.ts @@ -67,6 +67,7 @@ export type ChatAnthropicToolsInput = Partial & /** * Experimental wrapper over Anthropic chat models that adds support for * a function calling interface. + * @deprecated Prefer traditional tool use through ChatAnthropic. */ export class ChatAnthropicTools extends BaseChatModel { llm: BaseChatModel; diff --git a/libs/langchain-anthropic/src/output_parsers.ts b/libs/langchain-anthropic/src/output_parsers.ts new file mode 100644 index 000000000000..928a128c184b --- /dev/null +++ b/libs/langchain-anthropic/src/output_parsers.ts @@ -0,0 +1,54 @@ +import { BaseLLMOutputParser } from "@langchain/core/output_parsers"; +import { JsonOutputKeyToolsParserParams } from "@langchain/core/output_parsers/openai_tools"; +import { ChatGeneration } from "@langchain/core/outputs"; +import { AnthropicToolResponse } from "./types.js"; + +interface AnthropicToolsOutputParserParams + extends JsonOutputKeyToolsParserParams {} + +export class AnthropicToolsOutputParser< + // eslint-disable-next-line @typescript-eslint/no-explicit-any + T extends Record = Record +> extends BaseLLMOutputParser { + static lc_name() { + return "AnthropicToolsOutputParser"; + } + + lc_namespace = ["langchain", "anthropic", "output_parsers"]; + + returnId = false; + + /** The type of tool calls to return. */ + keyName: string; + + /** Whether to return only the first tool call. */ + returnSingle = false; + + constructor(params: AnthropicToolsOutputParserParams) { + super(params); + this.keyName = params.keyName; + this.returnSingle = params.returnSingle ?? this.returnSingle; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async parseResult(generations: ChatGeneration[]): Promise { + const tools = generations.flatMap((generation) => { + const { message } = generation; + if (typeof message === "string") { + return []; + } + if (!Array.isArray(message.content)) { + return []; + } + const tool = message.content.find((item) => item.type === "tool_use") as + | AnthropicToolResponse + | undefined; + return tool; + }); + if (tools.length === 0 || !tools[0]) { + throw new Error("No tools provided to AnthropicToolsOutputParser."); + } + const [tool] = tools; + return tool.input as T; + } +} diff --git a/libs/langchain-anthropic/src/tests/chat_models.int.test.ts b/libs/langchain-anthropic/src/tests/chat_models.int.test.ts index aa98197ef61b..34cadf253af6 100644 --- a/libs/langchain-anthropic/src/tests/chat_models.int.test.ts +++ b/libs/langchain-anthropic/src/tests/chat_models.int.test.ts @@ -11,7 +11,11 @@ import { SystemMessagePromptTemplate, } from "@langchain/core/prompts"; import { CallbackManager } from "@langchain/core/callbacks/manager"; +import { StructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { zodToJsonSchema } from "zod-to-json-schema"; import { ChatAnthropic } from "../chat_models.js"; +import { AnthropicToolResponse } from "../types.js"; test.skip("Test ChatAnthropic", async () => { const chat = new ChatAnthropic({ @@ -317,3 +321,225 @@ test("Test ChatAnthropic multimodal", async () => { ]); console.log(res); }); + +describe("Tool calling", () => { + const zodSchema = z + .object({ + location: z.string().describe("The name of city to get the weather for."), + }) + .describe( + "Get the weather of a specific location and return the temperature in Celsius." + ); + + class WeatherTool extends StructuredTool { + schema = z.object({ + location: z.string().describe("The name of city to get the weather for."), + }); + + description = + "Get the weather of a specific location and return the temperature in Celsius."; + + name = "get_weather"; + + async _call(input: z.infer) { + console.log(`WeatherTool called with input: ${input}`); + return `The weather in ${input.location} is 25°C`; + } + } + + const model = new ChatAnthropic({ + modelName: "claude-3-sonnet-20240229", + temperature: 0, + }); + + const anthropicTool = { + name: "get_weather", + description: + "Get the weather of a specific location and return the temperature in Celsius.", + input_schema: { + type: "object", + properties: { + location: { + type: "string", + description: "The name of city to get the weather for.", + }, + }, + required: ["location"], + }, + }; + + test("Can bind & invoke StructuredTools", async () => { + const tools = [new WeatherTool()]; + + const modelWithTools = model.bind({ + tools, + }); + + const result = await modelWithTools.invoke( + "What is the weather in London today?" + ); + console.log( + { + tool_calls: JSON.stringify(result.content, null, 2), + }, + "Can bind & invoke StructuredTools" + ); + expect(Array.isArray(result.content)).toBeTruthy(); + if (!Array.isArray(result.content)) { + throw new Error("Content is not an array"); + } + let toolCall: AnthropicToolResponse | undefined; + result.content.forEach((item) => { + if (item.type === "tool_use") { + toolCall = item as AnthropicToolResponse; + } + }); + if (!toolCall) { + throw new Error("No tool call found"); + } + expect(toolCall).toBeTruthy(); + const { name, input } = toolCall; + expect(name).toBe("get_weather"); + expect(input).toBeTruthy(); + expect(input.location).toBeTruthy(); + }); + + test("Can bind & invoke AnthropicTools", async () => { + const modelWithTools = model.bind({ + tools: [anthropicTool], + }); + + const result = await modelWithTools.invoke( + "What is the weather in London today?" + ); + console.log( + { + tool_calls: JSON.stringify(result.content, null, 2), + }, + "Can bind & invoke StructuredTools" + ); + expect(Array.isArray(result.content)).toBeTruthy(); + if (!Array.isArray(result.content)) { + throw new Error("Content is not an array"); + } + let toolCall: AnthropicToolResponse | undefined; + result.content.forEach((item) => { + if (item.type === "tool_use") { + toolCall = item as AnthropicToolResponse; + } + }); + if (!toolCall) { + throw new Error("No tool call found"); + } + expect(toolCall).toBeTruthy(); + const { name, input } = toolCall; + expect(name).toBe("get_weather"); + expect(input).toBeTruthy(); + expect(input.location).toBeTruthy(); + }); + + test("Can bind & stream AnthropicTools", async () => { + const modelWithTools = model.bind({ + tools: [anthropicTool], + }); + + const result = await modelWithTools.stream( + "What is the weather in London today?" + ); + let finalMessage; + for await (const item of result) { + console.log("item", JSON.stringify(item, null, 2)); + finalMessage = item; + } + + if (!finalMessage) { + throw new Error("No final message returned"); + } + + console.log( + { + tool_calls: JSON.stringify(finalMessage.content, null, 2), + }, + "Can bind & invoke StructuredTools" + ); + expect(Array.isArray(finalMessage.content)).toBeTruthy(); + if (!Array.isArray(finalMessage.content)) { + throw new Error("Content is not an array"); + } + let toolCall: AnthropicToolResponse | undefined; + finalMessage.content.forEach((item) => { + if (item.type === "tool_use") { + toolCall = item as AnthropicToolResponse; + } + }); + if (!toolCall) { + throw new Error("No tool call found"); + } + expect(toolCall).toBeTruthy(); + const { name, input } = toolCall; + expect(name).toBe("get_weather"); + expect(input).toBeTruthy(); + expect(input.location).toBeTruthy(); + }); + + test("withStructuredOutput with zod schema", async () => { + const modelWithTools = model.withStructuredOutput<{ location: string }>( + zodSchema, + { + name: "get_weather", + } + ); + + const result = await modelWithTools.invoke( + "What is the weather in London today?" + ); + console.log( + { + result, + }, + "withStructuredOutput with zod schema" + ); + expect(typeof result.location).toBe("string"); + }); + + test("withStructuredOutput with AnthropicTool", async () => { + const modelWithTools = model.withStructuredOutput<{ location: string }>( + anthropicTool, + { + name: anthropicTool.name, + } + ); + + const result = await modelWithTools.invoke( + "What is the weather in London today?" + ); + console.log( + { + result, + }, + "withStructuredOutput with AnthropicTool" + ); + expect(typeof result.location).toBe("string"); + }); + + test("withStructuredOutput JSON Schema only", async () => { + const jsonSchema = zodToJsonSchema(zodSchema); + const modelWithTools = model.withStructuredOutput<{ location: string }>( + jsonSchema, + { + name: "get_weather", + } + ); + + const result = await modelWithTools.invoke( + "What is the weather in London today?" + ); + console.log( + { + result, + }, + "withStructuredOutput JSON Schema only" + ); + expect(typeof result.location).toBe("string"); + }); +}); diff --git a/libs/langchain-anthropic/src/types.ts b/libs/langchain-anthropic/src/types.ts new file mode 100644 index 000000000000..a4f0846226b2 --- /dev/null +++ b/libs/langchain-anthropic/src/types.ts @@ -0,0 +1,7 @@ +export type AnthropicToolResponse = { + type: "tool_use"; + id: string; + name: string; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + input: Record; +}; diff --git a/libs/langchain-google-common/src/utils/gemini.ts b/libs/langchain-google-common/src/utils/gemini.ts index f0f9576222ad..8882af6166b1 100644 --- a/libs/langchain-google-common/src/utils/gemini.ts +++ b/libs/langchain-google-common/src/utils/gemini.ts @@ -91,14 +91,24 @@ export function messageContentToParts(content: MessageContent): GeminiPart[] { .map((content) => { switch (content.type) { case "text": - return messageContentText(content); + if ("text" in content) { + return messageContentText(content as MessageContentText); + } + break; case "image_url": - return messageContentImageUrl(content); + if ("image_url" in content) { + // Type guard for MessageContentImageUrl + return messageContentImageUrl(content as MessageContentImageUrl); + } + break; default: throw new Error( `Unsupported type received while converting message to message parts` ); } + throw new Error( + `Cannot coerce "${content.type}" message part into a string.` + ); }) .reduce((acc: GeminiPart[], val: GeminiPart | null | undefined) => { if (val) {