From f3fb958dd1ed79967790f5606e681cebf0fef156 Mon Sep 17 00:00:00 2001 From: Jeongho Nam Date: Sat, 14 Dec 2024 02:51:38 +0900 Subject: [PATCH] Enhance function calling test functions --- .../arguments/claude.sale.input.json | 45 ++++++++------- .../schemas/claude.sale.schema.json | 12 ++-- .../schemas/llama.sale.schema.json | 12 ++-- package.json | 16 +++--- test/controllers/AppController.ts | 3 +- .../test_chatgpt_function_calling_default.ts | 1 + .../test_chatgpt_function_calling_example.ts | 1 + ...test_chatgpt_function_calling_recursive.ts | 1 + .../test_chatgpt_function_calling_union.ts | 1 + ...e_function_calling_additionalProperties.ts | 1 + .../test_claude_function_calling_default.ts | 1 + .../test_claude_function_calling_example.ts | 1 + .../test_claude_function_calling_recursive.ts | 1 + .../test_claude_function_calling_tags.ts | 1 + .../test_claude_function_calling_union.ts | 1 + .../test_gemini_function_calling_default.ts | 1 + .../test_gemini_function_calling_example.ts | 1 + ...a_function_calling_additionalProperties.ts | 1 + .../test_llama_function_calling_default.ts | 1 + .../test_llama_function_calling_example.ts | 1 + .../test_llama_function_calling_nullable.ts | 1 + .../test_llama_function_calling_recursive.ts | 1 + .../llama/test_llama_function_calling_tags.ts | 1 + .../test_llama_function_calling_union.ts | 1 + test/utils/ChatGptFunctionCaller.ts | 52 ++++++++++++++--- test/utils/ClaudeFunctionCaller.ts | 56 +++++++++++++++---- test/utils/GeminiFunctionCaller.ts | 41 ++++++++++---- test/utils/LlamaFunctionCaller.ts | 54 +++++++++++++++--- test/utils/ShoppingSalePrompt.ts | 1 + 29 files changed, 234 insertions(+), 77 deletions(-) diff --git a/examples/function-calling/arguments/claude.sale.input.json b/examples/function-calling/arguments/claude.sale.input.json index a0a6e7e..f233ec6 100644 --- a/examples/function-calling/arguments/claude.sale.input.json +++ b/examples/function-calling/arguments/claude.sale.input.json @@ -31,23 +31,16 @@ "code": "samchon", "category_codes": [ "electronics", - "2in1_laptops", - "windows_tablets" + "laptops", + "2in1_laptops" ] } ], - "tags": [ - "Surface Pro", - "2-in-1", - "Windows", - "Laptop", - "Tablet" - ], "units": [ { "name": "Surface Pro 9 Entity", - "required": true, "primary": true, + "required": true, "options": [ { "type": "select", @@ -100,7 +93,7 @@ ], "stocks": [ { - "name": "Surface Pro 9 - i3/8GB/128GB", + "name": "Surface Pro 9 (i3/8GB/128GB)", "price": { "nominal": 1000000, "real": 899000 @@ -122,7 +115,7 @@ ] }, { - "name": "Surface Pro 9 - i3/16GB/256GB", + "name": "Surface Pro 9 (i3/16GB/256GB)", "price": { "nominal": 1200000, "real": 1099000 @@ -144,7 +137,7 @@ ] }, { - "name": "Surface Pro 9 - i3/16GB/512GB", + "name": "Surface Pro 9 (i3/16GB/512GB)", "price": { "nominal": 1400000, "real": 1299000 @@ -166,7 +159,7 @@ ] }, { - "name": "Surface Pro 9 - i5/16GB/256GB", + "name": "Surface Pro 9 (i5/16GB/256GB)", "price": { "nominal": 1500000, "real": 1399000 @@ -188,7 +181,7 @@ ] }, { - "name": "Surface Pro 9 - i5/32GB/512GB", + "name": "Surface Pro 9 (i5/32GB/512GB)", "price": { "nominal": 1800000, "real": 1699000 @@ -210,7 +203,7 @@ ] }, { - "name": "Surface Pro 9 - i7/16GB/512GB", + "name": "Surface Pro 9 (i7/16GB/512GB)", "price": { "nominal": 1800000, "real": 1699000 @@ -232,7 +225,7 @@ ] }, { - "name": "Surface Pro 9 - i7/32GB/512GB", + "name": "Surface Pro 9 (i7/32GB/512GB)", "price": { "nominal": 2000000, "real": 1899000 @@ -257,12 +250,12 @@ }, { "name": "Warranty Program", - "required": false, "primary": false, + "required": false, "options": [], "stocks": [ { - "name": "Surface Pro 9 Warranty Program", + "name": "Warranty Program", "price": { "nominal": 100000, "real": 89000 @@ -274,12 +267,12 @@ }, { "name": "Magnetic Keyboard", - "required": false, "primary": false, + "required": false, "options": [], "stocks": [ { - "name": "Surface Pro 9 Magnetic Keyboard", + "name": "Magnetic Keyboard", "price": { "nominal": 200000, "real": 169000 @@ -289,5 +282,15 @@ } ] } + ], + "tags": [ + "Surface", + "Pro", + "9", + "Microsoft", + "2-in-1", + "Tablet", + "Laptop", + "Windows" ] } \ No newline at end of file diff --git a/examples/function-calling/schemas/claude.sale.schema.json b/examples/function-calling/schemas/claude.sale.schema.json index f1ebd14..c53f2d1 100644 --- a/examples/function-calling/schemas/claude.sale.schema.json +++ b/examples/function-calling/schemas/claude.sale.schema.json @@ -327,16 +327,16 @@ }, "minItems": 1 }, - "name": { - "title": "Representative name of the unit", - "description": "Representative name of the unit.", - "type": "string" - }, "required": { "title": "Whether the unit is required or not", "description": "Whether the unit is required or not.\n\nWhen the unit is required, the customer must select the unit. If do not\nselect, customer can't buy it.\n\nFor example, if there's a sale \"Macbook Set\" and one of the unit is the\n\"Main Body\", is it possible to buy the \"Macbook Set\" without the\n\"Main Body\" unit? This property is for that case.", "type": "boolean" }, + "name": { + "title": "Representative name of the unit", + "description": "Representative name of the unit.", + "type": "string" + }, "primary": { "title": "Whether the unit is primary or not", "description": "Whether the unit is primary or not.\n\nJust a labeling value.", @@ -346,8 +346,8 @@ "required": [ "options", "stocks", - "name", "required", + "name", "primary" ] }, diff --git a/examples/function-calling/schemas/llama.sale.schema.json b/examples/function-calling/schemas/llama.sale.schema.json index f1ebd14..c53f2d1 100644 --- a/examples/function-calling/schemas/llama.sale.schema.json +++ b/examples/function-calling/schemas/llama.sale.schema.json @@ -327,16 +327,16 @@ }, "minItems": 1 }, - "name": { - "title": "Representative name of the unit", - "description": "Representative name of the unit.", - "type": "string" - }, "required": { "title": "Whether the unit is required or not", "description": "Whether the unit is required or not.\n\nWhen the unit is required, the customer must select the unit. If do not\nselect, customer can't buy it.\n\nFor example, if there's a sale \"Macbook Set\" and one of the unit is the\n\"Main Body\", is it possible to buy the \"Macbook Set\" without the\n\"Main Body\" unit? This property is for that case.", "type": "boolean" }, + "name": { + "title": "Representative name of the unit", + "description": "Representative name of the unit.", + "type": "string" + }, "primary": { "title": "Whether the unit is primary or not", "description": "Whether the unit is primary or not.\n\nJust a labeling value.", @@ -346,8 +346,8 @@ "required": [ "options", "stocks", - "name", "required", + "name", "primary" ] }, diff --git a/package.json b/package.json index 3bbbb46..0082558 100644 --- a/package.json +++ b/package.json @@ -44,10 +44,10 @@ "devDependencies": { "@anthropic-ai/sdk": "^0.32.1", "@google/generative-ai": "^0.21.0", - "@nestia/core": "4.0.0-dev.20241116", + "@nestia/core": "4.2.0", "@nestia/e2e": "0.7.0", - "@nestia/fetcher": "4.0.0-dev.20241116", - "@nestia/sdk": "4.0.0-dev.20241116", + "@nestia/fetcher": "4.2.0", + "@nestia/sdk": "4.2.0", "@nestjs/common": "^10.4.1", "@nestjs/core": "^10.4.1", "@nestjs/platform-express": "^10.4.1", @@ -55,6 +55,7 @@ "@rollup/plugin-typescript": "^11.1.6", "@trivago/prettier-plugin-sort-imports": "^4.3.0", "@types/js-yaml": "^4.0.9", + "@types/multer": "^1.4.12", "@types/node": "^20.12.7", "@types/uuid": "^10.0.0", "axios": "^1.7.7", @@ -62,6 +63,7 @@ "dotenv": "^16.4.5", "dotenv-expand": "^12.0.0", "js-yaml": "^4.1.0", + "multer": "^1.4.5-lts.1", "nestia": "^6.0.1", "openai": "^4.72.0", "prettier": "^3.2.5", @@ -69,11 +71,11 @@ "rollup": "^4.18.1", "source-map-support": "^0.5.21", "ts-node": "^10.9.2", - "ts-patch": "^3.2.1", + "ts-patch": "^3.3.0", "tstl": "^3.0.0", - "typescript": "~5.6.3", - "typescript-transform-paths": "^3.4.7", - "typia": "7.0.0-dev.20241201", + "typescript": "~5.7.2", + "typescript-transform-paths": "^3.5.2", + "typia": "7.3.0", "uuid": "^10.0.0" }, "sideEffects": false, diff --git a/test/controllers/AppController.ts b/test/controllers/AppController.ts index 1b15fbc..34a5a90 100644 --- a/test/controllers/AppController.ts +++ b/test/controllers/AppController.ts @@ -6,6 +6,7 @@ import { TypedRoute, } from "@nestia/core"; import { Controller, Query } from "@nestjs/common"; +import Multer from "multer"; import { tags } from "typia"; @Controller() @@ -76,7 +77,7 @@ export class AppController { @TypedParam("level") level: number, @TypedParam("optimal") optimal: boolean, @TypedQuery() query: IQuery, - @TypedFormData.Body() + @TypedFormData.Body(() => Multer()) body: IMultipart, ) { return { diff --git a/test/features/llm/chatgpt/test_chatgpt_function_calling_default.ts b/test/features/llm/chatgpt/test_chatgpt_function_calling_default.ts index cdd2b38..dd22758 100644 --- a/test/features/llm/chatgpt/test_chatgpt_function_calling_default.ts +++ b/test/features/llm/chatgpt/test_chatgpt_function_calling_default.ts @@ -12,6 +12,7 @@ export const test_chatgpt_function_calling_default = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[IParameters]>(), + validate: typia.createValidate(), texts: [ { role: "assistant", diff --git a/test/features/llm/chatgpt/test_chatgpt_function_calling_example.ts b/test/features/llm/chatgpt/test_chatgpt_function_calling_example.ts index 9701479..3cd2aa6 100644 --- a/test/features/llm/chatgpt/test_chatgpt_function_calling_example.ts +++ b/test/features/llm/chatgpt/test_chatgpt_function_calling_example.ts @@ -12,6 +12,7 @@ export const test_chatgpt_function_calling_example = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/chatgpt/test_chatgpt_function_calling_recursive.ts b/test/features/llm/chatgpt/test_chatgpt_function_calling_recursive.ts index a8bebbe..5f6e080 100644 --- a/test/features/llm/chatgpt/test_chatgpt_function_calling_recursive.ts +++ b/test/features/llm/chatgpt/test_chatgpt_function_calling_recursive.ts @@ -12,6 +12,7 @@ export const test_chatgpt_function_calling_recursive = () => name: "composeCategories", description: "Compose categories from the input.", collection: typia.json.schemas<[{ input: IShoppingCategory[] }]>(), + validate: typia.createValidate<[{ input: IShoppingCategory[] }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/chatgpt/test_chatgpt_function_calling_union.ts b/test/features/llm/chatgpt/test_chatgpt_function_calling_union.ts index 000377b..9d24ad9 100644 --- a/test/features/llm/chatgpt/test_chatgpt_function_calling_union.ts +++ b/test/features/llm/chatgpt/test_chatgpt_function_calling_union.ts @@ -12,6 +12,7 @@ export const test_chatgpt_function_calling_union = (): Promise => name: "draw", description: "Draw a shape with following geometry.", collection: typia.json.schemas<[{ input: Shape }]>(), + validate: typia.createValidate<[{ input: Shape }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/claude/test_claude_function_calling_additionalProperties.ts b/test/features/llm/claude/test_claude_function_calling_additionalProperties.ts index 69ddcd7..9e98616 100644 --- a/test/features/llm/claude/test_claude_function_calling_additionalProperties.ts +++ b/test/features/llm/claude/test_claude_function_calling_additionalProperties.ts @@ -11,6 +11,7 @@ export const test_claude_function_calling_additionalProperties = name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/claude/test_claude_function_calling_default.ts b/test/features/llm/claude/test_claude_function_calling_default.ts index bee202e..0ad86c6 100644 --- a/test/features/llm/claude/test_claude_function_calling_default.ts +++ b/test/features/llm/claude/test_claude_function_calling_default.ts @@ -13,6 +13,7 @@ export const test_claude_function_calling_default = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/claude/test_claude_function_calling_example.ts b/test/features/llm/claude/test_claude_function_calling_example.ts index aa24a05..25a4fa7 100644 --- a/test/features/llm/claude/test_claude_function_calling_example.ts +++ b/test/features/llm/claude/test_claude_function_calling_example.ts @@ -13,6 +13,7 @@ export const test_claude_function_calling_example = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/claude/test_claude_function_calling_recursive.ts b/test/features/llm/claude/test_claude_function_calling_recursive.ts index 23def97..f5dec7c 100644 --- a/test/features/llm/claude/test_claude_function_calling_recursive.ts +++ b/test/features/llm/claude/test_claude_function_calling_recursive.ts @@ -13,6 +13,7 @@ export const test_claude_function_calling_recursive = () => name: "composeCategories", description: "Compose categories from the input.", collection: typia.json.schemas<[{ input: IShoppingCategory[] }]>(), + validate: typia.createValidate<[{ input: IShoppingCategory[] }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/claude/test_claude_function_calling_tags.ts b/test/features/llm/claude/test_claude_function_calling_tags.ts index 398a2a8..bc6bec1 100644 --- a/test/features/llm/claude/test_claude_function_calling_tags.ts +++ b/test/features/llm/claude/test_claude_function_calling_tags.ts @@ -13,6 +13,7 @@ export const test_claude_function_calling_tags = (): Promise => name: "reserve", description: "Reserve some opening time.", collection: typia.json.schemas<[{ input: OpeningTime }]>(), + validate: typia.createValidate<[{ input: OpeningTime }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/claude/test_claude_function_calling_union.ts b/test/features/llm/claude/test_claude_function_calling_union.ts index 1a75e93..4263774 100644 --- a/test/features/llm/claude/test_claude_function_calling_union.ts +++ b/test/features/llm/claude/test_claude_function_calling_union.ts @@ -13,6 +13,7 @@ export const test_claude_function_calling_union = (): Promise => name: "draw", description: "Draw a shape with following geometry.", collection: typia.json.schemas<[{ input: Shape }]>(), + validate: typia.createValidate<[{ input: Shape }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/gemini/test_gemini_function_calling_default.ts b/test/features/llm/gemini/test_gemini_function_calling_default.ts index 7da9068..324b86d 100644 --- a/test/features/llm/gemini/test_gemini_function_calling_default.ts +++ b/test/features/llm/gemini/test_gemini_function_calling_default.ts @@ -9,6 +9,7 @@ export const test_gemini_function_calling_default = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/gemini/test_gemini_function_calling_example.ts b/test/features/llm/gemini/test_gemini_function_calling_example.ts index bff2546..a565d45 100644 --- a/test/features/llm/gemini/test_gemini_function_calling_example.ts +++ b/test/features/llm/gemini/test_gemini_function_calling_example.ts @@ -12,6 +12,7 @@ export const test_gemini_function_calling_example = (): Promise => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_additionalProperties.ts b/test/features/llm/llama/test_llama_function_calling_additionalProperties.ts index 95e0d9e..9af047a 100644 --- a/test/features/llm/llama/test_llama_function_calling_additionalProperties.ts +++ b/test/features/llm/llama/test_llama_function_calling_additionalProperties.ts @@ -11,6 +11,7 @@ export const test_llama_function_calling_additionalProperties = name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_default.ts b/test/features/llm/llama/test_llama_function_calling_default.ts index c8ebaf1..cbc9a67 100644 --- a/test/features/llm/llama/test_llama_function_calling_default.ts +++ b/test/features/llm/llama/test_llama_function_calling_default.ts @@ -10,6 +10,7 @@ export const test_llama_function_calling_default = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_example.ts b/test/features/llm/llama/test_llama_function_calling_example.ts index a521f36..4ba1448 100644 --- a/test/features/llm/llama/test_llama_function_calling_example.ts +++ b/test/features/llm/llama/test_llama_function_calling_example.ts @@ -10,6 +10,7 @@ export const test_llama_function_calling_example = () => name: "enrollPerson", description: "Enroll a person to the restaurant reservation list.", collection: typia.json.schemas<[{ input: IPerson }]>(), + validate: typia.createValidate<[{ input: IPerson }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_nullable.ts b/test/features/llm/llama/test_llama_function_calling_nullable.ts index 5c65ed7..a0ede5e 100644 --- a/test/features/llm/llama/test_llama_function_calling_nullable.ts +++ b/test/features/llm/llama/test_llama_function_calling_nullable.ts @@ -13,6 +13,7 @@ export const test_llama_function_calling_nullable = (): Promise => name: "drawPolygon", description: "Draw a polygon with given geometry.", collection: typia.json.schemas<[{ input: IPolygon }]>(), + validate: typia.createValidate<[{ input: IPolygon }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_recursive.ts b/test/features/llm/llama/test_llama_function_calling_recursive.ts index 46fb5f0..fa2840a 100644 --- a/test/features/llm/llama/test_llama_function_calling_recursive.ts +++ b/test/features/llm/llama/test_llama_function_calling_recursive.ts @@ -10,6 +10,7 @@ export const test_llama_function_calling_recursive = () => name: "composeCategories", description: "Compose categories from the input.", collection: typia.json.schemas<[{ input: IShoppingCategory[] }]>(), + validate: typia.createValidate<[{ input: IShoppingCategory[] }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_tags.ts b/test/features/llm/llama/test_llama_function_calling_tags.ts index 50ef955..8dc7650 100644 --- a/test/features/llm/llama/test_llama_function_calling_tags.ts +++ b/test/features/llm/llama/test_llama_function_calling_tags.ts @@ -10,6 +10,7 @@ export const test_llama_function_calling_tags = (): Promise => name: "reserve", description: "Reserve some opening time.", collection: typia.json.schemas<[{ input: OpeningTime }]>(), + validate: typia.createValidate<[{ input: OpeningTime }]>(), texts: [ { role: "assistant", diff --git a/test/features/llm/llama/test_llama_function_calling_union.ts b/test/features/llm/llama/test_llama_function_calling_union.ts index f55f48e..18cd654 100644 --- a/test/features/llm/llama/test_llama_function_calling_union.ts +++ b/test/features/llm/llama/test_llama_function_calling_union.ts @@ -13,6 +13,7 @@ export const test_llama_function_calling_union = (): Promise => name: "draw", description: "Draw a shape with following geometry.", collection: typia.json.schemas<[{ input: Shape }]>(), + validate: typia.createValidate<[{ input: Shape }]>(), texts: [ { role: "assistant", diff --git a/test/utils/ChatGptFunctionCaller.ts b/test/utils/ChatGptFunctionCaller.ts index 5adee12..3f6d09c 100644 --- a/test/utils/ChatGptFunctionCaller.ts +++ b/test/utils/ChatGptFunctionCaller.ts @@ -1,4 +1,4 @@ -import { ArrayUtil, TestValidator } from "@nestia/e2e"; +import { TestValidator } from "@nestia/e2e"; import { IChatGptSchema, IOpenApiSchemaError, @@ -7,25 +7,41 @@ import { } from "@samchon/openapi"; import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer"; import OpenAI from "openai"; -import typia, { IJsonSchemaCollection } from "typia"; +import typia, { IJsonSchemaCollection, IValidation } from "typia"; import { TestGlobal } from "../TestGlobal"; import { ILlmTextPrompt } from "../structures/ILlmTextPrompt"; export namespace ChatGptFunctionCaller { - export const test = async (props: { + export interface IProps { name: string; description: string; collection: IJsonSchemaCollection; + validate: (input: any) => IValidation; texts: ILlmTextPrompt[]; handleCompletion: (input: any) => Promise; handleParameters?: ( parameters: IChatGptSchema.IParameters, ) => Promise; config?: Partial; - }): Promise => { + } + + export const test = async (props: IProps): Promise => { if (TestGlobal.env.CHATGPT_API_KEY === undefined) return; + let result: IValidation | undefined = undefined; + for (let i: number = 0; i < 3; ++i) { + if (result && result.success === true) break; + result = await step(props, TestGlobal.env.CHATGPT_API_KEY, result); + } + await props.handleCompletion(result?.data); + }; + + const step = async ( + props: IProps, + apiKey: string, + previous?: IValidation.IFailure, + ): Promise> => { const parameters: IResult = LlmSchemaComposer.parameters("chatgpt")({ components: props.collection.components, @@ -45,12 +61,30 @@ export namespace ChatGptFunctionCaller { await props.handleParameters(parameters.value); const client: OpenAI = new OpenAI({ - apiKey: TestGlobal.env.CHATGPT_API_KEY, + apiKey, }); const completion: OpenAI.ChatCompletion = await client.chat.completions.create({ model: "gpt-4o", - messages: props.texts, + messages: previous + ? [ + ...props.texts.slice(0, -1), + + { + role: "assistant", + content: [ + "You A.I. assistant has composed wrong typed arguments.", + "", + "Here is the detailed list of type errors. Review and correct them at the next function calling.", + "", + "```json", + JSON.stringify(previous.errors, null, 2), + "```", + ].join("\n"), + } satisfies OpenAI.ChatCompletionMessageParam, + ...props.texts.slice(-1), + ] + : props.texts, tools: [ { type: "function", @@ -69,12 +103,14 @@ export namespace ChatGptFunctionCaller { completion.choices[0].message.tool_calls ?? []; if (toolCalls.length === 0) throw new Error("ChatGPT has not called any function."); - await ArrayUtil.asyncForEach(toolCalls)(async (call) => { + + const results: IValidation[] = toolCalls.map((call) => { TestValidator.equals("name")(call.function.name)(props.name); const { input } = typia.assert<{ input: any }>( JSON.parse(call.function.arguments), ); - await props.handleCompletion(input); + return props.validate(input); }); + return results.find((r) => r.success === true) ?? results[0]; }; } diff --git a/test/utils/ClaudeFunctionCaller.ts b/test/utils/ClaudeFunctionCaller.ts index 940371e..25d627d 100644 --- a/test/utils/ClaudeFunctionCaller.ts +++ b/test/utils/ClaudeFunctionCaller.ts @@ -1,5 +1,5 @@ import Anthropic from "@anthropic-ai/sdk"; -import { ArrayUtil, TestValidator } from "@nestia/e2e"; +import { TestValidator } from "@nestia/e2e"; import { ILlmSchema, IOpenApiSchemaError, @@ -7,28 +7,44 @@ import { OpenApi, } from "@samchon/openapi"; import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer"; -import typia, { IJsonSchemaCollection } from "typia"; +import typia, { IJsonSchemaCollection, IValidation } from "typia"; import { TestGlobal } from "../TestGlobal"; import { ILlmTextPrompt } from "../structures/ILlmTextPrompt"; export namespace ClaudeFunctionCaller { - export const test = async < - Model extends "chatgpt" | "claude" | "gemini", - >(props: { + export interface IProps { model: Model; config?: Partial; name: string; description: string; collection: IJsonSchemaCollection; texts: ILlmTextPrompt[]; + validate: (input: any) => IValidation; handleCompletion: (input: any) => Promise; handleParameters?: ( parameters: ILlmSchema.ModelParameters[Model], ) => Promise; - }): Promise => { + } + + export const test = async ( + props: IProps, + ): Promise => { if (TestGlobal.env.CLAUDE_API_KEY === undefined) return; + let result: IValidation | undefined = undefined; + for (let i: number = 0; i < 3; ++i) { + if (result && result.success === true) break; + result = await step(props, TestGlobal.env.CLAUDE_API_KEY, result); + } + await props.handleCompletion(result?.data); + }; + + const step = async ( + props: IProps, + apiKey: string, + previous?: IValidation.IFailure, + ): Promise> => { const parameters: IResult< ILlmSchema.ModelParameters[Model], IOpenApiSchemaError @@ -50,12 +66,30 @@ export namespace ClaudeFunctionCaller { await props.handleParameters(parameters.value); const client: Anthropic = new Anthropic({ - apiKey: TestGlobal.env.CLAUDE_API_KEY, + apiKey, }); const completion: Anthropic.Message = await client.messages.create({ model: "claude-3-5-sonnet-latest", max_tokens: 8_192, - messages: props.texts, + messages: previous + ? [ + ...props.texts.slice(0, -1), + + { + role: "assistant", + content: [ + "You A.I. assistant has composed wrong typed arguments.", + "", + "Here is the detailed list of type errors. Review and correct them at the next function calling.", + "", + "```json", + JSON.stringify(previous.errors, null, 2), + "```", + ].join("\n"), + } satisfies Anthropic.MessageParam, + ...props.texts.slice(-1), + ] + : props.texts, tools: [ { name: props.name, @@ -74,10 +108,12 @@ export namespace ClaudeFunctionCaller { ); if (toolCalls.length === 0) throw new Error("Claude has not called any function."); - await ArrayUtil.asyncForEach(toolCalls)(async (call) => { + + const results: IValidation[] = toolCalls.map((call) => { TestValidator.equals("name")(call.name)(props.name); const { input } = typia.assert<{ input: any }>(call.input); - await props.handleCompletion(input); + return props.validate(input); }); + return results.find((r) => r.success === true) ?? results[0]; }; } diff --git a/test/utils/GeminiFunctionCaller.ts b/test/utils/GeminiFunctionCaller.ts index 3d81bbb..432b724 100644 --- a/test/utils/GeminiFunctionCaller.ts +++ b/test/utils/GeminiFunctionCaller.ts @@ -5,7 +5,7 @@ import { GenerativeModel, GoogleGenerativeAI, } from "@google/generative-ai"; -import { ArrayUtil, TestValidator } from "@nestia/e2e"; +import { TestValidator } from "@nestia/e2e"; import { IGeminiSchema, IOpenApiSchemaError, @@ -13,23 +13,39 @@ import { OpenApi, } from "@samchon/openapi"; import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer"; -import typia, { IJsonSchemaCollection } from "typia"; +import typia, { IJsonSchemaCollection, IValidation } from "typia"; import { TestGlobal } from "../TestGlobal"; import { ILlmTextPrompt } from "../structures/ILlmTextPrompt"; export namespace GeminiFunctionCaller { - export const test = async (props: { + export interface IProps { name: string; description: string; collection: IJsonSchemaCollection; texts: ILlmTextPrompt[]; + validate: (input: any) => IValidation; handleCompletion: (input: any) => Promise; handleParameters?: (parameters: IGeminiSchema.IParameters) => Promise; config?: Partial; - }): Promise => { + } + + export const test = async (props: IProps): Promise => { if (TestGlobal.env.GEMINI_API_KEY === undefined) return; + let result: IValidation | undefined = undefined; + for (let i: number = 0; i < 3; ++i) { + if (result && result.success === true) break; + result = await step(props, TestGlobal.env.GEMINI_API_KEY, result); + } + await props.handleCompletion(result?.data); + }; + + const step = async ( + props: IProps, + apiKey: string, + previous?: IValidation.IFailure, + ): Promise> => { const parameters: IResult = LlmSchemaComposer.parameters("gemini")({ components: props.collection.components, @@ -47,12 +63,15 @@ export namespace GeminiFunctionCaller { if (props.handleParameters) await props.handleParameters(parameters.value); const model: GenerativeModel = new GoogleGenerativeAI( - TestGlobal.env.GEMINI_API_KEY, + apiKey, ).getGenerativeModel({ model: "gemini-1.5-pro", }); - const result: GenerateContentResult = await model.generateContent({ - contents: props.texts.map((p) => ({ + const completion: GenerateContentResult = await model.generateContent({ + contents: (previous + ? [...props.texts.slice(0, -1), ...props.texts.slice(-1)] + : props.texts + ).map((p) => ({ role: p.role === "assistant" ? "model" : p.role, parts: [ { @@ -79,13 +98,15 @@ export namespace GeminiFunctionCaller { }, }); - const toolCalls: FunctionCall[] = result.response.functionCalls() ?? []; + const toolCalls: FunctionCall[] = completion.response.functionCalls() ?? []; if (toolCalls.length === 0) throw new Error("Gemini has not called any function."); - await ArrayUtil.asyncForEach(toolCalls)(async (call) => { + + const results: IValidation[] = toolCalls.map((call) => { TestValidator.equals("name")(call.name)(props.name); const { input } = typia.assert<{ input: any }>(call.args); - await props.handleCompletion(input); + return props.validate(input); }); + return results.find((r) => r.success === true) ?? results[0]; }; } diff --git a/test/utils/LlamaFunctionCaller.ts b/test/utils/LlamaFunctionCaller.ts index 9db88f8..d66bbe1 100644 --- a/test/utils/LlamaFunctionCaller.ts +++ b/test/utils/LlamaFunctionCaller.ts @@ -1,4 +1,4 @@ -import { ArrayUtil, TestValidator } from "@nestia/e2e"; +import { TestValidator } from "@nestia/e2e"; import { ILlmSchema, IOpenApiSchemaError, @@ -7,27 +7,45 @@ import { } from "@samchon/openapi"; import { LlmSchemaComposer } from "@samchon/openapi/lib/composers/LlmSchemaComposer"; import OpenAI from "openai"; -import typia, { IJsonSchemaCollection } from "typia"; +import typia, { IJsonSchemaCollection, IValidation } from "typia"; // import { v4 } from "uuid"; import { TestGlobal } from "../TestGlobal"; import { ILlmTextPrompt } from "../structures/ILlmTextPrompt"; export namespace LlamaFunctionCaller { - export const test = async (props: { + export interface IProps { model: Model; config?: Partial; name: string; description: string; + validate: (input: any) => IValidation; collection: IJsonSchemaCollection; texts: ILlmTextPrompt[]; handleCompletion: (input: any) => Promise; handleParameters?: ( parameters: ILlmSchema.ModelParameters[Model], ) => Promise; - }): Promise => { + } + + export const test = async ( + props: IProps, + ): Promise => { if (TestGlobal.env.LLAMA_API_KEY === undefined) return; + let result: IValidation | undefined = undefined; + for (let i: number = 0; i < 3; ++i) { + if (result && result.success === true) break; + result = await step(props, TestGlobal.env.LLAMA_API_KEY, result); + } + await props.handleCompletion(result?.data); + }; + + const step = async ( + props: IProps, + apiKey: string, + previous?: IValidation.IFailure, + ): Promise> => { const parameters: IResult< ILlmSchema.IParameters, IOpenApiSchemaError @@ -49,13 +67,31 @@ export namespace LlamaFunctionCaller { await props.handleParameters(parameters.value); const client: OpenAI = new OpenAI({ - apiKey: TestGlobal.env.LLAMA_API_KEY, + apiKey, baseURL: "https://api.llama-api.com", }); const completion: OpenAI.ChatCompletion = await client.chat.completions.create({ model: "llama3.2-90b-vision", - messages: props.texts, + messages: previous + ? [ + ...props.texts.slice(0, -1), + + { + role: "assistant", + content: [ + "You A.I. assistant has composed wrong typed arguments.", + "", + "Here is the detailed list of type errors. Review and correct them at the next function calling.", + "", + "```json", + JSON.stringify(previous.errors, null, 2), + "```", + ].join("\n"), + } satisfies OpenAI.ChatCompletionMessageParam, + ...props.texts.slice(-1), + ] + : props.texts, tools: [ { type: "function", @@ -90,12 +126,14 @@ export namespace LlamaFunctionCaller { ]; if (toolCalls.length === 0) throw new Error("Llama has not called any function."); - await ArrayUtil.asyncForEach(toolCalls)(async (call) => { + + const results: IValidation[] = toolCalls.map((call) => { TestValidator.equals("name")(call.function.name)(props.name); const { input } = typia.assert<{ input: any }>( JSON.parse(call.function.arguments), ); - await props.handleCompletion(input); + return props.validate(input); }); + return results.find((r) => r.success === true) ?? results[0]; }; } diff --git a/test/utils/ShoppingSalePrompt.ts b/test/utils/ShoppingSalePrompt.ts index c8aa069..5a430c4 100644 --- a/test/utils/ShoppingSalePrompt.ts +++ b/test/utils/ShoppingSalePrompt.ts @@ -20,6 +20,7 @@ export namespace ShoppingSalePrompt { description: "Create a sale and returns the detailed information.", collection: typia.json.schemas<[{ input: IShoppingSale.ICreate }, IShoppingSale]>(), + validate: typia.createValidate(), }); export const texts = async (