From 26829d00e026f5e153577d7966fd953b170e6e6b Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Wed, 16 Oct 2024 15:14:19 +1300 Subject: [PATCH 1/8] Fix repl exit --- src/util.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util.ts b/src/util.ts index f79fa05..4302e52 100644 --- a/src/util.ts +++ b/src/util.ts @@ -28,7 +28,8 @@ export function setSpinner(ora: Ora) { export function getPrompt(): string | null { const response = prompt(brightBlue(`Enter a message: `)); - if (response === "/bye") { + // null occurs with ctrl+c + if (response === "/bye" || response === null) { Deno.exit(0); } From 2083cce26452a1c9e8c1042f55f050fd8151608d Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 11:24:20 +1300 Subject: [PATCH 2/8] Add init command --- src/index.ts | 37 ++++++++-- src/{ => subcommands}/bundle.ts | 6 +- src/{ => subcommands}/bundle_test.ts | 2 +- src/{ => subcommands}/httpCli.ts | 4 +- src/{ => subcommands}/info.ts | 8 +-- src/subcommands/init.ts | 100 +++++++++++++++++++++++++++ src/util.ts | 7 +- subquery-delegator/manifest.ts | 4 +- 8 files changed, 149 insertions(+), 19 deletions(-) rename src/{ => subcommands}/bundle.ts (97%) rename src/{ => subcommands}/bundle_test.ts (95%) rename src/{ => subcommands}/httpCli.ts (92%) rename src/{ => subcommands}/info.ts (90%) create mode 100644 src/subcommands/init.ts diff --git a/src/index.ts b/src/index.ts index 8987793..f7156c4 100755 --- a/src/index.ts +++ b/src/index.ts @@ -18,8 +18,7 @@ import yargs, { import { IPFSClient } from "./ipfs.ts"; import ora from "ora"; -import { setSpinner } from "./util.ts"; - +import { getPrompt, setSpinner } from "./util.ts"; const DEFAULT_PORT = 7827; const sharedArgs = { @@ -112,7 +111,7 @@ yargs(Deno.args) }, async (argv) => { try { - const { projectInfo } = await import("./info.ts"); + const { projectInfo } = await import("./subcommands/info.ts"); await projectInfo(argv.project, ipfsFromArgs(argv), argv.json); Deno.exit(0); } catch (e) { @@ -177,7 +176,7 @@ yargs(Deno.args) }, }, async (argv) => { - const { httpCli } = await import("./httpCli.ts"); + const { httpCli } = await import("./subcommands/httpCli.ts"); await httpCli(argv.host); }, ) @@ -193,7 +192,7 @@ yargs(Deno.args) }, async (argv) => { try { - const { publishProject } = await import("./bundle.ts"); + const { publishProject } = await import("./subcommands/bundle.ts"); if (argv.silent) { setSpinner(ora({ isSilent: true })); } @@ -211,6 +210,34 @@ yargs(Deno.args) } }, ) + .command( + "init", + "Create a new project skeleton", + { + name: { + description: + "The name of your project, this will create a directory with that name.", + type: "string", + }, + model: { + description: "The LLM model you wish to use", + type: "string", + }, + }, + async (argv) => { + try { + argv.name ??= getPrompt("Enter a project name: "); + argv.model ??= getPrompt("Enter a LLM model", "llama3.1"); + + const { initProject } = await import("./subcommands/init.ts"); + + await initProject({ name: argv.name, model: argv.model }); + } catch (e) { + console.log(e); + Deno.exit(1); + } + }, + ) // .fail(() => {}) // Disable logging --help if theres an error with a command // TODO need to fix so it only logs when error is with yargs .help() .argv; diff --git a/src/bundle.ts b/src/subcommands/bundle.ts similarity index 97% rename from src/bundle.ts rename to src/subcommands/bundle.ts index d4d7fcd..e15e359 100644 --- a/src/bundle.ts +++ b/src/subcommands/bundle.ts @@ -3,15 +3,15 @@ import { Tar } from "@std/archive/tar"; import { walk } from "@std/fs/walk"; import { dirname } from "@std/path/dirname"; import { Buffer } from "@std/io/buffer"; -import type { IPFSClient } from "./ipfs.ts"; +import type { IPFSClient } from "../ipfs.ts"; // Supporting WASM would allow dropping `--allow-run` option but its not currently supported https://github.com/evanw/esbuild/pull/2968 // import * as esbuild from "https://deno.land/x/esbuild@v0.24.0/wasm.js"; // import * as esbuild from "esbuild"; import { denoPlugins } from "@luca/esbuild-deno-loader"; import { toReadableStream } from "@std/io/to-readable-stream"; import { readerFromStreamReader } from "@std/io/reader-from-stream-reader"; -import { getSpinner } from "./util.ts"; -import { Loader } from "./loader.ts"; +import { getSpinner } from "../util.ts"; +import { Loader } from "../loader.ts"; export async function publishProject( projectPath: string, diff --git a/src/bundle_test.ts b/src/subcommands/bundle_test.ts similarity index 95% rename from src/bundle_test.ts rename to src/subcommands/bundle_test.ts index d1a65b8..8354d1e 100644 --- a/src/bundle_test.ts +++ b/src/subcommands/bundle_test.ts @@ -1,6 +1,6 @@ import { generateBundle, publishProject } from "./bundle.ts"; import { expect } from "jsr:@std/expect"; -import { IPFSClient } from "./ipfs.ts"; +import { IPFSClient } from "../ipfs.ts"; Deno.test("Generates a bundle", async () => { const code = await generateBundle("./subquery-delegator/index.ts"); diff --git a/src/httpCli.ts b/src/subcommands/httpCli.ts similarity index 92% rename from src/httpCli.ts rename to src/subcommands/httpCli.ts index 93eeab3..dcc7599 100644 --- a/src/httpCli.ts +++ b/src/subcommands/httpCli.ts @@ -1,8 +1,8 @@ import ora from "ora"; import type { Message } from "ollama"; import { brightMagenta, brightRed } from "@std/fmt/colors"; -import type { ChatResponse } from "./http.ts"; -import { getPrompt } from "./util.ts"; +import type { ChatResponse } from "../http.ts"; +import { getPrompt } from "../util.ts"; export async function httpCli(host: string): Promise { const messages: Message[] = []; diff --git a/src/info.ts b/src/subcommands/info.ts similarity index 90% rename from src/info.ts rename to src/subcommands/info.ts index 55ebe2a..adb4b96 100644 --- a/src/info.ts +++ b/src/subcommands/info.ts @@ -1,8 +1,8 @@ import { brightBlue, brightMagenta } from "@std/fmt/colors"; -import { getDefaultSandbox } from "./sandbox/index.ts"; -import type { ProjectManifest } from "./project/project.ts"; -import type { IPFSClient } from "./ipfs.ts"; -import { Loader } from "./loader.ts"; +import { getDefaultSandbox } from "../sandbox/index.ts"; +import type { ProjectManifest } from "../project/project.ts"; +import type { IPFSClient } from "../ipfs.ts"; +import { Loader } from "../loader.ts"; type StaticProject = ProjectManifest & { tools?: string[]; diff --git a/src/subcommands/init.ts b/src/subcommands/init.ts new file mode 100644 index 0000000..85608cf --- /dev/null +++ b/src/subcommands/init.ts @@ -0,0 +1,100 @@ +import { resolve } from "@std/path/resolve"; + +type Options = { + name: string; + model: string; +}; + +const manifestTemplate = (model: string): string => { + return `import type { ProjectManifest } from "jsr:@subql/ai-app-framework"; +import { ConfigType } from "./project.ts"; + +const project: ProjectManifest = { + specVersion: "0.0.1", + // Specify any hostnames your tools will make network requests too + endpoints: [], + + // If you wish to add RAG data to your project you can reference the DB here + // vectorStorage: { + // type: "lancedb", + // path: "./db", + // }, + + // Your projects runtime configuration options + config: JSON.parse(JSON.stringify(ConfigType)), // Convert to JSON Schema + model: "${model}", + entry: "./project.ts", +}; + +export default project;`; +}; + +const projectTemplate = (): string => { + return `import { type Static, Type } from "npm:@sinclair/typebox"; +import { type Project, type ProjectEntry, FunctionTool } from "jsr:@subql/ai-app-framework"; + +export const ConfigType = Type.Object({ + EXAMPLE_ENDPOINT: Type.String({ + default: 'https://example.com', + description: 'This is an example config option', + }), +}); + +export type Config = Static; + +class GreetingTool extends FunctionTool { + + description = \`This tool responds with a welcome greeting when the user shares their name.\`, + + parameters = { + type: "object", + required: ["name"], + properties: { + name: { + type: "string", + description: + "The name of the user", + }, + }, + }; + + async call({ name }: { name: string }): Promise { + return \`Hello ${name}, nice to meet you\`; + } +} + +// deno-lint-ignore require-await +const entrypoint: ProjectEntry = async (config: Config): Promise => { + return { + tools: [ + new GreetingTool(), + ], + systemPrompt: \`You are an agent designed to help a user answer their questions. + Given an input question, use the available tools to answer the users question quickly and concisely. + You answer must use the result of the tools available. + Do not mention that you used a tool or the name of a tool. + If you need more information to answer the question, ask the user for more details.\`, + }; +}; + +export default entrypoint;`; +}; + +export async function initProject(opts: Options): Promise { + const dir = resolve(Deno.cwd(), opts.name); + + try { + await Deno.mkdir(dir); + } catch (e) { + if (!(e instanceof Deno.errors.AlreadyExists)) { + throw e; + } + throw new Error(`A directory with the name ${opts.name} already exists`); + } + + await Deno.writeTextFile( + resolve(dir, "manifest.ts"), + manifestTemplate(opts.model), + ); + await Deno.writeTextFile(resolve(dir, "project.ts"), projectTemplate()); +} diff --git a/src/util.ts b/src/util.ts index 4302e52..504c727 100644 --- a/src/util.ts +++ b/src/util.ts @@ -25,8 +25,11 @@ export function setSpinner(ora: Ora) { spinner = ora; } -export function getPrompt(): string | null { - const response = prompt(brightBlue(`Enter a message: `)); +export function getPrompt( + message = "Enter a message: ", + defaultValue?: string, +): string { + const response = prompt(brightBlue(message), defaultValue); // null occurs with ctrl+c if (response === "/bye" || response === null) { diff --git a/subquery-delegator/manifest.ts b/subquery-delegator/manifest.ts index 29238c5..07f5165 100644 --- a/subquery-delegator/manifest.ts +++ b/subquery-delegator/manifest.ts @@ -1,4 +1,4 @@ -import { type Config, ConfigType } from "./index.ts"; +import { type Config, ConfigType } from "./project.ts"; import type { ProjectManifest } from "../src/project/project.ts"; import { Value } from "@sinclair/typebox/value"; import { extractConfigHostNames } from "../src/util.ts"; @@ -14,7 +14,7 @@ const project: ProjectManifest = { }, config: JSON.parse(JSON.stringify(ConfigType)), // Convert to JSON Schema model: "llama3.1", - entry: "./index.ts", + entry: "./project.ts", }; export default project; From 95a02fddbe7337601c573bec2a50c3a66923e7ca Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 14:52:34 +1300 Subject: [PATCH 3/8] Allow setting embedding model --- src/app.ts | 2 +- src/project/project.ts | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/app.ts b/src/app.ts index 0481c2a..e2ddd7b 100644 --- a/src/app.ts +++ b/src/app.ts @@ -79,7 +79,7 @@ async function makeContext( if (!loadRes) throw new Error("Failed to load vector db"); const connection = await lancedb.connect(loadRes[0]); - return new Context(model, connection); + return new Context(model, connection, sandbox.manifest.embeddingsModel); } async function cli(runnerHost: RunnerHost): Promise { diff --git a/src/project/project.ts b/src/project/project.ts index 290f6c4..057a456 100644 --- a/src/project/project.ts +++ b/src/project/project.ts @@ -24,13 +24,23 @@ export const VectorConfig = Type.Object({ export const ProjectManifest = Type.Object({ specVersion: Type.Literal("0.0.1"), - model: Type.String(), - entry: Type.String(), + model: Type.String({ description: "The Ollama LLM model to be used" }), + embeddingsModel: Type.Optional(Type.String({ + description: "The Ollama LLM model to be used for vector embeddings", + })), + entry: Type.String({ + description: "File path to the project entrypoint", + }), vectorStorage: Type.Optional(Type.Object({ - type: Type.String(), - path: Type.String(), + type: Type.String({ + description: + "The type of vector storage, currently only lancedb is supported.", + }), + path: Type.String({ description: "The path to the db" }), })), - endpoints: Type.Optional(Type.Array(Type.String())), + endpoints: Type.Optional(Type.Array(Type.String({ + description: "Allowed endpoints the tools are allowed to make requests to", + }))), config: Type.Optional(Type.Any()), // TODO how can this be a JSON Schema type? }); From 141e071f33b6418ac708c99513f864f49a0a788b Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 15:50:32 +1300 Subject: [PATCH 4/8] Add RagTool --- src/tools/ragTool.ts | 42 +++++++++++++++++++++++++++++++++++++ src/{ => tools}/tool.ts | 0 subquery-delegator/tools.ts | 31 +++++---------------------- 3 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 src/tools/ragTool.ts rename src/{ => tools}/tool.ts (100%) diff --git a/src/tools/ragTool.ts b/src/tools/ragTool.ts new file mode 100644 index 0000000..c61eb8a --- /dev/null +++ b/src/tools/ragTool.ts @@ -0,0 +1,42 @@ +import type { IContext } from "../context/context.ts"; +import { FunctionTool } from "./tool.ts"; + +export class RagTool extends FunctionTool { + /** + * RagTool is a default implementation allowing querying RAG data + * @param tableName The name of the table to query + * @param column The column on the table to extract results from + */ + constructor( + readonly tableName: string, + readonly column: string, + ) { + super(); + } + + get description(): string { + return `This tool gets relevant information from the ${this.tableName}. It returns a list of results separated by newlines.`; + } + + parameters = { + type: "object", + required: ["query"], + properties: { + account: { + type: "string", + description: "A search string, generally the users prompt", + }, + }, + }; + + async call({ query }: { query: string }, ctx: IContext): Promise { + const vector = await ctx.computeQueryEmbedding(query); + const raw = await ctx.vectorSearch(this.tableName, vector); + + const res = raw.map((r) => r[this.column]) + .filter((c) => !!c) + .join("\n"); + + return res; + } +} diff --git a/src/tool.ts b/src/tools/tool.ts similarity index 100% rename from src/tool.ts rename to src/tools/tool.ts diff --git a/subquery-delegator/tools.ts b/subquery-delegator/tools.ts index 90062cb..effd9e9 100644 --- a/subquery-delegator/tools.ts +++ b/subquery-delegator/tools.ts @@ -5,9 +5,9 @@ import { formatUnits, toBigInt, } from "npm:ethers"; -import { FunctionTool } from "../src/tool.ts"; +import { FunctionTool } from "../src/tools/tool.ts"; import { grahqlRequest } from "./utils.ts"; -import type { IContext } from "../src/context/context.ts"; +import { RagTool } from "../src/tools/ragTool.ts"; type Amount = { era: number; @@ -401,29 +401,8 @@ export class BetterIndexerApy extends FunctionTool { } } -export class SubqueryDocs extends FunctionTool { - description = - `This tool gets relevant information from the Subquery Docs. It returns a list of results separated by newlines.`; - - parameters = { - type: "object", - required: ["query"], - properties: { - account: { - type: "string", - description: "A search string, generally the users prompt", - }, - }, - }; - - async call({ query }: { query: string }, ctx: IContext): Promise { - const vector = await ctx.computeQueryEmbedding(query); - const raw = await ctx.vectorSearch("subql-docs", vector); - - const res = raw.map((r) => r.content) - .filter((c) => !!c) - .join("\n"); - - return res; +export class SubqueryDocs extends RagTool { + constructor() { + super("subql-docs", "content"); } } From beb77bfd59e952d27229154f807a6ad74c6e0976 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 16:23:48 +1300 Subject: [PATCH 5/8] Fix import path --- src/loader_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loader_test.ts b/src/loader_test.ts index a2209ec..8306085 100644 --- a/src/loader_test.ts +++ b/src/loader_test.ts @@ -2,7 +2,7 @@ import { expect } from "@std/expect/expect"; import { getOSTempDir, pullContent } from "./loader.ts"; import { resolve } from "@std/path/resolve"; import { IPFSClient } from "./ipfs.ts"; -import { tarDir } from "./bundle.ts"; +import { tarDir } from "./subcommands/bundle.ts"; const ipfs = new IPFSClient( Deno.env.get("IPFS_ENDPOINT") ?? From 08674b9e9ca9be4f124fe2962603f03547ed9855 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 17:02:32 +1300 Subject: [PATCH 6/8] Implement timeouts for tool calls --- src/app.ts | 3 +- src/index.ts | 7 ++ src/runner.ts | 18 ++-- src/sandbox/index.ts | 3 +- src/sandbox/webWorker/webWorkerSandbox.ts | 124 +++++++++++++++------- src/util.ts | 6 ++ 6 files changed, 113 insertions(+), 48 deletions(-) diff --git a/src/app.ts b/src/app.ts index e2ddd7b..5268bc2 100644 --- a/src/app.ts +++ b/src/app.ts @@ -20,6 +20,7 @@ export async function runApp(config: { port: number; ipfs: IPFSClient; forceReload?: boolean; + toolTimeout: number; }): Promise { const model = new Ollama({ host: config.host }); @@ -30,7 +31,7 @@ export async function runApp(config: { config.forceReload, ); - const sandbox = await getDefaultSandbox(loader); + const sandbox = await getDefaultSandbox(loader, config.toolTimeout); const ctx = await makeContext( sandbox, diff --git a/src/index.ts b/src/index.ts index f7156c4..845dfa9 100755 --- a/src/index.ts +++ b/src/index.ts @@ -80,6 +80,12 @@ yargs(Deno.args) type: "boolean", default: false, }, + toolTimeout: { + description: + "Set a limit for how long a tool can take to run, unit is MS", + type: "number", + default: 10_000, // 10s + }, }, async (argv) => { try { @@ -91,6 +97,7 @@ yargs(Deno.args) port: argv.port, ipfs: ipfsFromArgs(argv), forceReload: argv.forceReload, + toolTimeout: argv.toolTimeout, }); } catch (e) { console.log(e); diff --git a/src/runner.ts b/src/runner.ts index ea06ecc..a378e14 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -61,13 +61,17 @@ export class Runner { // Run tools and use their responses const toolResponses = await Promise.all( (res.message.tool_calls ?? []).map(async (toolCall) => { - const res = await this.sandbox.runTool( - toolCall.function.name, - toolCall.function.arguments, - this.#context, - ); - - return res; + try { + return await this.sandbox.runTool( + toolCall.function.name, + toolCall.function.arguments, + this.#context, + ); + } catch (e: unknown) { + console.error(`Tool call failed: ${e}`); + // Don't throw the error this will exit the application, instead pass the message back to the LLM + return (e as Error).message; + } }), ); diff --git a/src/sandbox/index.ts b/src/sandbox/index.ts index d194cd2..c695a3a 100644 --- a/src/sandbox/index.ts +++ b/src/sandbox/index.ts @@ -9,10 +9,11 @@ export * from "./unsafeSandbox.ts"; export function getDefaultSandbox( loader: Loader, + timeout: number, ): Promise { // return UnsafeSandbox.create(loader); - return WebWorkerSandbox.create(loader); + return WebWorkerSandbox.create(loader, timeout); } export { WebWorkerSandbox }; diff --git a/src/sandbox/webWorker/webWorkerSandbox.ts b/src/sandbox/webWorker/webWorkerSandbox.ts index b66419f..b75f58f 100644 --- a/src/sandbox/webWorker/webWorkerSandbox.ts +++ b/src/sandbox/webWorker/webWorkerSandbox.ts @@ -10,12 +10,14 @@ import { CtxComputeQueryEmbedding, CtxVectorSearch, Init, + type IProjectJson, Load, } from "./messages.ts"; import { extractConfigHostNames, loadRawConfigFromEnv, type Source, + timeout, } from "../../util.ts"; import type { IContext } from "../../context/context.ts"; import type { ProjectManifest } from "../../project/project.ts"; @@ -57,13 +59,54 @@ function getPermisionsForSource( } } -export class WebWorkerSandbox implements ISandbox { - #connection: rpc.MessageConnection; +async function workerFactory( + manifest: ProjectManifest, + entryPath: string, + config: Record, + permissions: Deno.PermissionOptionsObject, +): Promise<[Worker, rpc.MessageConnection, IProjectJson]> { + const w = new Worker( + import.meta.resolve("./webWorker.ts"), + { + type: "module", + deno: { + permissions: permissions, + }, + }, + ); + + // Setup a JSON RPC for interaction to the worker + const conn = rpc.createMessageConnection( + new BrowserMessageReader(w), + new BrowserMessageWriter(w), + ); + + conn.listen(); + await conn.sendRequest(Load, entryPath); + + const pJson = await conn.sendRequest( + Init, + manifest, + config, + ); + + return [w, conn, pJson]; +} + +export class WebWorkerSandbox implements ISandbox { #tools: Tool[]; + #initWorker: () => ReturnType; + /** + * Create a new WebWorkerSandbox + * @param loader The loader for loading any project resources + * @param timeout Tool call timeout in MS + * @returns A sandbox instance + */ public static async create( loader: Loader, + timeout: number, ): Promise { const [manifestPath, manifest, source] = await loader.getManifest(); const config = loadRawConfigFromEnv(manifest.config); @@ -78,55 +121,42 @@ export class WebWorkerSandbox implements ISandbox { ]), ]; - const w = new Worker( - import.meta.resolve("./webWorker.ts"), - { - type: "module", - deno: { - permissions: { - ...permissions, - env: false, // Should be passed through in loadRawConfigFromEnv - net: hostnames, - run: false, - write: false, - }, - }, - }, - ); - - // Setup a JSON RPC for interaction to the worker - const conn = rpc.createMessageConnection( - new BrowserMessageReader(w), - new BrowserMessageWriter(w), - ); - - conn.listen(); - const [entryPath] = await loader.getProject(); - await conn.sendRequest(Load, entryPath); - const { tools, systemPrompt } = await conn.sendRequest( - Init, - manifest, - config, - ); + const initProjectWorker = () => + workerFactory( + manifest, + entryPath, + config as Record, + { + ...permissions, + env: false, + net: hostnames, + run: false, + write: false, + }, + ); + + const [_worker, _conn, { tools, systemPrompt }] = await initProjectWorker(); return new WebWorkerSandbox( - conn, manifest, systemPrompt, tools, + initProjectWorker, + timeout, ); } private constructor( - connection: rpc.MessageConnection, readonly manifest: ProjectManifest, readonly systemPrompt: string, tools: Tool[], + initWorker: () => ReturnType, + readonly timeout: number = 100, ) { this.#tools = tools; - this.#connection = connection; + this.#initWorker = initWorker; } // deno-lint-ignore require-await @@ -134,19 +164,35 @@ export class WebWorkerSandbox implements ISandbox { return this.#tools; } - runTool(toolName: string, args: unknown, ctx: IContext): Promise { + async runTool( + toolName: string, + args: unknown, + ctx: IContext, + ): Promise { + // Create a worker just for the tool call, this is so we can terminate if it exceeds the timeout. + const [worker, conn] = await this.#initWorker(); + // Connect up context so sandbox can call application - this.#connection.onRequest(CtxVectorSearch, async (tableName, vector) => { + conn.onRequest(CtxVectorSearch, async (tableName, vector) => { const res = await ctx.vectorSearch(tableName, vector); // lancedb returns classes (Apache Arrow - Struct Row). It needs to be made serializable // This is done here as its specific to the webworker sandbox return res.map((r) => JSON.parse(JSON.stringify(r))); }); - this.#connection.onRequest(CtxComputeQueryEmbedding, async (query) => { + conn.onRequest(CtxComputeQueryEmbedding, async (query) => { return await ctx.computeQueryEmbedding(query); }); - return this.#connection.sendRequest(CallTool, toolName, args); + // Add timeout to the tool call, then clean up the worker. + return Promise.race([ + timeout(this.timeout).then(() => { + throw new Error(`Timeout calling tool ${toolName}`); + }), + conn.sendRequest(CallTool, toolName, args), + ]).finally(() => { + // Dispose of the worker, a new one will be created for each tool call + worker.terminate(); + }); } } diff --git a/src/util.ts b/src/util.ts index 504c727..9f820b3 100644 --- a/src/util.ts +++ b/src/util.ts @@ -79,3 +79,9 @@ export function extractConfigHostNames( // Make unique return [...new Set(hosts)]; } + +export function timeout(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(() => resolve(), ms); + }); +} From d59f315fb916467dbdadc756e91e146c0779fc8c Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 17:13:26 +1300 Subject: [PATCH 7/8] Fix tests, build errors --- src/project/project.ts | 1 + src/subcommands/bundle_test.ts | 4 ++-- src/tools/tool.ts | 6 +++--- subquery-delegator/manifest.ts | 23 +++++++++++++++++++++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/project/project.ts b/src/project/project.ts index 057a456..458216b 100644 --- a/src/project/project.ts +++ b/src/project/project.ts @@ -54,6 +54,7 @@ export const ProjectEntry = Type.Function( Type.Union([Project, Type.Promise(Project)]), ); +export type FunctionToolType = Static; export type ProjectManifest = Static; export type Project = Static; export type ProjectEntry = Static; diff --git a/src/subcommands/bundle_test.ts b/src/subcommands/bundle_test.ts index 8354d1e..fd21f70 100644 --- a/src/subcommands/bundle_test.ts +++ b/src/subcommands/bundle_test.ts @@ -3,7 +3,7 @@ import { expect } from "jsr:@std/expect"; import { IPFSClient } from "../ipfs.ts"; Deno.test("Generates a bundle", async () => { - const code = await generateBundle("./subquery-delegator/index.ts"); + const code = await generateBundle("./subquery-delegator/manifest.ts"); expect(code.length).toBeTruthy(); }); @@ -11,7 +11,7 @@ Deno.test("Generates a bundle", async () => { Deno.test("Publishing a project to ipfs", async () => { // WebWorkers don't work in tests, use the unsafe sandbox instead const cid = await publishProject( - "./subquery-delegator/project.ts", + "./subquery-delegator/manifest.ts", new IPFSClient( Deno.env.get("IPFS_ENDPOINT") ?? "https://unauthipfs.subquery.network/ipfs/api/v0", diff --git a/src/tools/tool.ts b/src/tools/tool.ts index 9ca7d96..3f5bb80 100644 --- a/src/tools/tool.ts +++ b/src/tools/tool.ts @@ -1,6 +1,6 @@ import type { Tool } from "ollama"; -import type { IFunctionTool } from "./project/project.ts"; -import type { IContext } from "./context/context.ts"; +import type { FunctionToolType } from "../project/project.ts"; +import type { IContext } from "../context/context.ts"; type Parameters = Tool["function"]["parameters"]; @@ -31,7 +31,7 @@ type OptionalParams

= { ]?: ExtractParameters

[K]; }; -export type ITool

= IFunctionTool & { +export type ITool

= FunctionToolType & { parameters: P; }; diff --git a/subquery-delegator/manifest.ts b/subquery-delegator/manifest.ts index 07f5165..e21ab30 100644 --- a/subquery-delegator/manifest.ts +++ b/subquery-delegator/manifest.ts @@ -1,7 +1,26 @@ import { type Config, ConfigType } from "./project.ts"; import type { ProjectManifest } from "../src/project/project.ts"; -import { Value } from "@sinclair/typebox/value"; -import { extractConfigHostNames } from "../src/util.ts"; +import { Value } from "npm:@sinclair/typebox/value"; + +// TODO import from the framework once published +/** Gets the host names of any urls in a record */ +export function extractConfigHostNames( + config: Record, +): string[] { + const hosts = Object.values(config) + .filter((v) => typeof v === "string") + .map((v) => { + try { + return new URL(v).hostname; + } catch (_e) { + return undefined; + } + }) + .filter((v) => !!v) as string[]; // Cast should be unnecessary with latest TS versions + + // Make unique + return [...new Set(hosts)]; +} const defaultConfig = Value.Default(ConfigType, {} as Config) as Config; From cf17ca9c6fbb18dbfdf09937ce15f9c2026ab95a Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 17 Oct 2024 17:14:57 +1300 Subject: [PATCH 8/8] Fix timeout for info command --- src/subcommands/info.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subcommands/info.ts b/src/subcommands/info.ts index adb4b96..9388285 100644 --- a/src/subcommands/info.ts +++ b/src/subcommands/info.ts @@ -15,7 +15,7 @@ export async function getProjectJson( sandboxFactory = getDefaultSandbox, ): Promise { try { - const sandbox = await sandboxFactory(loader); + const sandbox = await sandboxFactory(loader, 10_000); return { ...sandbox.manifest,