Tidy up

src/decorators.ts

@@ -0,0 +1,58 @@
+import { getSpinner } from "./util.ts";
+
+/** Creates a logging spinner using Ora for progress on a function */
+export function SpinnerLog(
+  messages: { start: string; success: string; fail: string },
+) {
+  // deno-lint-ignore no-explicit-any
+  return function (fn: any, _ctx: ClassMethodDecoratorContext) {
+    return function (...args: unknown[]) {
+      const spinner = getSpinner().start(messages.start);
+      try {
+        // @ts-ignore need to apply this function call but unable to type "this"
+        const v = fn.apply(this, ...args);
+
+        if (v instanceof Promise) {
+          return v.then((r) => {
+            spinner.succeed(messages.success);
+            return r;
+          });
+        }
+        spinner.succeed(messages.success);
+        return v;
+      } catch (e) {
+        spinner.fail(messages.fail);
+        throw e;
+      }
+    };
+  };
+}
+
+export function Memoize() {
+  const cache = new Map<string, unknown>();
+
+  // deno-lint-ignore no-explicit-any
+  return function (fn: any, _ctx: ClassMethodDecoratorContext) {
+    return function (...args: unknown[]) {
+      const key = JSON.stringify(args);
+
+      if (cache.has(key)) {
+        return cache.get(key);
+      }
+
+      // @ts-ignore need to apply this function call but unable to type "this"
+      const result = fn.apply(this, args);
+
+      // If the method is async, wait for the promise to resolve
+      if (result instanceof Promise) {
+        return result.then((resolvedResult) => {
+          cache.set(key, resolvedResult);
+          return resolvedResult;
+        });
+      }
+
+      cache.set(key, result);
+      return result;
+    };
+  };
+}

src/index.ts

@@ -1,4 +1,4 @@
-#!/usr/bin/env -S deno run --allow-env --allow-net --allow-sys --allow-read --allow-write --allow-ffi --allow-run --unstable-worker-options
+#!/usr/bin/env -S deno run --allow-env --allow-net --allow-sys --allow-read --allow-write --allow-ffi --allow-run --unstable-worker-options --no-prompt
 // TODO limit --allow-ffi to just lancedb
 // TODO limit --deny-net on localhost except ollama/db
 // TODO limit --allow-run needed for Deno.exit

src/loader.ts

@@ -3,9 +3,10 @@ import { CIDReg, type IPFSClient } from "./ipfs.ts";
 import { resolve } from "@std/path/resolve";
 import { UntarStream } from "@std/tar";
 import { ensureDir, exists } from "@std/fs";
-import { getSpinner, type Source, SpinnerLog } from "./util.ts";
+import type { Source } from "./util.ts";
 import { ProjectManifest } from "./project/project.ts";
 import { Value } from "@sinclair/typebox/value";
+import { Memoize, SpinnerLog } from "./decorators.ts";
 
 export const getOSTempDir = () =>
   Deno.env.get("TMPDIR") || Deno.env.get("TMP") || Deno.env.get("TEMP") ||
@@ -40,10 +41,10 @@ export async function loadManfiest(path: string): Promise<ProjectManifest> {
   return manifest;
 }
 
-// TODO support tar
 /**
  * @param path The content path or cid
  * @param ipfs The IPFS client to fetch content if from IPFS
+ * @param fileName The name to save the file under, if using .gz exension it will unarchive
  * @param tmpDir (optional) The location to cache content, defaults to the OS temp directory
  * @param force  (optional) If true and the content is from IPFS it will check if its already been fetched
  * @param workingPath (optional) If the content is local it will resolve the path relative to this
@@ -110,8 +111,6 @@ export class Loader {
   #ipfs: IPFSClient;
   #force: boolean;
 
-  #manifest?: [manifestPath: string, ProjectManifest, Source];
-
   constructor(
     readonly projectPath: string,
     ipfs: IPFSClient,
@@ -138,26 +137,20 @@ export class Loader {
     );
   }
 
-  // @SpinnerLog({ start: "Loading project manifest", success: "Loaded project manifest", fail: "Failed to load project manfiest"})
+  @Memoize()
+  @SpinnerLog({
+    start: "Loading project manifest",
+    success: "Loaded project manifest",
+    fail: "Failed to load project manfiest",
+  })
   async getManifest(): Promise<[string, ProjectManifest, Source]> {
-    if (!this.#manifest) {
-      const spinner = getSpinner().start("Loading project manifest");
-      try {
-        const [manifestPath, source] = await this.pullContent(
-          this.projectPath,
-          "manifest.json",
-        );
-
-        const manifest = await loadManfiest(manifestPath);
-
-        this.#manifest = [manifestPath, manifest, source];
-        spinner.succeed("Loaded project manifest");
-      } catch (e) {
-        spinner.fail("Failed to load project manifest");
-        throw e;
-      }
-    }
-    return this.#manifest;
+    const [manifestPath, source] = await this.pullContent(
+      this.projectPath,
+      "manifest.json",
+    );
+
+    const manifest = await loadManfiest(manifestPath);
+    return [manifestPath, manifest, source];
   }
 
   @SpinnerLog({
@@ -187,7 +180,6 @@ export class Loader {
       return undefined;
     }
 
-    // TODO resovle local paths
     const res = await this.pullContent(
       manifest.vectorStorage.path,
       "db.gz",

src/sandbox/webWorker/webWorkerSandbox.ts

@@ -12,7 +12,11 @@ import {
   Init,
   Load,
 } from "./messages.ts";
-import { loadRawConfigFromEnv, type Source } from "../../util.ts";
+import {
+  extractConfigHostNames,
+  loadRawConfigFromEnv,
+  type Source,
+} from "../../util.ts";
 import type { IContext } from "../../context/context.ts";
 import type { ProjectManifest } from "../../project/project.ts";
 import type { Loader } from "../../loader.ts";
@@ -62,18 +66,27 @@ export class WebWorkerSandbox implements ISandbox {
     loader: Loader,
   ): Promise<WebWorkerSandbox> {
     const [manifestPath, manifest, source] = await loader.getManifest();
+    const config = loadRawConfigFromEnv(manifest.config);
 
     const permissions = getPermisionsForSource(source, manifestPath);
 
+    // Add any project host names as well as any configured host names
+    const hostnames = [
+      ...new Set(
+        ...(manifest.endpoints ?? []),
+        ...extractConfigHostNames(config as Record<string, string>),
+      ),
+    ];
+
     const w = new Worker(
       import.meta.resolve("./webWorker.ts"),
       {
         type: "module",
         deno: {
           permissions: {
             ...permissions,
-            env: true, // Should be passed through in loadConfigFromEnv below
-            net: manifest.endpoints, // TODO add config endpoints
+            env: false, // Should be passed through in loadRawConfigFromEnv
+            net: hostnames,
             run: false,
             write: false,
           },
@@ -92,7 +105,6 @@ export class WebWorkerSandbox implements ISandbox {
     const [entryPath] = await loader.getProject();
     await conn.sendRequest(Load, entryPath);
 
-    const config = loadRawConfigFromEnv(manifest.config);
     const { tools, systemPrompt } = await conn.sendRequest(
       Init,
       manifest,
@@ -122,23 +134,6 @@ export class WebWorkerSandbox implements ISandbox {
     return this.#tools;
   }
 
-  // #hasSetupCxt = false;
-  // private setupCtxMethods(ctx: IContext) {
-  //   if (this.#hasSetupCxt) return;
-  //   // Connect up context so sandbox can call application
-  //   this.#connection.onRequest(CtxVectorSearch, async (tableName, vector) => {
-  //     const res = await ctx.vectorSearch(tableName, vector);
-
-  //     // lancedb returns classes (Apache Arrow - Struct Row). It needs to be made serializable
-  //     // This is done here as its specific to the webworker sandbox
-  //     return res.map((r) => JSON.parse(JSON.stringify(r)));
-  //   });
-  //   this.#connection.onRequest(CtxComputeQueryEmbedding, async (query) => {
-  //     return await ctx.computeQueryEmbedding(query);
-  //   });
-  //   this.#hasSetupCxt = true;
-  // }
-
   runTool(toolName: string, args: unknown, ctx: IContext): Promise<string> {
     // Connect up context so sandbox can call application
     this.#connection.onRequest(CtxVectorSearch, async (tableName, vector) => {

src/util.ts

@@ -57,22 +57,21 @@ export function PrettyTypeboxError(
   return error;
 }
 
-export function SpinnerLog(
-  messages: { start: string; success: string; fail: string },
-) {
-  // deno-lint-ignore no-explicit-any
-  return function (fn: any, _ctx: ClassMethodDecoratorContext) {
-    return async function (...args: unknown[]) {
-      const spinner = getSpinner().start(messages.start);
+/** Gets the host names of any urls in a record */
+export function extractConfigHostNames(
+  config: Record<string, string>,
+): string[] {
+  const hosts = Object.values(config)
+    .filter((v) => typeof v === "string")
+    .map((v) => {
       try {
-        // @ts-ignore need to apply this function call but unable to type "this"
-        const v = await fn.apply(this, ...args);
-        spinner.succeed(messages.success);
-        return v;
-      } catch (e) {
-        spinner.fail(messages.fail);
-        throw e;
+        return new URL(v).hostname;
+      } catch (_e) {
+        return undefined;
       }
-    };
-  };
+    })
+    .filter((v) => !!v) as string[]; // Cast should be unnecessary with latest TS versions
+
+  // Make unique
+  return [...new Set(hosts)];
 }

subquery-delegator/project.ts

@@ -1,23 +1,13 @@
 import { type Config, ConfigType } from "./index.ts";
 import type { ProjectManifest } from "../src/project/project.ts";
 import { Value } from "@sinclair/typebox/value";
+import { extractConfigHostNames } from "../src/util.ts";
 
 const defaultConfig = Value.Default(ConfigType, {} as Config) as Config;
 
-const endpoints = Object.values(defaultConfig)
-  .filter((v) => typeof v === "string")
-  .map((v) => {
-    try {
-      return new URL(v).hostname;
-    } catch (_e) {
-      return undefined;
-    }
-  })
-  .filter((v) => !!v) as string[]; // Cast should be unnecessary with latest TS versions
-
 const project: ProjectManifest = {
   specVersion: "0.0.1",
-  endpoints: [...new Set(endpoints)],
+  endpoints: extractConfigHostNames(defaultConfig),
   vectorStorage: {
     type: "lancedb",
     path: "../.db",