Merge pull request fireproof-storage#144 from fireproof-storage/dev

cleanup file/table-name generation

.gitignore

@@ -22,3 +22,4 @@ smoke/package.json
 smoke/pnpm-lock.yaml
 smoke/react/package.json
 smoke/react/pnpm-lock.yaml
+**/__screenshots__/

eslint.config.mjs

@@ -15,15 +15,14 @@ const opts = tseslint.config(
       "**/node_modules/",
       "**/scripts/",
       "**/examples/",
+      "scripts/",
       "smoke/react/",
     ],
-    /*
-    languageOptions: {
-	globals: {
-           process: true
-        }
-    }
-*/
+  },
+  {
+    rules: {
+      "no-console": ["warn"],
+    },
   },
 );
 

publish-package.ts

@@ -1,3 +1,4 @@
+/* eslint no-console: "off" */
 import { $ } from "zx";
 import { SemVer } from "semver";
 import fs from "fs";

scripts/react/src/useDocument.tsx

@@ -1,4 +1,3 @@
-/* eslint-disable react-hooks/rules-of-hooks */
 import { Database, Doc, DocRecord } from "@fireproof/core";
 
 import { UseDocument, UseDocumentResult, useFireproof } from "./useFireproof";

smoke/node-test.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 import { fireproof, PACKAGE_VERSION } from "@fireproof/core";
 
 async function main() {

smoke/patch-package.json.mjs

@@ -3,6 +3,7 @@ import * as fs from "fs";
 import * as process from "process";
 let version = process.argv[process.argv.length - 1];
 version = version.split("/").slice(-1)[0].replace(/^v/, "");
+// eslint-disable-next-line no-console
 console.error(`Patch package.json version to ${version}`);
 const packageJson = JSON.parse(fs.readFileSync("package.json").toString());
 packageJson.type = "module";

src/blockstore/loader.ts

@@ -334,8 +334,7 @@ export class Loader implements Loadable {
         ? await encryptedEncodeCarFile(this.logger, this.ebOpts.crypto, theKey, cid, t)
         : await encodeCarFile([cid], t);
     } catch (e) {
-      console.error("error creating car file", e);
-      throw e;
+      throw this.logger.Error().Err(e).Msg("error creating car file").AsError();
     }
   }
 

src/crdt-helpers.ts

@@ -211,11 +211,16 @@ async function getValueFromLink<T extends DocTypes>(blocks: BlockFetcher, link:
 }
 
 class DirtyEventFetcher<T> extends EventFetcher<T> {
+  readonly logger: Logger;
+  constructor(logger: Logger, blocks: BlockFetcher) {
+    super(blocks);
+    this.logger = logger;
+  }
   async get(link: EventLink<T>): Promise<EventBlockView<T>> {
     try {
       return super.get(link);
     } catch (e) {
-      console.error("missing event", link.toString(), e);
+      this.logger.Error().Ref("link", link.toString()).Err(e).Msg("Missing event");
       return { value: undefined } as unknown as EventBlockView<T>;
     }
   }
@@ -229,7 +234,7 @@ export async function clockChangesSince<T extends DocTypes>(
   logger: Logger,
 ): Promise<{ result: DocUpdate<T>[]; head: ClockHead }> {
   const eventsFetcher = (
-    opts.dirty ? new DirtyEventFetcher<Operation>(blocks) : new EventFetcher<Operation>(blocks)
+    opts.dirty ? new DirtyEventFetcher<Operation>(logger, blocks) : new EventFetcher<Operation>(blocks)
   ) as EventFetcher<Operation>;
   const keys = new Set<string>();
   const updates = await gatherUpdates<T>(

src/react/useFireproof.ts

@@ -166,7 +166,6 @@ export function useFireproof(name: string | Database = "useFireproof", config: C
 
     const saveDoc: StoreDocFn<T> = useCallback(
       async (existingDoc) => {
-        console.log("saveDoc", existingDoc, doc);
         const res = await database.put(existingDoc ?? doc);
         // If the document was created, then we need to update the local state with the new `_id`
         if (!existingDoc && !doc._id) setDoc((d) => ({ ...d, _id: res.id }));

src/runtime/store-file-utils.ts

@@ -1,34 +1,31 @@
 import { Logger, getStore } from "../utils.js";
 import { SysContainer } from "./sys-container.js";
 
-export async function getPath(url: URL, logger: Logger): Promise<string> {
+export function getPath(url: URL, logger: Logger): string {
   const basePath = url
     .toString()
     .replace(new RegExp(`^${url.protocol}//`), "")
     .replace(/\?.*$/, "");
   const name = url.searchParams.get("name");
   if (name) {
     const version = url.searchParams.get("version");
-    if (!version) throw logger.Error().Str("url", url.toString()).Msg(`version not found`).AsError();
+    if (!version) throw logger.Error().Url(url).Msg(`version not found`).AsError();
     return SysContainer.join(basePath, version, name);
   }
   return SysContainer.join(basePath);
 }
 
-export function getFileName(url: URL, key: string, logger: Logger): string {
-  switch (getStore(url, logger, (...a: string[]) => a.join("/"))) {
+export function getFileName(url: URL, logger: Logger): string {
+  const key = url.searchParams.get("key");
+  if (!key) throw logger.Error().Url(url).Msg(`key not found`).AsError();
+  const res = getStore(url, logger, (...a: string[]) => a.join("-"));
+  switch (res.store) {
     case "data":
-      return key + ".car";
+      return SysContainer.join(res.name, key + ".car");
+    case "wal":
     case "meta":
-      return key + ".json";
+      return SysContainer.join(res.name, key + ".json");
     default:
-      throw logger.Error().Str("url", url.toString()).Msg(`unsupported store type`).AsError();
+      throw logger.Error().Url(url).Msg(`unsupported store type`).AsError();
   }
 }
-
-export function ensureIndexName(url: URL, name: string): string {
-  if (url.searchParams.has("index")) {
-    name = (url.searchParams.get("index")?.replace(/[^a-zA-Z0-9]/g, "") || "idx") + "-" + name;
-  }
-  return name;
-}

src/runtime/store-file.ts

@@ -2,9 +2,9 @@ import { SysContainer } from "./sys-container.js";
 import { TestStore } from "../blockstore/types.js";
 import { FILESTORE_VERSION } from "./store-file-version.js";
 import { Logger, ResolveOnce, Result } from "@adviser/cement";
-import { ensureLogger, exception2Result, exceptionWrapper, getStore } from "../utils.js";
+import { ensureLogger, exception2Result, exceptionWrapper } from "../utils.js";
 import { Gateway, GetResult, isNotFoundError, NotFoundError } from "../blockstore/gateway.js";
-import { ensureIndexName, getFileName, getPath } from "./store-file-utils.js";
+import { getFileName, getPath } from "./store-file-utils.js";
 
 const versionFiles = new Map<string, ResolveOnce<void>>();
 async function ensureVersionFile(path: string, logger: Logger): Promise<string> {
@@ -25,7 +25,7 @@ async function ensureVersionFile(path: string, logger: Logger): Promise<string>
     }
     const v = await SysContainer.readfile(vFile);
     if (v.toString() !== FILESTORE_VERSION) {
-      console.warn(`version mismatch:${vFile}: ${v.toString()}!=${FILESTORE_VERSION}`);
+      logger.Warn().Str("file", vFile).Str("from", v.toString()).Str("expected", FILESTORE_VERSION).Msg(`version mismatch`);
     }
   });
   return path;
@@ -50,24 +50,28 @@ abstract class FileGateway implements Gateway {
       await ensureVersionFile(dbroot, this.logger);
     });
   }
+
+  async buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
+    const url = new URL(baseUrl.toString());
+    // url.pathname = SysContainer.join(getPath(baseUrl, this.logger), getFileName(baseUrl, key, this.logger));
+    url.searchParams.set("key", key);
+    return Result.Ok(url);
+  }
+
   async close(): Promise<Result<void>> {
     return Result.Ok(undefined);
   }
-  abstract destroy(baseUrl: URL): Promise<Result<void>>;
-  abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
+  // abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
 
   getFilePath(url: URL): string {
-    const path = url
-      .toString()
-      .replace(/^file:\/\//, "")
-      .replace(/\?.*$/, "");
-    this.logger.Debug().Str("url", url.toString()).Str("path", path).Msg("getFilePath");
-    return path;
+    const key = url.searchParams.get("key");
+    if (!key) throw this.logger.Error().Url(url).Msg(`key not found`).AsError();
+    return SysContainer.join(getPath(url, this.logger), getFileName(url, this.logger));
   }
 
   async put(url: URL, body: Uint8Array): Promise<Result<void>> {
     return exception2Result(async () => {
-      const file = this.getFilePath(url);
+      const file = await this.getFilePath(url);
       this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put");
       await SysContainer.writefile(file, body);
     });
@@ -94,7 +98,7 @@ abstract class FileGateway implements Gateway {
     });
   }
 
-  async destroyDir(baseURL: URL): Promise<Result<void>> {
+  async destroy(baseURL: URL): Promise<Result<void>> {
     const url = await this.buildUrl(baseURL, "x");
     if (url.isErr()) return url;
     const filepath = SysContainer.dirname(this.getFilePath(url.Ok()));
@@ -124,30 +128,12 @@ export class FileWALGateway extends FileGateway {
   constructor(logger: Logger) {
     super(ensureLogger(logger, "FileWALGateway"));
   }
-
-  async destroy(baseURL: URL): Promise<Result<void>> {
-    return this.destroyDir(baseURL);
-  }
-  async buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
-    const url = new URL(baseUrl.toString());
-    url.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "wal"), key + ".json");
-    return Result.Ok(url);
-  }
 }
 
 export class FileMetaGateway extends FileGateway {
   constructor(logger: Logger) {
     super(ensureLogger(logger, "FileMetaGateway"));
   }
-
-  async destroy(baseURL: URL): Promise<Result<void>> {
-    return this.destroyDir(baseURL);
-  }
-  async buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
-    const url = new URL(baseUrl.toString());
-    url.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "meta"), key + ".json");
-    return Result.Ok(url);
-  }
 }
 
 export class FileDataGateway extends FileGateway {
@@ -156,15 +142,6 @@ export class FileDataGateway extends FileGateway {
     // console.log("FileDataGateway->", logger);
     super(ensureLogger(logger, "FileDataGateway"));
   }
-
-  async destroy(baseURL: URL): Promise<Result<void>> {
-    return this.destroyDir(baseURL);
-  }
-  async buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
-    const url = new URL(baseUrl.toString());
-    url.pathname = SysContainer.join(await getPath(baseUrl, this.logger), ensureIndexName(baseUrl, "data"), key + ".car");
-    return Result.Ok(url);
-  }
 }
 
 function toArrayBuffer(buffer: Buffer) {
@@ -185,16 +162,13 @@ export class FileTestStore implements TestStore {
     this.logger = ensureLogger(logger, "FileTestStore");
   }
 
-  async get(url: URL, key: string) {
-    const logger = ensureLogger(this.logger, "get", { url: url.toString(), key });
-    const dbFile = SysContainer.join(
-      await getPath(url, this.logger),
-      getStore(url, this.logger, SysContainer.join),
-      getFileName(url, key, this.logger),
-    );
-    logger.Debug().Str("dbFile", dbFile).Msg("get");
+  async get(iurl: URL, key: string) {
+    const url = new URL(iurl.toString());
+    url.searchParams.set("key", key);
+    const dbFile = SysContainer.join(getPath(url, this.logger), getFileName(url, this.logger));
+    this.logger.Debug().Url(url).Str("dbFile", dbFile).Msg("get");
     const buffer = await SysContainer.readfile(dbFile);
-    logger.Debug().Str("dbFile", dbFile).Len(buffer).Msg("got");
+    this.logger.Debug().Url(url).Str("dbFile", dbFile).Len(buffer).Msg("got");
     return toArrayBuffer(buffer);
   }
 }

src/runtime/store-indexdb.ts

@@ -91,7 +91,7 @@ export function getIndexDBName(iurl: URL, logger: Logger): DbName {
   const dbName = url.searchParams.get("name");
   if (!dbName) throw logger.Error().Str("url", url.toString()).Msg(`name not found`).AsError();
   const result = joinDBName(fullDb, dbName);
-  const objStore = getStore(url, logger, joinDBName);
+  const objStore = getStore(url, logger, joinDBName).name;
   const connectionKey = [result, objStore].join(":");
   return {
     fullDb: result,
@@ -125,7 +125,7 @@ abstract class IndexDBGateway implements Gateway {
   async destroy(baseUrl: URL): Promise<Result<void>> {
     return exception2Result(async () => {
       // return deleteDB(getIndexDBName(this.url).fullDb);
-      const type = getStore(baseUrl, this.logger, joinDBName);
+      const type = getStore(baseUrl, this.logger, joinDBName).name;
       // console.log("IndexDBDataStore:destroy", type);
       const idb = this.db;
       const trans = idb.transaction(type, "readwrite");
@@ -141,12 +141,16 @@ abstract class IndexDBGateway implements Gateway {
     });
   }
 
-  abstract buildUrl(baseUrl: URL, key: string): Promise<Result<URL>>;
+  buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
+    const url = new URL(baseUrl.toString());
+    url.searchParams.set("key", key);
+    return Promise.resolve(Result.Ok(url));
+  }
 
   async get(url: URL) {
     return exceptionWrapper(async () => {
       const key = getKey(url, this.logger);
-      const store = getStore(url, this.logger, joinDBName);
+      const store = getStore(url, this.logger, joinDBName).name;
       this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("getting");
       const tx = this.db.transaction([store], "readonly");
       const bytes = await tx.objectStore(store).get(sanitzeKey(key));
@@ -160,7 +164,7 @@ abstract class IndexDBGateway implements Gateway {
   async put(url: URL, value: Uint8Array) {
     return exception2Result(async () => {
       const key = getKey(url, this.logger);
-      const store = getStore(url, this.logger, joinDBName);
+      const store = getStore(url, this.logger, joinDBName).name;
       this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("putting");
       const tx = this.db.transaction([store], "readwrite");
       await tx.objectStore(store).put(value, sanitzeKey(key));
@@ -170,7 +174,7 @@ abstract class IndexDBGateway implements Gateway {
   async delete(url: URL) {
     return exception2Result(async () => {
       const key = getKey(url, this.logger);
-      const store = getStore(url, this.logger, joinDBName);
+      const store = getStore(url, this.logger, joinDBName).name;
       this.logger.Debug().Url(url).Str("key", key).Str("store", store).Msg("deleting");
       const tx = this.db.transaction([store], "readwrite");
       await tx.objectStore(store).delete(sanitzeKey(key));
@@ -184,36 +188,17 @@ export class IndexDBDataGateway extends IndexDBGateway {
   constructor(logger: Logger) {
     super(ensureLogger(logger, "IndexDBDataGateway", {}));
   }
-
-  buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
-    const url = new URL(baseUrl.toString());
-    url.searchParams.set("key", key);
-    return Promise.resolve(Result.Ok(url));
-  }
 }
 
 export class IndexDBWalGateway extends IndexDBGateway {
   constructor(logger: Logger) {
     super(ensureLogger(logger, "IndexDBWalGateway", {}));
   }
-  buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
-    const url = new URL(baseUrl.toString());
-    url.searchParams.set("key", key);
-    return Promise.resolve(Result.Ok(url));
-  }
 }
 export class IndexDBMetaGateway extends IndexDBGateway {
   constructor(logger: Logger) {
     super(ensureLogger(logger, "IndexDBDataGateway", {}));
   }
-
-  readonly branches = new Set<string>();
-  async buildUrl(baseUrl: URL, key: string): Promise<Result<URL>> {
-    const url = new URL(baseUrl.toString());
-    this.branches.add(key);
-    url.searchParams.set("key", key);
-    return Result.Ok(url);
-  }
 }
 
 const txtEncoder = new TextEncoder();
@@ -224,7 +209,7 @@ export class IndexDBTestStore implements TestStore {
   }
   async get(url: URL, key: string) {
     const db = await connectIdb(url, this.logger);
-    const store = getStore(url, this.logger, joinDBName);
+    const store = getStore(url, this.logger, joinDBName).name;
     this.logger.Debug().Str("key", key).Str("store", store).Msg("getting");
     let bytes = await db.get(store, sanitzeKey(key));
     this.logger.Debug().Str("key", key).Str("store", store).Int("len", bytes.length).Msg("got");

src/runtime/sys-container.ts

@@ -215,6 +215,7 @@ class sysContainer {
   logSeeded(method: string) {
     if (this.freight.state === "seeded") {
       const err = new Error();
+      // eslint-disable-next-line no-console
       console.warn(`SysContainer.${method} is not available in seeded state:`, err.stack);
     }
   }