Merge branch 'fireproof-storage:main' into updated

.npmrc

@@ -1 +1 @@
-auto-install-peers=false
+auto-install-peers=true

packages/connect-ipfs/package.json

@@ -50,15 +50,15 @@
     "webpack-cli": "^5.1.4"
   },
   "dependencies": {
-    "@web3-storage/pail": "^0.4.2",
-    "@fireproof/core": "workspace:^",
     "@fireproof/connect": "workspace:^",
+    "@fireproof/core": "workspace:^",
     "@fireproof/encrypted-blockstore": "workspace:^",
     "@ucanto/core": "^8.2.0",
     "@ucanto/interface": "^8.1.0",
     "@web3-storage/clock": "^0.3.0",
+    "@web3-storage/pail": "^0.5.0",
     "@web3-storage/w3up-client": "^8.0.1",
     "cross-fetch": "^4.0.0",
     "multiformats": "^12.0.1"
   }
-}
+}

packages/connect-partykit/package.json

@@ -53,12 +53,12 @@
     "webpack-cli": "^5.1.4"
   },
   "dependencies": {
-    "@web3-storage/pail": "^0.4.2",
     "@fireproof/connect": "workspace:^",
     "@fireproof/encrypted-blockstore": "workspace:^",
+    "@web3-storage/pail": "^0.5.0",
     "cross-fetch": "^4.0.0",
     "js-base64": "^3.7.5",
     "multiformats": "^12.0.1",
     "partysocket": "0.0.25"
   }
-}
+}

packages/connect-partykit/test/app/package.json

@@ -7,10 +7,10 @@
     "deploy": "partykit deploy"
   },
   "dependencies": {
-    "partysocket": "0.0.21"
+    "partysocket": "^1.0.1"
   },
   "devDependencies": {
-    "partykit": "0.0.72",
+    "partykit": "^0.0.99",
     "typescript": "^5.3.3"
   }
-}
+}

packages/connect/package.json

@@ -49,11 +49,11 @@
     "webpack-cli": "^5.1.4"
   },
   "dependencies": {
-    "@web3-storage/pail": "^0.4.2",
-    "@ipld/dag-json": "^10.1.2",
     "@fireproof/encrypted-blockstore": "workspace:^",
+    "@ipld/dag-json": "^10.1.2",
+    "@web3-storage/pail": "^0.5.0",
     "cross-fetch": "^4.0.0",
-    "multiformats": "^12.0.1",
-    "js-base64": "^3.7.5"
+    "js-base64": "^3.7.5",
+    "multiformats": "^12.0.1"
   }
 }

packages/encrypted-blockstore/jsr.json

@@ -0,0 +1,5 @@
+{
+  "name": "@fireproof/encrypted-blockstore",
+  "version": "0.17.6",
+  "exports": "./src/index.ts"
+}

packages/encrypted-blockstore/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/encrypted-blockstore",
-  "version": "0.17.5",
+  "version": "0.17.6",
   "description": "Content-addressed blockstore with pluggable CRDT storage connectors.",
   "type": "module",
   "main": "./dist/lib/index.cjs",
@@ -82,7 +82,7 @@
     "test:browser": "node ./scripts/browser-test.js",
     "test:coverage": "c8 --reporter=html --include='dist/lib/*' node ./scripts/test.js && open coverage/src/index.html",
     "test:node": "node ./scripts/test.js",
-    "test:serve":"npx serve-http test/www -p 8080",
+    "test:serve": "npx serve-http test/www -p 8080",
     "test:watch": "nodemon -w src -w test -e ts,js --exec \"npm run build:test && npm run test:node\"",
     "test:watch:follow": "nodemon -w src -w test -e ts,js --exec \"sleep 2 && npm run test:node\"",
     "tsc:watch": "tsc --watch"
@@ -151,18 +151,19 @@
     "util": "^0.12.5"
   },
   "dependencies": {
-    "@web3-storage/pail": "^0.4.2",
     "@ipld/car": "^5.2.0",
     "@ipld/dag-cbor": "^9.0.3",
     "@ipld/dag-json": "^10.1.2",
     "@ipld/unixfs": "^2.1.1",
     "@peculiar/webcrypto": "^1.4.3",
+    "@web3-storage/pail": "^0.5.0",
     "charwise": "^3.0.1",
+    "eslint-plugin-n": "^16.6.2",
     "idb": "^7.1.1",
     "ipfs-unixfs-exporter": "^13.1.7",
     "multiformats": "^12.0.1",
     "p-limit": "^4.0.0",
     "prolly-trees": "^1.0.4",
     "uuidv7": "^0.6.2"
   }
-}
+}

packages/encrypted-blockstore/src/loader.ts

@@ -51,7 +51,7 @@ export abstract class Loadable {
 export class Loader implements Loadable {
   name: string
   ebOpts: BlockstoreOpts
-  commitQueue = new CommitQueue<AnyLink>()
+  commitQueue: CommitQueue<AnyLink> = new CommitQueue<AnyLink>()
   isCompacting = false
   isWriting = false
   remoteMetaStore?: AbstractRemoteMetaStore
@@ -150,7 +150,7 @@ export class Loader implements Loadable {
   }
 
   // eslint-disable-next-line @typescript-eslint/require-await
-  async _getKey() {
+  async _getKey(): Promise<string | undefined> {
     if (this.key) return this.key
     // generate a random key
     if (!this.ebOpts.public) {
@@ -448,7 +448,7 @@ export class Loader implements Loadable {
     return this.carReaders.get(cidString) as Promise<CarReader>
   }
 
-  protected async ensureDecryptedReader(reader: CarReader) {
+  protected async ensureDecryptedReader(reader: CarReader): Promise<CarReader> {
     const theKey = await this._getKey()
     if (this.ebOpts.public || !(theKey && this.ebOpts.crypto)) return reader
     const { blocks, root } = await decodeEncryptedCar(this.ebOpts.crypto, theKey, reader)
@@ -478,7 +478,7 @@ export class Loader implements Loadable {
     await Promise.all(missing.map(cid => limit(() => this.loadCar(cid))))
   }
 
-  async _setWaitForWrite(_writingFn: () => Promise<any>) {
+  async _setWaitForWrite(_writingFn: () => Promise<any>): Promise<void> {
     const wr = this.writing
     this.writing = wr.then(async () => {
       await _writingFn()

packages/encrypted-blockstore/src/remote-wal.ts

@@ -19,19 +19,19 @@ export abstract class RemoteWAL {
 
   walState: WALState = { operations: [], noLoaderOps: [], fileOperations: [] }
   processing: Promise<void> | undefined = undefined
-  processQueue = new CommitQueue<void>()
+  processQueue: CommitQueue<void> = new CommitQueue<void>()
 
   constructor(loader: Loadable) {
     this.loader = loader
-    
-    this.ready = Promise.resolve().then((async () => {
+
+    this.ready = Promise.resolve().then(async () => {
       const walState = await this.load().catch(e => {
         console.error('error loading wal', e)
         return null
       })
       this.walState.operations = walState?.operations || []
       this.walState.fileOperations = walState?.fileOperations || []
-    }))
+    })
   }
 
   async enqueue(dbMeta: DbMeta, opts: CommitOpts) {

packages/encrypted-blockstore/src/transaction.ts

@@ -88,7 +88,7 @@ export class EncryptedBlockstore implements BlockFetcher {
     return await this.loader.getBlock(cid)
   }
 
-  async getFile(car: AnyLink, cid: AnyLink, isPublic = false) {
+  async getFile(car: AnyLink, cid: AnyLink, isPublic = false): Promise<Uint8Array> {
     await this.ready
     if (!this.loader) throw new Error('loader required to get file')
     const reader = await this.loader.loadFileCar(car, isPublic)
@@ -115,7 +115,7 @@ export class EncryptedBlockstore implements BlockFetcher {
     this.compacting = false
   }
 
-  async defaultCompact(blocks: CompactionFetcher) {
+  async defaultCompact(blocks: CompactionFetcher): Promise<TransactionMeta> {
     // console.log('eb compact')
     if (!this.loader) {
       throw new Error('no loader')
@@ -165,7 +165,7 @@ export class CompactionFetcher implements BlockFetcher {
     this.loggedBlocks = new CarTransaction(blocks)
   }
 
-  async get(cid: AnyLink) {
+  async get(cid: AnyLink): Promise<AnyAnyBlock | undefined> {
     const block = await this.blockstore.get(cid)
     if (block) this.loggedBlocks.putSync(cid, block.bytes)
     return block

packages/encrypted-blockstore/src/version.ts

@@ -1 +1 @@
-export const PACKAGE_VERSION = "0.17.5";
+export const PACKAGE_VERSION = "0.17.6";

packages/fireproof/jsr.json

@@ -0,0 +1,5 @@
+{
+  "name": "@fireproof/core",
+  "version": "0.17.7",
+  "exports": "./src/fireproof.ts"
+}

packages/fireproof/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fireproof/core",
-  "version": "0.17.6",
+  "version": "0.17.7",
   "description": "Live database for the web.",
   "type": "module",
   "module": "./dist/browser/fireproof.js",
@@ -131,13 +131,13 @@
     "util": "^0.12.5"
   },
   "dependencies": {
-    "@web3-storage/pail": "^0.4.2",
     "@fireproof/encrypted-blockstore": "workspace:^",
     "@ipld/car": "^5.2.0",
     "@ipld/dag-cbor": "^9.0.3",
     "@ipld/dag-json": "^10.1.2",
     "@ipld/unixfs": "^2.1.1",
     "@peculiar/webcrypto": "^1.4.3",
+    "@web3-storage/pail": "^0.5.0",
     "charwise": "^3.0.1",
     "idb": "^7.1.1",
     "ipfs-unixfs-exporter": "^13.1.7",

packages/fireproof/src/crdt-helpers.ts

@@ -3,7 +3,7 @@ import { parse } from 'multiformats/link'
 import { sha256 as hasher } from 'multiformats/hashes/sha2'
 import * as codec from '@ipld/dag-cbor'
 import { put, get, entries, root } from '@web3-storage/pail/crdt'
-import { Operation, PutOperation } from '@web3-storage/pail/src/crdt/api'
+import { Operation, PutOperation } from '@web3-storage/pail/crdt/api'
 import { EventFetcher, vis } from '@web3-storage/pail/clock'
 import * as Batch from '@web3-storage/pail/crdt/batch'
 
@@ -26,7 +26,7 @@ import type {
   DocFiles
 } from './types'
 import { decodeFile, encodeFile } from './files'
-import { Result } from '@web3-storage/pail/src/crdt/api'
+import { Result } from '@web3-storage/pail/crdt/api'
 
 function time(tag: string) {
   // console.time(tag)
@@ -120,10 +120,14 @@ async function processFileset(blocks: CarTransaction, files: DocFiles, publicFil
         t.putSync(block.cid, block.bytes)
       }
       files[filename] = { cid, type: file.type, size: file.size } as DocFileMeta
+    } else {
+      const { cid, type, size, car } = files[filename] as DocFileMeta
+      if (cid && type && size && car) {
+        files[filename] = { cid, type, size, car }
+      }
     }
   }
-  // todo option to bypass this limit
-  // if (totalSize > 1024 * 1024 * 1) throw new Error('Sync limit for files in a single update is 1MB')
+
   if (didPut.length) {
     const car = await dbBlockstore.loader?.commitFiles(t, { files } as unknown as TransactionMeta, {
       public: publicFiles

packages/fireproof/src/crdt.ts

@@ -23,10 +23,12 @@ import type {
   ClockHead,
   ConfigOpts,
   ChangesOptions,
-  IdxMetaMap
+  IdxMetaMap,
+  DocValue
 } from './types'
 import { index, type Index } from './index'
 import { CRDTClock } from './crdt-clock'
+import { Block } from 'multiformats'
 
 export class CRDT {
   name: string | null
@@ -82,7 +84,7 @@ export class CRDT {
   async bulk(updates: DocUpdate[]): Promise<CRDTMeta> {
     await this.ready
     const prevHead = [...this.clock.head]
-    
+
     const meta = (await this.blockstore.transaction(
       async (blocks: CarTransaction): Promise<TransactionMeta> => {
         const { head } = await applyBulkUpdateToCrdt(blocks, this.clock.head, updates)
@@ -99,7 +101,7 @@ export class CRDT {
 
   // if (snap) await this.clock.applyHead(crdtMeta.head, this.clock.head)
 
-  async allDocs() {
+  async allDocs(): Promise<{ result: DocUpdate[]; head: ClockHead }> {
     await this.ready
     const result: DocUpdate[] = []
     for await (const entry of getAllEntries(this.blockstore, this.clock.head)) {
@@ -108,7 +110,7 @@ export class CRDT {
     return { result, head: this.clock.head }
   }
 
-  async vis() {
+  async vis(): Promise<string> {
     await this.ready
     const txt: string[] = []
     for await (const line of clockVis(this.blockstore, this.clock.head)) {
@@ -117,24 +119,30 @@ export class CRDT {
     return txt.join('\n')
   }
 
-  async getBlock(cidString: string) {
+  async getBlock(cidString: string): Promise<Block> {
     await this.ready
     return await getBlock(this.blockstore, cidString)
   }
 
-  async get(key: string) {
+  async get(key: string): Promise<DocValue | null> {
     await this.ready
     const result = await getValueFromCrdt(this.blockstore, this.clock.head, key)
     if (result.del) return null
     return result
   }
 
-  async changes(since: ClockHead = [], opts: ChangesOptions = {}) {
+  async changes(
+    since: ClockHead = [],
+    opts: ChangesOptions = {}
+  ): Promise<{
+    result: DocUpdate[]
+    head: ClockHead
+  }> {
     await this.ready
     return await clockChangesSince(this.blockstore, this.clock.head, since, opts)
   }
 
-  async compact() {
+  async compact(): Promise<void> {
     return await this.blockstore.compact()
   }
 }