diff --git a/scripts/connect-partykit/README.md b/scripts/connect-partykit/README.md deleted file mode 100644 index c1818920..00000000 --- a/scripts/connect-partykit/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# `@fireproof/partykit` - -[Fireproof](https://use-fireproof.com) is an embedded JavaScript document database that runs in the browser (or anywhere with JavaScript) and **[connects to any cloud](https://www.npmjs.com/package/@fireproof/connect)**. - -🎈 [PartyKit](https://www.partykit.io) is a realtime connection library that's the perfect complement to Fireproof's verifiable sync. - -## Get started - -We assume you already have an app that uses Fireproof in the browser, and you want to setup collaboration among multiple users via the cloud or peer-to-peer. To write your first Fireproof app, see the [Fireproof quickstart](https://use-fireproof.com/docs/react-tutorial), othwerwise read on. It's also easy to add Fireproof to PartyKit apps, check out this demo repo for [live magnetic poetry with database persistence.](https://github.com/fireproof-storage/sketch-magnetic-poetry) - -PartyKit uses websockets and CloudFlare workers to manage a real-time group. Adding Fireproof requires one-line of config, and it syncs in its own party so you can use it with your existing [PartyKit](https://docs.partykit.io) apps without impacting existing code. - -### 1. Install - -In your existing Fireproof app install the connector: - -```sh -npm install @fireproof/partykit -``` - -### 2. Configure - -If you already have PartyKit configured in your project, all you need to do is add one line to the config defining a `fireproof` party.: - -```js -{ - "name": "my-app-name", - "main": "src/partykit/server.ts", - "parties": { - "fireproof": "node_modules/@fireproof/partykit/src/server.ts" - } -} -``` - -If you haven't added PartyKit to your app, you want to run the PartyKit CLI to set up the basics: - -```sh -npx partykit init -``` - -Refer to the [PartyKit docs](https://docs.partykit.io) for more info on configuring PartyKit. - -### 3. Connect - -You're all done on the server, and ready to develop locally and then deploy with no further changes. Now you just need to connect to the party in your client code: - -```js -// you already have this in your app -import { useFireproof } from "use-fireproof"; -// add this line -import { connect } from "@fireproof/partykit"; -``` - -Now later in your app connect to the party (be sure to do this a component that runs on every render, like your root component or layout): - -```js -const { database } = useFireproof('my-app-database-name') -const connection = connect.partykit(database, process.env.NEXT_PUBLIC_PARTYKIT_HOST!) -``` - -The `connect.partykit` function is idempotent, and designed to be safe to call on every render. It takes two arguments, the current database, and the host of your PartyKit server. This will be the same host you are using in your app when calling `usePartySocket` and other PartyKit APIs, so once you have it set, you won't need to think about it again. - -### 4. Collaborate - -Now you can use Fireproof as you normally would, and it will sync in realtime with other users. Any existing apps you have that use the [live query](https://use-fireproof.com/docs/react-hooks/use-live-query) or [subscription](https://use-fireproof.com/docs/database-api/database#subscribe) APIs will automatically render multi-user updates. - -## Remix connectors - -In this example we use both the S3 and PartyKit connectors. You can use any combination of connectors in your app. - -```ts -function partykitS3({ name, blockstore }: Connectable, partyHost?: string, refresh?: boolean) { - if (!name) throw new Error("database name is required"); - if (!refresh && partyCxs.has(name)) { - return partyCxs.get(name)!; - } - const s3conf = { - // example values, replace with your own by deploying https://github.com/fireproof-storage/valid-cid-s3-bucket - upload: "https://04rvvth2b4.execute-api.us-east-2.amazonaws.com/uploads", - download: "https://sam-app-s3uploadbucket-e6rv1dj2kydh.s3.us-east-2.amazonaws.com", - }; - const s3conn = new ConnectS3(s3conf.upload, s3conf.download, ""); - s3conn.connectStorage(blockstore); - - if (!partyHost) { - console.warn("partyHost not provided, using localhost:1999"); - partyHost = "http://localhost:1999"; - } - const connection = new ConnectPartyKit({ name, host: partyHost } as ConnectPartyKitParams); - connection.connectMeta(blockstore); - partyCxs.set(name, connection); - return connection; -} -``` diff --git a/scripts/connect-partykit/connect-partykit.ts b/scripts/connect-partykit/connect-partykit.ts deleted file mode 100644 index bd97ce7d..00000000 --- a/scripts/connect-partykit/connect-partykit.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { Base64 } from "js-base64"; -import { DownloadMetaFnParams, DownloadDataFnParams, UploadMetaFnParams, UploadDataFnParams } from "./types"; -import { Connection } from "@fireproof/encrypted-blockstore"; -import PartySocket from "partysocket"; -import type { Loader } from "@fireproof/encrypted-blockstore"; - -export interface ConnectPartyKitParams { - name: string; - host: string; -} - -export class ConnectPartyKit extends Connection { - name: string; - host: string; - party: PartySocket; - messagePromise: Promise; - messageResolve?: (value: Uint8Array[] | PromiseLike) => void; - - constructor(params: ConnectPartyKitParams) { - super(); - this.name = params.name; - this.host = params.host; - this.party = new PartySocket({ - party: "fireproof", - host: params.host, - room: params.name, - }); - - this.ready = new Promise((resolve, reject) => { - this.party.addEventListener("open", () => { - resolve(); - }); - }); - this.messagePromise = new Promise((resolve, reject) => { - this.messageResolve = resolve; - }); - // this.ready = this.messagePromise.then(() => {}) - } - - async onConnect() { - if (!this.loader || !this.taskManager) { - throw new Error("loader and taskManager must be set"); - } - this.party.addEventListener("message", (event: MessageEvent) => { - const afn = async () => { - const base64String = event.data; - const uint8ArrayBuffer = Base64.toUint8Array(base64String); - const eventBlock = await this.decodeEventBlock(uint8ArrayBuffer); - // there should be a callback in the MetaStore that we can call from places like this - await this.taskManager!.handleEvent(eventBlock); - // @ts-ignore - this.messageResolve?.([eventBlock.value.data.dbMeta as Uint8Array]); - // add the cid to our parents so we delete it when we send the update - this.parents.push(eventBlock.cid); - setTimeout(() => { - this.messagePromise = new Promise((resolve, reject) => { - this.messageResolve = resolve; - }); - }, 0); - }; - void afn(); - }); - } - - async dataUpload(bytes: Uint8Array, params: UploadDataFnParams) { - const uploadUrl = `${this.host}/parties/fireproof/${this.name}?car=${params.car}`; - const response = await fetch(uploadUrl, { method: "PUT", body: bytes }); - if (response.status === 404) { - throw new Error("Failure in uploading data!"); - } - } - - async dataDownload(params: DownloadDataFnParams) { - const uploadUrl = `${this.host}/parties/fireproof/${this.name}?car=${params.car}`; - const response = await fetch(uploadUrl, { method: "GET" }); - if (response.status === 404) { - throw new Error("Failure in downloading data!"); - } - const data = await response.arrayBuffer(); - // const data = Base64.toUint8Array(base64String) - return new Uint8Array(data); - } - - async metaUpload(bytes: Uint8Array, params: UploadMetaFnParams) { - await this.ready; - const event = await this.createEventBlock(bytes); - const base64String = Base64.fromUint8Array(event.bytes); - const partyMessage = { - data: base64String, - cid: event.cid.toString(), - parents: this.parents.map((p) => p.toString()), - }; - this.party.send(JSON.stringify(partyMessage)); - this.parents = [event.cid]; - return null; - } - - async metaDownload(params: DownloadMetaFnParams) { - const datas = await this.messagePromise; - return datas; - } -} diff --git a/scripts/connect-partykit/index.ts b/scripts/connect-partykit/index.ts deleted file mode 100644 index 1a960e9a..00000000 --- a/scripts/connect-partykit/index.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { ConnectREST, Connectable } from "@fireproof/encrypted-blockstore"; -import { ConnectPartyKit, ConnectPartyKitParams } from "./connect-partykit"; -export { ConnectPartyKit, ConnectPartyKitParams } from "./connect-partykit"; - -const partyCxs = new Map(); - -export const connect = { - partykit: ({ name, blockstore }: Connectable, partyHost?: string, refresh?: boolean) => { - if (!name) throw new Error("database name is required"); - if (!refresh && partyCxs.has(name)) { - return partyCxs.get(name)!; - } - if (!partyHost) { - console.warn("Party host not provided, using localhost:1999"); - partyHost = "http://localhost:1999"; - } - const connection = new ConnectPartyKit({ name, host: partyHost } as ConnectPartyKitParams); - blockstore.ebOpts.threshold = 50000; - connection.connect(blockstore); - partyCxs.set(name, connection); - return connection; - }, - partykitS3: ({ name, blockstore }: Connectable, partyHost?: string, refresh?: boolean) => { - throw new Error("Removed, use connect.partykit() instead or see README"); - }, - partykitRest: ({ name, blockstore }: Connectable, partyHost?: string, refresh?: boolean) => { - if (!name) throw new Error("database name is required"); - if (!refresh && partyCxs.has(name)) { - return partyCxs.get(name)!; - } - - const restConn = new ConnectREST("http://localhost:8000/"); - restConn.connectStorage(blockstore); - - if (!partyHost) { - console.warn("partyHost not provided, using localhost:1999"); - partyHost = "http://localhost:1999"; - } - const connection = new ConnectPartyKit({ name, host: partyHost } as ConnectPartyKitParams); - connection.connectMeta(blockstore); - partyCxs.set(name, connection); - return connection; - }, -}; diff --git a/scripts/connect-partykit/server.ts b/scripts/connect-partykit/server.ts deleted file mode 100644 index deebe754..00000000 --- a/scripts/connect-partykit/server.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type * as Party from "partykit/server"; - -interface PartyMessage { - data: string; - cid: string; - parents: string[]; -} - -const CORS = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS, PUT, DELETE", -}; - -const json = (data: T, status = 200) => Response.json(data, { status, headers: CORS }); - -const ok = () => json({ ok: true }); -export default class Server implements Party.Server { - clockHead = new Map(); - constructor(public party: Party.Party) {} - - async onStart() { - return this.party.storage.get("main").then((head) => { - if (head) { - this.clockHead = head as Map; - } - }); - } - - async onRequest(request: Party.Request) { - // Check if it's a preflight request (OPTIONS) and handle it - if (request.method === "OPTIONS") { - return ok(); - } - - const url = new URL(request.url); - const carId = url.searchParams.get("car"); - - if (carId) { - if (request.method === "PUT") { - const carArrayBuffer = await request.arrayBuffer(); - if (carArrayBuffer) { - await this.party.storage.put(`car-${carId}`, carArrayBuffer); - return json({ ok: true }, 201); - } - return json({ ok: false }, 400); - } else if (request.method === "GET") { - const carArrayBuffer = (await this.party.storage.get(`car-${carId}`)) as Uint8Array; - if (carArrayBuffer) { - return new Response(carArrayBuffer, { status: 200, headers: CORS }); - } - return json({ ok: false }, 404); - } else { - return json({ error: "Method not allowed" }, 405); - } - } else { - return json({ error: "Invalid URL path" }, 400); - } - } - - async onConnect(conn: Party.Connection) { - for (const value of this.clockHead.values()) { - conn.send(value); - } - } - - onMessage(message: string, sender: Party.Connection) { - const { data, cid, parents } = JSON.parse(message) as PartyMessage; - - this.clockHead.set(cid, data); - for (const p of parents) { - this.clockHead.delete(p); - } - - this.party.broadcast(data, [sender.id]); - // console.log('clockHead', sender.id, [...this.clockHead.keys()]) - void this.party.storage.put("main", this.clockHead); - } -} - -Server satisfies Party.Worker; diff --git a/scripts/connect-partykit/types.ts b/scripts/connect-partykit/types.ts deleted file mode 100644 index 70d4854e..00000000 --- a/scripts/connect-partykit/types.ts +++ /dev/null @@ -1,24 +0,0 @@ -export interface UploadMetaFnParams { - name: string; - branch: string; -} - -export interface UploadDataFnParams { - type: "data" | "file"; - name: string; - car: string; - size: string; -} - -export type DownloadFnParamTypes = "data" | "file"; - -export interface DownloadDataFnParams { - type: DownloadFnParamTypes; - name: string; - car: string; -} - -export interface DownloadMetaFnParams { - name: string; - branch: string; -} diff --git a/src/blockstore/store-factory.ts b/src/blockstore/store-factory.ts index 48b79758..96189916 100644 --- a/src/blockstore/store-factory.ts +++ b/src/blockstore/store-factory.ts @@ -141,24 +141,24 @@ export function registerStoreProtocol(item: GatewayFactoryItem): () => void { // }) // } -// const onceDataStoreFactory = new KeyedResolvOnce(); +const onceDataStoreFactory = new KeyedResolvOnce(); async function dataStoreFactory(loader: Loadable): Promise { const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.data, loader)).build().setParam("store", "data").URI(); const sthis = ensureSuperLog(loader.sthis, "dataStoreFactory", { url: url.toString() }); - // return onceDataStoreFactory.get(url.toString()).once(async () => { - const gateway = await getGatewayFromURL(url, sthis); - if (!gateway) { - throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError(); - } - const store = new DataStoreImpl(sthis, loader.name, url, { - gateway: gateway.gateway, - keybag: () => - getKeyBag(loader.sthis, { - ...loader.ebOpts.keyBag, - }), + return onceDataStoreFactory.get(url.toString()).once(async () => { + const gateway = await getGatewayFromURL(url, sthis); + if (!gateway) { + throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError(); + } + const store = new DataStoreImpl(sthis, loader.name, url, { + gateway: gateway.gateway, + keybag: () => + getKeyBag(loader.sthis, { + ...loader.ebOpts.keyBag, + }), + }); + return store; }); - return store; - // }); } // const onceLoadMetaGateway = new KeyedResolvOnce(); @@ -173,30 +173,31 @@ async function dataStoreFactory(loader: Loadable): Promise { // }); // } -// const onceMetaStoreFactory = new KeyedResolvOnce(); +const onceMetaStoreFactory = new KeyedResolvOnce(); + async function metaStoreFactory(loader: Loadable): Promise { const url = ensureName(loader.name, buildURL(loader.ebOpts.store.stores?.meta, loader)).build().setParam("store", "meta").URI(); const sthis = ensureSuperLog(loader.sthis, "metaStoreFactory", { url: () => url.toString() }); - // return onceMetaStoreFactory.get(url.toString()).once(async () => { - sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch"); - const gateway = await getGatewayFromURL(url, sthis); - if (!gateway) { - throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError(); - } - const store = new MetaStoreImpl(loader.sthis, loader.name, url, { - gateway: gateway.gateway, - keybag: () => - getKeyBag(loader.sthis, { - ...loader.ebOpts.keyBag, - }), + return onceMetaStoreFactory.get(url.toString()).once(async () => { + sthis.logger.Debug().Str("protocol", url.protocol).Msg("pre-protocol switch"); + const gateway = await getGatewayFromURL(url, sthis); + if (!gateway) { + throw sthis.logger.Error().Url(url).Msg("gateway not found").AsError(); + } + const store = new MetaStoreImpl(loader.sthis, loader.name, url, { + gateway: gateway.gateway, + keybag: () => + getKeyBag(loader.sthis, { + ...loader.ebOpts.keyBag, + }), + }); + // const ret = await store.start(); + // if (ret.isErr()) { + // throw logger.Error().Result("start", ret).Msg("start failed").AsError(); + // } + // logger.Debug().Url(ret.Ok(), "prepared").Msg("produced"); + return store; }); - // const ret = await store.start(); - // if (ret.isErr()) { - // throw logger.Error().Result("start", ret).Msg("start failed").AsError(); - // } - // logger.Debug().Url(ret.Ok(), "prepared").Msg("produced"); - return store; - // }); } // const onceWalGateway = new KeyedResolvOnce(); diff --git a/src/blockstore/store.ts b/src/blockstore/store.ts index a679788d..60ae1c1d 100644 --- a/src/blockstore/store.ts +++ b/src/blockstore/store.ts @@ -478,7 +478,7 @@ export class WALStoreImpl extends BaseStoreImpl implements WALStore { async load(): Promise { this.logger.Debug().Msg("loading"); - const filepath = await this.gateway.buildUrl(this.url(), "main"); + const filepath = await this.gateway.buildUrl(this.url(), "wal"); if (filepath.isErr()) { throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError(); } @@ -497,7 +497,7 @@ export class WALStoreImpl extends BaseStoreImpl implements WALStore { } async save(state: WALState) { - const filepath = await this.gateway.buildUrl(this.url(), "main"); + const filepath = await this.gateway.buildUrl(this.url(), "wal"); if (filepath.isErr()) { throw this.logger.Error().Err(filepath.Err()).Url(this.url()).Msg("error building url").AsError(); }