From e429ae6a7783dd8efdb0cda117516567007184b1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 4 Jun 2024 15:05:33 +0200 Subject: [PATCH 1/3] chore: use w3up client final version --- package-lock.json | 30 +++++++++++++++--------------- package.json | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index 63acdce..e3da849 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "@web3-storage/access": "^20.0.0", "@web3-storage/data-segment": "^5.0.0", "@web3-storage/did-mailto": "^2.1.0", - "@web3-storage/w3up-client": "^14.0.0-rc.2", + "@web3-storage/w3up-client": "^14.0.0", "ansi-escapes": "^6.2.0", "chalk": "^5.3.0", "files-from-path": "^1.0.4", @@ -1679,14 +1679,14 @@ } }, "node_modules/@web3-storage/blob-index": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@web3-storage/blob-index/-/blob-index-1.0.2.tgz", - "integrity": "sha512-N+yMIk2cmgaGYVy9EewsRx1sxSDv67i2IBlZ4y72a/+lVIAmb3ZP0IwZ+Med0xrNZShA4blxIGJm1LVF7Q4mSg==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@web3-storage/blob-index/-/blob-index-1.0.3.tgz", + "integrity": "sha512-VjGLhf6Gf4ZmzjJXS6wU4aRvnM+HLcuRCJHegjQ36ka52sR2WWOcqDNNVvabtlpnYjGtVFQCPUzaCcs18wpqHQ==", "dependencies": { "@ipld/dag-cbor": "^9.0.6", "@ucanto/core": "^10.0.1", "@ucanto/interface": "^10.0.1", - "@web3-storage/capabilities": "^17.1.0", + "@web3-storage/capabilities": "^17.1.1", "carstream": "^2.1.0", "multiformats": "^13.0.1", "uint8arrays": "^5.0.3" @@ -1962,9 +1962,9 @@ "dev": true }, "node_modules/@web3-storage/upload-client": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-16.0.1.tgz", - "integrity": "sha512-PVkeSIaq2MNJxHAA9JUChA8d2rG3LNV5HD1qDlXCXHPZo/bA/eIsjMHBWqysGVYR2aW3gE0v65Xq/YZzDgJVbA==", + "version": "16.0.2", + "resolved": "https://registry.npmjs.org/@web3-storage/upload-client/-/upload-client-16.0.2.tgz", + "integrity": "sha512-noNN3RiLl3FZz4nDDcQhfV8+5wPe2LljPkiayTQLI3PPSx9dOimG8h25qZNaACnA0W2GKbXOz/RkWgSlaRzfSw==", "dependencies": { "@ipld/car": "^5.2.2", "@ipld/dag-cbor": "^9.0.6", @@ -1974,7 +1974,7 @@ "@ucanto/core": "^10.0.1", "@ucanto/interface": "^10.0.1", "@ucanto/transport": "^9.1.1", - "@web3-storage/blob-index": "^1.0.2", + "@web3-storage/blob-index": "^1.0.3", "@web3-storage/capabilities": "^17.1.1", "@web3-storage/data-segment": "^5.1.0", "@web3-storage/filecoin-client": "^3.3.3", @@ -2013,9 +2013,9 @@ } }, "node_modules/@web3-storage/w3up-client": { - "version": "14.0.0-rc.2", - "resolved": "https://registry.npmjs.org/@web3-storage/w3up-client/-/w3up-client-14.0.0-rc.2.tgz", - "integrity": "sha512-Wt7nBTCoRKvwgZ6vgvFMsCCFNogyPeAmLQxn+/CIDeH5j5qMN120Scqolr3kIlOel7mVaeoHk66vgcjW+b7n6w==", + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/@web3-storage/w3up-client/-/w3up-client-14.0.0.tgz", + "integrity": "sha512-ZJ3YQI6av42hnUrRcQoDoVNDMeQO3HrnmNWkmsd7uMBDQcuEG2wmIWviFkcH0L4+RVBugWEwjMV1+XulW+X4rQ==", "dependencies": { "@ipld/dag-ucan": "^3.4.0", "@ucanto/client": "^9.0.1", @@ -2024,11 +2024,11 @@ "@ucanto/principal": "^9.0.1", "@ucanto/transport": "^9.1.1", "@web3-storage/access": "^20.0.0", - "@web3-storage/blob-index": "^1.0.2", + "@web3-storage/blob-index": "^1.0.3", "@web3-storage/capabilities": "^17.1.1", - "@web3-storage/did-mailto": "^2.0.2", + "@web3-storage/did-mailto": "^2.1.0", "@web3-storage/filecoin-client": "^3.3.3", - "@web3-storage/upload-client": "^16.0.1" + "@web3-storage/upload-client": "^16.0.2" }, "engines": { "node": ">=18" diff --git a/package.json b/package.json index 029a559..a349aee 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "@web3-storage/access": "^20.0.0", "@web3-storage/data-segment": "^5.0.0", "@web3-storage/did-mailto": "^2.1.0", - "@web3-storage/w3up-client": "^14.0.0-rc.2", + "@web3-storage/w3up-client": "^14.0.0", "ansi-escapes": "^6.2.0", "chalk": "^5.3.0", "files-from-path": "^1.0.4", From d2be2a474ecdcf4b8ba39857361f8de5d6c1e022 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 4 Jun 2024 16:05:54 +0200 Subject: [PATCH 2/3] fix: add receipt endpoint --- can.js | 5 +-- lib.js | 8 ++++- test/helpers/context.js | 2 ++ test/helpers/env.js | 4 ++- test/helpers/http-server.js | 51 +++++++++++++++++++++++++++++++ test/helpers/random.js | 61 +++++++++++++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 test/helpers/random.js diff --git a/can.js b/can.js index 5fdf63a..d5a61f9 100644 --- a/can.js +++ b/can.js @@ -19,8 +19,9 @@ import { /** * @param {string} [blobPath] + * @param {import('@web3-storage/w3up-client/types.js').RequestOptions} [options] */ -export async function blobAdd(blobPath) { +export async function blobAdd(blobPath, options) { const client = await getClient() const spinner = ora('Reading data').start() @@ -37,7 +38,7 @@ export async function blobAdd(blobPath) { } spinner.start('Storing') - const digest = await client.capability.blob.add(blob) + const digest = await client.capability.blob.add(blob, options) const cid = Link.create(raw.code, digest) spinner.stopAndPersist({ symbol: '⁂', text: `Stored ${base58btc.encode(digest.bytes)} (${cid})` }) } diff --git a/lib.js b/lib.js index 48ba49a..51388b3 100644 --- a/lib.js +++ b/lib.js @@ -96,6 +96,12 @@ export function getClient() { process.env.W3UP_SERVICE_DID || process.env.W3_UPLOAD_SERVICE_DID const uploadServiceURL = process.env.W3UP_SERVICE_URL || process.env.W3_UPLOAD_SERVICE_URL + const receiptsEndpointString = (process.env.W3UP_RECEIPTS_ENDPOINT || process.env.W3_UPLOAD_RECEIPTS_URL) + let receiptsEndpoint + if (receiptsEndpointString) { + receiptsEndpoint = new URL(receiptsEndpointString) + } + let serviceConf if ( accessServiceDID && @@ -134,7 +140,7 @@ export function getClient() { } /** @type {import('@web3-storage/w3up-client/types').ClientFactoryOptions} */ - const createConfig = { store, serviceConf } + const createConfig = { store, serviceConf, receiptsEndpoint } const principal = process.env.W3_PRINCIPAL if (principal) { diff --git a/test/helpers/context.js b/test/helpers/context.js index 1272732..08b2a0a 100644 --- a/test/helpers/context.js +++ b/test/helpers/context.js @@ -72,11 +72,13 @@ export const setup = async () => { storeName: `w3cli-test-alice-${context.service.did()}`, servicePrincipal: context.service, serviceURL: serverURL, + receiptsEndpoint: new URL('receipt', serverURL), }), bob: createEnv({ storeName: `w3cli-test-bob-${context.service.did()}`, servicePrincipal: context.service, serviceURL: serverURL, + receiptsEndpoint: new URL('receipt', serverURL), }), }, }) diff --git a/test/helpers/env.js b/test/helpers/env.js index ed93740..16efcd7 100644 --- a/test/helpers/env.js +++ b/test/helpers/env.js @@ -3,14 +3,16 @@ * @param {import('@ucanto/interface').Principal} [options.servicePrincipal] * @param {URL} [options.serviceURL] * @param {string} [options.storeName] + * @param {URL} [options.receiptsEndpoint] */ export function createEnv(options = {}) { - const { servicePrincipal, serviceURL, storeName } = options + const { servicePrincipal, serviceURL, storeName, receiptsEndpoint } = options const env = { W3_STORE_NAME: storeName ?? 'w3cli-test' } if (servicePrincipal && serviceURL) { Object.assign(env, { W3UP_SERVICE_DID: servicePrincipal.did(), W3UP_SERVICE_URL: serviceURL.toString(), + W3UP_RECEIPTS_ENDPOINT: receiptsEndpoint?.toString() }) } return env diff --git a/test/helpers/http-server.js b/test/helpers/http-server.js index ef68714..902ed23 100644 --- a/test/helpers/http-server.js +++ b/test/helpers/http-server.js @@ -1,6 +1,13 @@ import http from 'http' import { once } from 'events' +import { parseLink } from '@ucanto/server' +import * as Signer from '@ucanto/principal/ed25519' +import { Receipt, Message } from '@ucanto/core' +import * as CAR from '@ucanto/transport/car' +import { Assert } from '@web3-storage/content-claims/capability' +import { randomCAR } from './random.js' + /** * @typedef {import('@ucanto/interface').HTTPRequest} HTTPRequest * @typedef {import('@ucanto/server').HTTPResponse} HTTPResponse @@ -23,6 +30,14 @@ export async function createServer(router) { * @param {http.ServerResponse} response */ const listener = async (request, response) => { + if (request.url?.includes('receipt')) { + const taskCid = request.url?.split('/')[1] ?? '' + const body = await generateReceipt(taskCid) + response.writeHead(200) + response.end(body) + return undefined + } + const chunks = [] for await (const chunk of request) { chunks.push(chunk) @@ -59,3 +74,39 @@ export async function createServer(router) { serverURL: new URL(`http://127.0.0.1:${server.address().port}`), } } + +/** + * @param {string} taskCid + */ +const generateReceipt = async (taskCid) => { + const issuer = await Signer.generate() + const content = (await randomCAR(128)).cid + const locationClaim = await Assert.location.delegate({ + issuer, + audience: issuer, + with: issuer.toDIDKey(), + nb: { + content, + location: ['http://localhost'], + }, + expiration: Infinity, + }) + + const receipt = await Receipt.issue({ + issuer, + fx: { + fork: [locationClaim], + }, + ran: parseLink(taskCid), + result: { + ok: { + site: locationClaim.link(), + }, + }, + }) + + const message = await Message.build({ + receipts: [receipt], + }) + return CAR.request.encode(message).body +} diff --git a/test/helpers/random.js b/test/helpers/random.js new file mode 100644 index 0000000..7b26e40 --- /dev/null +++ b/test/helpers/random.js @@ -0,0 +1,61 @@ +import { CarWriter } from '@ipld/car' +import * as CAR from '@ucanto/transport/car' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' + +/** @param {number} size */ +export async function randomBytes(size) { + const bytes = new Uint8Array(size) + while (size) { + const chunk = new Uint8Array(Math.min(size, 65_536)) + if (!globalThis.crypto) { + try { + const { webcrypto } = await import('node:crypto') + webcrypto.getRandomValues(chunk) + } catch (err) { + throw new Error( + 'unknown environment - no global crypto and not Node.js', + { cause: err } + ) + } + } else { + crypto.getRandomValues(chunk) + } + size -= chunk.length + bytes.set(chunk, size) + } + return bytes +} + +/** @param {number} size */ +export async function randomCAR(size) { + const bytes = await randomBytes(size) + return toCAR(bytes) +} + +/** @param {Uint8Array} bytes */ +export async function toBlock(bytes) { + const hash = await sha256.digest(bytes) + const cid = CID.createV1(raw.code, hash) + return { cid, bytes } +} + +/** + * @param {Uint8Array} bytes + */ +export async function toCAR(bytes) { + const block = await toBlock(bytes) + const { writer, out } = CarWriter.create(block.cid) + writer.put(block) + writer.close() + + const chunks = [] + for await (const chunk of out) { + chunks.push(chunk) + } + const blob = new Blob(chunks) + const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) + + return Object.assign(blob, { cid, roots: [block.cid] }) +} From 4e7f40c2f1eef8433c916aea3e88c2648718b3b0 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 4 Jun 2024 16:26:24 +0200 Subject: [PATCH 3/3] fix: add receipt endpoint --- can.js | 11 ++-- index.js | 1 + test/helpers/context.js | 9 +++- test/helpers/http-server.js | 51 ------------------- test/helpers/receipt-http-server.js | 79 +++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 58 deletions(-) create mode 100644 test/helpers/receipt-http-server.js diff --git a/can.js b/can.js index d5a61f9..ed4362b 100644 --- a/can.js +++ b/can.js @@ -19,9 +19,8 @@ import { /** * @param {string} [blobPath] - * @param {import('@web3-storage/w3up-client/types.js').RequestOptions} [options] */ -export async function blobAdd(blobPath, options) { +export async function blobAdd(blobPath) { const client = await getClient() const spinner = ora('Reading data').start() @@ -38,9 +37,11 @@ export async function blobAdd(blobPath, options) { } spinner.start('Storing') - const digest = await client.capability.blob.add(blob, options) - const cid = Link.create(raw.code, digest) - spinner.stopAndPersist({ symbol: '⁂', text: `Stored ${base58btc.encode(digest.bytes)} (${cid})` }) + const { multihash } = await client.capability.blob.add(blob, { + receiptsEndpoint: client._receiptsEndpoint.toString() + }) + const cid = Link.create(raw.code, multihash) + spinner.stopAndPersist({ symbol: '⁂', text: `Stored ${base58btc.encode(multihash.bytes)} (${cid})` }) } /** diff --git a/index.js b/index.js index 014b70b..d4607d0 100644 --- a/index.js +++ b/index.js @@ -177,6 +177,7 @@ export async function upload(firstPath, opts) { concurrentRequests: opts?.['concurrent-requests'] && parseInt(String(opts?.['concurrent-requests'])), + receiptsEndpoint: client._receiptsEndpoint.toString() }) spinner.stopAndPersist({ symbol: '⁂', diff --git a/test/helpers/context.js b/test/helpers/context.js index 08b2a0a..132ff4e 100644 --- a/test/helpers/context.js +++ b/test/helpers/context.js @@ -7,6 +7,7 @@ import { import { createEnv } from './env.js' import { Signer } from '@ucanto/principal/ed25519' import { createServer as createHTTPServer } from './http-server.js' +import { createReceiptsServer } from './receipt-http-server.js' import http from 'node:http' import { StoreConf } from '@web3-storage/w3up-client/stores/conf' import * as FS from 'node:fs/promises' @@ -49,6 +50,7 @@ export const provisionSpace = async (context, { space, account, provider }) => { * @typedef {import('@web3-storage/w3up-client/types').StoreAddSuccess} StoreAddSuccess * @typedef {UcantoServerTestContext & { * server: import('./http-server').TestingServer['server'] + * receiptsServer: import('./receipt-http-server.js').TestingServer['server'] * router: import('./http-server').Router * env: { alice: Record, bob: Record } * serverURL: URL @@ -61,10 +63,12 @@ export const setup = async () => { const { server, serverURL, router } = await createHTTPServer({ '/': context.connection.channel.request.bind(context.connection.channel), }) + const { server: receiptsServer, serverURL: receiptsServerUrl } = await createReceiptsServer() return Object.assign(context, { server, serverURL, + receiptsServer, router, serverRouter: router, env: { @@ -72,13 +76,13 @@ export const setup = async () => { storeName: `w3cli-test-alice-${context.service.did()}`, servicePrincipal: context.service, serviceURL: serverURL, - receiptsEndpoint: new URL('receipt', serverURL), + receiptsEndpoint: new URL('receipt', receiptsServerUrl), }), bob: createEnv({ storeName: `w3cli-test-bob-${context.service.did()}`, servicePrincipal: context.service, serviceURL: serverURL, - receiptsEndpoint: new URL('receipt', serverURL), + receiptsEndpoint: new URL('receipt', receiptsServerUrl), }), }, }) @@ -90,6 +94,7 @@ export const setup = async () => { export const teardown = async (context) => { await cleanupContext(context) context.server.close() + context.receiptsServer.close() const stores = [ context.env.alice.W3_STORE_NAME, diff --git a/test/helpers/http-server.js b/test/helpers/http-server.js index 902ed23..ef68714 100644 --- a/test/helpers/http-server.js +++ b/test/helpers/http-server.js @@ -1,13 +1,6 @@ import http from 'http' import { once } from 'events' -import { parseLink } from '@ucanto/server' -import * as Signer from '@ucanto/principal/ed25519' -import { Receipt, Message } from '@ucanto/core' -import * as CAR from '@ucanto/transport/car' -import { Assert } from '@web3-storage/content-claims/capability' -import { randomCAR } from './random.js' - /** * @typedef {import('@ucanto/interface').HTTPRequest} HTTPRequest * @typedef {import('@ucanto/server').HTTPResponse} HTTPResponse @@ -30,14 +23,6 @@ export async function createServer(router) { * @param {http.ServerResponse} response */ const listener = async (request, response) => { - if (request.url?.includes('receipt')) { - const taskCid = request.url?.split('/')[1] ?? '' - const body = await generateReceipt(taskCid) - response.writeHead(200) - response.end(body) - return undefined - } - const chunks = [] for await (const chunk of request) { chunks.push(chunk) @@ -74,39 +59,3 @@ export async function createServer(router) { serverURL: new URL(`http://127.0.0.1:${server.address().port}`), } } - -/** - * @param {string} taskCid - */ -const generateReceipt = async (taskCid) => { - const issuer = await Signer.generate() - const content = (await randomCAR(128)).cid - const locationClaim = await Assert.location.delegate({ - issuer, - audience: issuer, - with: issuer.toDIDKey(), - nb: { - content, - location: ['http://localhost'], - }, - expiration: Infinity, - }) - - const receipt = await Receipt.issue({ - issuer, - fx: { - fork: [locationClaim], - }, - ran: parseLink(taskCid), - result: { - ok: { - site: locationClaim.link(), - }, - }, - }) - - const message = await Message.build({ - receipts: [receipt], - }) - return CAR.request.encode(message).body -} diff --git a/test/helpers/receipt-http-server.js b/test/helpers/receipt-http-server.js new file mode 100644 index 0000000..bc34667 --- /dev/null +++ b/test/helpers/receipt-http-server.js @@ -0,0 +1,79 @@ +import http from 'http' +import { once } from 'events' + +import { parseLink } from '@ucanto/server' +import * as Signer from '@ucanto/principal/ed25519' +import { Receipt, Message } from '@ucanto/core' +import * as CAR from '@ucanto/transport/car' +import { Assert } from '@web3-storage/content-claims/capability' +import { randomCAR } from './random.js' + +/** + * @typedef {{ + * server: http.Server + * serverURL: URL + * }} TestingServer + */ + +/** + * @returns {Promise} + */ +export async function createReceiptsServer() { + /** + * @param {http.IncomingMessage} request + * @param {http.ServerResponse} response + */ + const listener = async (request, response) => { + const taskCid = request.url?.split('/')[1] ?? '' + const body = await generateReceipt(taskCid) + response.writeHead(200) + response.end(body) + return undefined + } + + const server = http.createServer(listener).listen() + + await once(server, 'listening') + + return { + server, + // @ts-expect-error + serverURL: new URL(`http://127.0.0.1:${server.address().port}`), + } +} + +/** + * @param {string} taskCid + */ +const generateReceipt = async (taskCid) => { + const issuer = await Signer.generate() + const content = (await randomCAR(128)).cid + const locationClaim = await Assert.location.delegate({ + issuer, + audience: issuer, + with: issuer.toDIDKey(), + nb: { + content, + location: ['http://localhost'], + }, + expiration: Infinity, + }) + + const receipt = await Receipt.issue({ + issuer, + fx: { + fork: [locationClaim], + }, + ran: parseLink(taskCid), + result: { + ok: { + site: locationClaim.link(), + }, + }, + }) + + const message = await Message.build({ + receipts: [receipt], + }) + return CAR.request.encode(message).body +}