From 7f7a707c9613c2ffd8e00d5deed7fdeb3beba2ee Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Sat, 1 Mar 2025 10:42:00 +0100 Subject: [PATCH 1/9] add tests for cbor encoding --- indexer/lib/advertisement-walker.js | 62 ++++++- indexer/test/advertisement-walker.test.js | 197 +++++++++++++++++++++- 2 files changed, 253 insertions(+), 6 deletions(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index e752621..d48e747 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -311,9 +311,16 @@ export async function fetchAdvertisedPayload (providerAddress, advertisementCid, } } - debug('entriesChunk %s %j', entriesCid, entriesChunk.Entries.slice(0, 5)) - const entryHash = entriesChunk.Entries[0]['/'].bytes - const payloadCid = CID.create(1, 0x55 /* raw */, multihash.decode(Buffer.from(entryHash, 'base64'))).toString() + let payloadCid + try { + payloadCid = processEntries(entriesCid, entriesChunk) + } catch (err) { + debug('Error processing entries: %s', err) + return { + error: /** @type {const} */('ENTRIES_NOT_RETRIEVABLE'), + previousAdvertisementCid + } + } return { previousAdvertisementCid, @@ -328,14 +335,29 @@ export async function fetchAdvertisedPayload (providerAddress, advertisementCid, * @param {number} [options.fetchTimeout] * @returns {Promise} */ -async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { +export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { const url = new URL(cid, new URL('/ipni/v1/ad/_cid_placeholder_', providerBaseUrl)) debug('Fetching %s', url) try { const res = await fetch(url, { signal: AbortSignal.timeout(fetchTimeout ?? 30_000) }) debug('Response from %s → %s %o', url, res.status, res.headers) await assertOkResponse(res) - return await res.json() + + // Determine the codec based on the CID + const parsedCid = CID.parse(cid) + const codec = parsedCid.code + + switch (codec) { + case 297: // DAG-JSON: https://github.com/multiformats/multicodec/blob/master/table.csv#L113 + return await res.json() + + case 113: // DAG-CBOR: https://github.com/multiformats/multicodec/blob/master/table.csv#L46 + const buffer = await res.arrayBuffer() + return cbor.decode(new Uint8Array(buffer)) + + default: + throw new Error(`Unknown codec ${codec} for CID ${cid}`) + } } catch (err) { if (err && typeof err === 'object') { Object.assign(err, { url }) @@ -371,3 +393,33 @@ export function parseMetadata (meta) { return { protocol } } } + +/** + * Process entries from either DAG-JSON or DAG-CBOR format + * @param {string} entriesCid - The CID of the entries + * @param {any} entriesChunk - The decoded entries + * @returns {string} The payload CID + */ +export function processEntries(entriesCid, entriesChunk) { + if (!entriesChunk.Entries || !entriesChunk.Entries.length) { + throw new Error('No entries found in DAG-CBOR response') + } + const parsedCid = CID.parse(entriesCid) + const codec = parsedCid.code + let entryBytes + switch (codec){ + case 297: // DAG-JSON + // For DAG-JSON format, the entry is a base64 encoded string + const entryHash = entriesChunk.Entries[0]['/'].bytes + entryBytes = Buffer.from(entryHash, 'base64') + break + case 113: // DAG-CBOR + // For DAG-CBOR format, the entry is already a Uint8Array with the multihash + entryBytes = entriesChunk.Entries[0] + break + default: + throw new Error(`Unsupported codec ${codec}`) + } + assert(entryBytes, 'Entry bytes must be set') + return CID.create(1, 0x55 /* raw */, multihash.decode(entryBytes)).toString() +} \ No newline at end of file diff --git a/indexer/test/advertisement-walker.test.js b/indexer/test/advertisement-walker.test.js index 3056b8a..9aa7dc5 100644 --- a/indexer/test/advertisement-walker.test.js +++ b/indexer/test/advertisement-walker.test.js @@ -13,7 +13,11 @@ import { FRISBII_ADDRESS, FRISBII_AD_CID } from './helpers/test-data.js' import { assertOkResponse } from '../lib/http-assertions.js' import * as stream from 'node:stream' import { pipeline } from 'node:stream/promises' - +import * as cbor from '@ipld/dag-cbor' +import { CID } from 'multiformats/cid' +import { processEntries } from '../lib/advertisement-walker.js' +import * as crypto from 'node:crypto' +import * as multihash from 'multiformats/hashes/digest' /** @import { ProviderInfo, WalkerState } from '../lib/typings.js' */ // TODO(bajtos) We may need to replace this with a mock index provider @@ -35,6 +39,16 @@ const knownAdvertisement = { payloadCid: 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq', pieceCid: 'baga6ea4seaqlwzed5tgjtyhrugjziutzthx2wrympvsuqhfngwdwqzvosuchmja' } + +// Known Curio advertisement in DAG-CBOR format +const knownCurioAdvertisement = { + adCid: 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm', + previousAdCid: 'bafyreibpxkmu65ezxy7rynxotbghfz3ktiapjisntepd67hghfn4hde3na', + entriesCid: 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm', + payloadCid: 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq', + pieceCid: 'baga6ea4seaqlng5bfkppozoltkk5hhyhzansp6nqndsyr3mmwubvvcnswbba' +} + describe('processNextAdvertisement', () => { it('ignores non-HTTP(s) addresses and explains the problem in the status', async () => { /** @type {ProviderInfo} */ @@ -517,3 +531,184 @@ describe('data schema for REST API', () => { }) }) }) + +describe('processEntries', () => { + const dagJsonCid = 'baguqeerayzpbdctxk4iyps45uldgibsvy6zro33vpfbehggivhcxcq5suaia' + const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' + const base64EncodedMultihash = 'Ekj/4tKFgn/U3zXiw20IkqsINRvgHdHpmKuRkFVqMswSOw==' + + // Standard multihash byte array + const entryBytes = new Uint8Array([ + 18, 32, 255, 226, 210, 133, 130, 124, + 212, 223, 53, 226, 203, 109, 8, 146, + 171, 8, 53, 27, 224, 29, 209, 233, + 152, 171, 145, 144, 85, 106, 50, 204, + 18, 55 + ]) + + const dagJsonChunk = { + Entries: [ + { + '/': { + bytes: base64EncodedMultihash + } + } + ] + } + + const dagCborChunk = { + Entries: [entryBytes] + } + it('processes DAG-JSON entries correctly', () => { + // Process the entries with the real CID + const result = processEntries(dagJsonCid, dagJsonChunk) + + // Verify the result is a valid CID string + assert(result.startsWith('bafy'), 'Result should be a valid CID starting with bafy') + assert(CID.parse(result), 'Result should be a parseable CID') + }) + + it('processes DAG-CBOR entries correctly', () => { + // Process the entries with the real CID + const result = processEntries(dagCborCid, dagCborChunk) + + // Verify the result is a valid CID string + assert(result.startsWith('bafy'), 'Result should be a valid CID starting with bafy') + assert(CID.parse(result), 'Result should be a parseable CID') + }) + + // Error handling tests + it('throws an error when entries array is empty', () => { + assert.throws( + () => processEntries(dagCborCid, { Entries: [] }), + /No entries found/ + ) + }) + + it('throws an error when Entries field is missing', () => { + assert.throws( + () => processEntries(dagCborCid, {}), + /No entries found/ + ) + }) + + it('throws an error for unsupported codec', () => { + // Use a CID with an unsupported codec + // This is a fabricated CID based on a hypothetical unsupported codec + const unsupportedCid = 'bafybgqbuttvsgv2iopu6isl75byj4qbssh7lujfnac2e6eao4dopmfwk4' + + assert.throws( + () => processEntries(unsupportedCid, dagJsonChunk), + /Unsupported codec/ + ) + }) + + // Data integrity test using real multihash operations + it('correctly creates a CID from entry data', () => { + // Create a SHA-256 hash (0x12) of a known string + const testData = 'test data for multihash' + const digest = crypto.createHash('sha256').update(testData).digest() + + // Create a proper multihash from this digest + const mh = multihash.create(0x12, digest) + + // Encode the multihash to bytes + const encodedHash = multihash.encode(mh) + + // Convert to base64 for DAG-JSON format + const base64Hash = Buffer.from(encodedHash).toString('base64') + + // Create an entries chunk with this hash + const entriesChunk = { + Entries: [ + { + '/': { + bytes: base64Hash + } + } + ] + } + + // Process the entries + const result = processEntries(dagJsonCid, entriesChunk) + + // Create the expected CID directly + const expectedCid = CID.create(1, 0x55, mh).toString() + + // They should match + assert.strictEqual(result, expectedCid, 'CID should match the one created directly') + }) + + // Test with entries from CBOR encoding/decoding + it('correctly handles DAG-CBOR entries serialized with @ipld/dag-cbor', () => { + // Use a real DAG-CBOR CID + const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' + + // Create a SHA-256 hash (0x12) of a known string + const testData = 'test data for DAG-CBOR' + const digest = require('crypto').createHash('sha256').update(testData).digest() + + // Create a proper multihash from this digest + const mh = multihash.create(0x12, digest) + + // Encode the multihash to bytes + const encodedHash = multihash.encode(mh) + + // Create an entries data structure + const entriesData = { + Entries: [encodedHash] + } + + // Encode it with the CBOR library + const encoded = cbor.encode(entriesData) + + // Decode it back to simulate network response + const decoded = cbor.decode(encoded) + + // Process the entries + const result = processEntries(dagCborCid, decoded) + + // Create the expected CID directly + const expectedCid = CID.create(1, 0x55, mh).toString() + + // They should match + assert.strictEqual(result, expectedCid, 'CID should match the one created directly') + }) + + // Error case tests with real data + it('handles malformed base64 in DAG-JSON gracefully', () => { + // Use a real DAG-JSON CID + const dagJsonCid = 'baguqeerayzpbdctxk4iyps45uldgibsvy6zro33vpfbehggivhcxcq5suaia' + + const malformedChunk = { + Entries: [ + { + '/': { + bytes: 'This-is-not-valid-base64!' + } + } + ] + } + + // We expect an error when processing this malformed data + assert.throws( + () => processEntries(dagJsonCid, malformedChunk), + /Invalid character/ + ) + }) + + it('handles invalid multihash in DAG-CBOR gracefully', () => { + // Use a real DAG-CBOR CID + const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' + + const invalidChunk = { + Entries: [new Uint8Array([0, 1, 2, 3])] // Too short to be a valid multihash + } + + // We expect an error when processing this invalid multihash + assert.throws( + () => processEntries(dagCborCid, invalidChunk), + /Unexpected/ + ) + }) + }) \ No newline at end of file From 6f8023000ae94562e9afce73789b303b5ed5ebd1 Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Sat, 1 Mar 2025 13:52:17 +0100 Subject: [PATCH 2/9] fmt --- indexer/lib/advertisement-walker.js | 55 ++-- indexer/package.json | 2 +- indexer/test/advertisement-walker.test.js | 381 ++++++++++++---------- 3 files changed, 240 insertions(+), 198 deletions(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index d48e747..0369d92 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -343,21 +343,21 @@ export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { debug('Response from %s → %s %o', url, res.status, res.headers) await assertOkResponse(res) - // Determine the codec based on the CID - const parsedCid = CID.parse(cid) - const codec = parsedCid.code - - switch (codec) { - case 297: // DAG-JSON: https://github.com/multiformats/multicodec/blob/master/table.csv#L113 - return await res.json() - - case 113: // DAG-CBOR: https://github.com/multiformats/multicodec/blob/master/table.csv#L46 - const buffer = await res.arrayBuffer() - return cbor.decode(new Uint8Array(buffer)) - - default: - throw new Error(`Unknown codec ${codec} for CID ${cid}`) - } + // Determine the codec based on the CID + const parsedCid = CID.parse(cid) + const codec = parsedCid.code + + switch (codec) { + case 297: // DAG-JSON: https://github.com/multiformats/multicodec/blob/master/table.csv#L113 + return await res.json() + + case 113: // DAG-CBOR: https://github.com/multiformats/multicodec/blob/master/table.csv#L46 + const buffer = await res.arrayBuffer() + return cbor.decode(new Uint8Array(buffer)) + + default: + throw new Error(`Unknown codec ${codec} for CID ${cid}`) + } } catch (err) { if (err && typeof err === 'object') { Object.assign(err, { url }) @@ -397,29 +397,42 @@ export function parseMetadata (meta) { /** * Process entries from either DAG-JSON or DAG-CBOR format * @param {string} entriesCid - The CID of the entries - * @param {any} entriesChunk - The decoded entries + * @param {{Entries: Array}} entriesChunk - The decoded entries * @returns {string} The payload CID */ -export function processEntries(entriesCid, entriesChunk) { +export function processEntries (entriesCid, entriesChunk) { if (!entriesChunk.Entries || !entriesChunk.Entries.length) { throw new Error('No entries found in DAG-CBOR response') } const parsedCid = CID.parse(entriesCid) const codec = parsedCid.code let entryBytes - switch (codec){ + switch (codec) { case 297: // DAG-JSON // For DAG-JSON format, the entry is a base64 encoded string - const entryHash = entriesChunk.Entries[0]['/'].bytes - entryBytes = Buffer.from(entryHash, 'base64') + const entry = entriesChunk.Entries[0] + // Check that entry is an object with a '/' property + if (!entry || typeof entry !== 'object' || !('/' in entry)) { + throw new Error('DAG-JSON entry must have a "/" property') + } + + // Verify the '/' property is an object with 'bytes' property + // In DAG-JSON, CIDs are represented as objects with a '/' property that contains 'bytes' + if (!entry['/'] || typeof entry['/'] !== 'object' || !('bytes' in entry['/'])) { + throw new Error('DAG-JSON entry\'s "/" property must be a CID object with a bytes property') + } + + const entryHash = entry['/'].bytes + entryBytes = Buffer.from(String(entryHash), 'base64') break case 113: // DAG-CBOR // For DAG-CBOR format, the entry is already a Uint8Array with the multihash entryBytes = entriesChunk.Entries[0] + assert(entryBytes instanceof Uint8Array, 'DAG-CBOR entry must be a Uint8Array') break default: throw new Error(`Unsupported codec ${codec}`) } assert(entryBytes, 'Entry bytes must be set') return CID.create(1, 0x55 /* raw */, multihash.decode(entryBytes)).toString() -} \ No newline at end of file +} diff --git a/indexer/package.json b/indexer/package.json index c06c3d0..e305114 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "node bin/piece-indexer.js", "lint": "standard", - "test": "node --test --test-reporter=spec" + "test": "node --test --test-reporter=spec --test-concurrency=1" }, "devDependencies": { "standard": "^17.1.2" diff --git a/indexer/test/advertisement-walker.test.js b/indexer/test/advertisement-walker.test.js index 9aa7dc5..b1e2a35 100644 --- a/indexer/test/advertisement-walker.test.js +++ b/indexer/test/advertisement-walker.test.js @@ -1,12 +1,14 @@ import { RedisRepository } from '@filecoin-station/spark-piece-indexer-repository' import { Redis } from 'ioredis' import assert from 'node:assert' -import { after, before, beforeEach, describe, it } from 'node:test' +import { after, afterEach, before, beforeEach, describe, it, mock } from 'node:test' import { setTimeout } from 'node:timers/promises' import { fetchAdvertisedPayload, processNextAdvertisement, walkOneStep + , processEntries, + fetchCid } from '../lib/advertisement-walker.js' import { givenHttpServer } from './helpers/http-server.js' import { FRISBII_ADDRESS, FRISBII_AD_CID } from './helpers/test-data.js' @@ -15,7 +17,6 @@ import * as stream from 'node:stream' import { pipeline } from 'node:stream/promises' import * as cbor from '@ipld/dag-cbor' import { CID } from 'multiformats/cid' -import { processEntries } from '../lib/advertisement-walker.js' import * as crypto from 'node:crypto' import * as multihash from 'multiformats/hashes/digest' /** @import { ProviderInfo, WalkerState } from '../lib/typings.js' */ @@ -40,15 +41,6 @@ const knownAdvertisement = { pieceCid: 'baga6ea4seaqlwzed5tgjtyhrugjziutzthx2wrympvsuqhfngwdwqzvosuchmja' } -// Known Curio advertisement in DAG-CBOR format -const knownCurioAdvertisement = { - adCid: 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm', - previousAdCid: 'bafyreibpxkmu65ezxy7rynxotbghfz3ktiapjisntepd67hghfn4hde3na', - entriesCid: 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm', - payloadCid: 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq', - pieceCid: 'baga6ea4seaqlng5bfkppozoltkk5hhyhzansp6nqndsyr3mmwubvvcnswbba' -} - describe('processNextAdvertisement', () => { it('ignores non-HTTP(s) addresses and explains the problem in the status', async () => { /** @type {ProviderInfo} */ @@ -533,182 +525,219 @@ describe('data schema for REST API', () => { }) describe('processEntries', () => { - const dagJsonCid = 'baguqeerayzpbdctxk4iyps45uldgibsvy6zro33vpfbehggivhcxcq5suaia' - const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' - const base64EncodedMultihash = 'Ekj/4tKFgn/U3zXiw20IkqsINRvgHdHpmKuRkFVqMswSOw==' - - // Standard multihash byte array - const entryBytes = new Uint8Array([ - 18, 32, 255, 226, 210, 133, 130, 124, - 212, 223, 53, 226, 203, 109, 8, 146, - 171, 8, 53, 27, 224, 29, 209, 233, - 152, 171, 145, 144, 85, 106, 50, 204, - 18, 55 - ]) - + // Use a real DAG-JSON CID that will naturally have codec 0x0129 (297) + // CIDs that start with 'bagu' are DAG-JSON encoded + const dagJsonCid = 'baguqeeraa5mjufqdwuwrrrqboctnn3vhdlq63rj3hce2igpzbmae7sazkfea' + // Use a real DAG-CBOR CID that will naturally have codec 0x71 (113) + // CIDs that start with 'bafy' are DAG-CBOR encoded + const dagCborCid = 'bafyreibpxkmu65ezxy7rynxotbghfz3ktiapjisntepd67hghfn4hde3na' + const testData = 'test data for multihash' + // Create a proper multihash from this digest + const mh = multihash.create(0x12, crypto.createHash('sha256').update(testData).digest()) + // @ts-ignore + const entryBytes = Buffer.from(mh.bytes).toString('base64') const dagJsonChunk = { Entries: [ { '/': { - bytes: base64EncodedMultihash + bytes: entryBytes } } ] } - + const dagCborChunk = { - Entries: [entryBytes] + Entries: [mh.bytes] } - it('processes DAG-JSON entries correctly', () => { - // Process the entries with the real CID - const result = processEntries(dagJsonCid, dagJsonChunk) - - // Verify the result is a valid CID string - assert(result.startsWith('bafy'), 'Result should be a valid CID starting with bafy') - assert(CID.parse(result), 'Result should be a parseable CID') - }) - - it('processes DAG-CBOR entries correctly', () => { - // Process the entries with the real CID - const result = processEntries(dagCborCid, dagCborChunk) - - // Verify the result is a valid CID string - assert(result.startsWith('bafy'), 'Result should be a valid CID starting with bafy') - assert(CID.parse(result), 'Result should be a parseable CID') - }) - - // Error handling tests - it('throws an error when entries array is empty', () => { - assert.throws( - () => processEntries(dagCborCid, { Entries: [] }), - /No entries found/ - ) - }) - - it('throws an error when Entries field is missing', () => { - assert.throws( - () => processEntries(dagCborCid, {}), - /No entries found/ - ) - }) - - it('throws an error for unsupported codec', () => { - // Use a CID with an unsupported codec - // This is a fabricated CID based on a hypothetical unsupported codec - const unsupportedCid = 'bafybgqbuttvsgv2iopu6isl75byj4qbssh7lujfnac2e6eao4dopmfwk4' - - assert.throws( - () => processEntries(unsupportedCid, dagJsonChunk), - /Unsupported codec/ - ) - }) - - // Data integrity test using real multihash operations - it('correctly creates a CID from entry data', () => { - // Create a SHA-256 hash (0x12) of a known string - const testData = 'test data for multihash' - const digest = crypto.createHash('sha256').update(testData).digest() - - // Create a proper multihash from this digest - const mh = multihash.create(0x12, digest) - - // Encode the multihash to bytes - const encodedHash = multihash.encode(mh) - - // Convert to base64 for DAG-JSON format - const base64Hash = Buffer.from(encodedHash).toString('base64') - - // Create an entries chunk with this hash - const entriesChunk = { - Entries: [ - { - '/': { - bytes: base64Hash - } + it('processes DAG-JSON entries correctly', () => { + // Process the entries with the real CID + const result = processEntries(dagJsonCid, dagJsonChunk) + + // Verify the result is a valid CID string + assert(CID.parse(result), 'Result should be a parseable CID') + }) + + it('processes DAG-CBOR entries correctly', () => { + // Process the entries with the real CID + const result = processEntries(dagCborCid, dagCborChunk) + + // Verify the result is a valid CID string + assert(CID.parse(result), 'Result should be a parseable CID') + }) + + // Error handling tests + it('throws an error when entries array is empty', () => { + assert.throws( + () => processEntries(dagCborCid, { Entries: [] }), + /No entries found/ + ) + }) + + it('throws an error when Entries field is missing', () => { + assert.throws( + // @ts-ignore + () => processEntries(dagCborCid, {}), + /No entries found/ + ) + }) + + it('throws an error for unsupported codec', () => { + // Use a CID with an unsupported codec + const unsupportedCid = 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq' + + assert.throws( + () => processEntries(unsupportedCid, dagJsonChunk), + /Unsupported codec/ + ) + }) + + // Data integrity test using real multihash operations + it('correctly creates a CID from entry data', () => { + // Process the entries + const result = processEntries(dagJsonCid, dagJsonChunk) + + // Create the expected CID directly + const expectedCid = CID.create(1, 0x55, mh).toString() + + // They should match + assert.strictEqual(result, expectedCid, 'CID should match the one created directly') + }) + + // Test with entries from CBOR encoding/decoding + it('correctly handles DAG-CBOR entries serialized with @ipld/dag-cbor', () => { + // Process the entries + const result = processEntries(dagCborCid, dagCborChunk) + + // Create the expected CID directly + const expectedCid = CID.create(1, 0x55, mh).toString() + + // They should match + assert.strictEqual(result, expectedCid, 'CID should match the one created directly') + }) + + // Error case tests with real data + it('handles malformed base64 in DAG-JSON gracefully', () => { + const malformedChunk = { + Entries: [ + { + '/': { + bytes: 'This-is-not-valid-base64!' } - ] - } - - // Process the entries - const result = processEntries(dagJsonCid, entriesChunk) - - // Create the expected CID directly - const expectedCid = CID.create(1, 0x55, mh).toString() - - // They should match - assert.strictEqual(result, expectedCid, 'CID should match the one created directly') - }) - - // Test with entries from CBOR encoding/decoding - it('correctly handles DAG-CBOR entries serialized with @ipld/dag-cbor', () => { - // Use a real DAG-CBOR CID - const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' - - // Create a SHA-256 hash (0x12) of a known string - const testData = 'test data for DAG-CBOR' - const digest = require('crypto').createHash('sha256').update(testData).digest() - - // Create a proper multihash from this digest - const mh = multihash.create(0x12, digest) - - // Encode the multihash to bytes - const encodedHash = multihash.encode(mh) - - // Create an entries data structure - const entriesData = { - Entries: [encodedHash] - } - - // Encode it with the CBOR library - const encoded = cbor.encode(entriesData) - - // Decode it back to simulate network response - const decoded = cbor.decode(encoded) - - // Process the entries - const result = processEntries(dagCborCid, decoded) - - // Create the expected CID directly - const expectedCid = CID.create(1, 0x55, mh).toString() - - // They should match - assert.strictEqual(result, expectedCid, 'CID should match the one created directly') + } + ] + } + + // We expect an error when processing this malformed data + assert.throws( + () => processEntries(dagJsonCid, malformedChunk), + /Incorrect length/ + ) + }) + + it('handles invalid multihash in DAG-CBOR gracefully', () => { + const invalidChunk = { + Entries: [new Uint8Array([0, 1, 2, 3])] // Too short to be a valid multihash + } + + // We expect an error when processing this invalid multihash + assert.throws( + () => processEntries(dagCborCid, invalidChunk), + /Incorrect length/ + ) + }) +}) + +describe('fetchCid', () => { + // Store the original fetch function before each test + /** + * @type {{ (input: string | URL | globalThis.Request, init?: RequestInit): Promise; (input: string | URL | globalThis.Request, init?: RequestInit): Promise; }} + */ + let originalFetch + // Use a real DAG-JSON CID that will naturally have codec 0x0129 (297) + // CIDs that start with 'bagu' are DAG-JSON encoded + const dagJsonCid = 'baguqeerayzpbdctxk4iyps45uldgibsvy6zro33vpfbehggivhcxcq5suaia' + // Sample JSON response + const jsonResponse = { test: 'value' } + // Use a real DAG-CBOR CID that will naturally have codec 0x71 (113) + // CIDs that start with 'bafy' are DAG-CBOR encoded + const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' + + beforeEach(() => { + originalFetch = globalThis.fetch + }) + + afterEach(() => { + // Restore the original fetch function after each test + globalThis.fetch = originalFetch + }) + + it('uses DAG-JSON codec (0x0129) to parse response as JSON', async () => { + // Mock fetch to return JSON + globalThis.fetch = mock.fn(() => { + return Promise.resolve(new Response(JSON.stringify({ + ok: true, + status: 200, + json: () => Promise.resolve(jsonResponse), + arrayBuffer: () => { throw new Error('Should not call arrayBuffer for JSON') } + }))) }) - - // Error case tests with real data - it('handles malformed base64 in DAG-JSON gracefully', () => { - // Use a real DAG-JSON CID - const dagJsonCid = 'baguqeerayzpbdctxk4iyps45uldgibsvy6zro33vpfbehggivhcxcq5suaia' - - const malformedChunk = { - Entries: [ - { - '/': { - bytes: 'This-is-not-valid-base64!' - } - } - ] - } - - // We expect an error when processing this malformed data - assert.throws( - () => processEntries(dagJsonCid, malformedChunk), - /Invalid character/ - ) + + const parsedCid = CID.parse(dagJsonCid) + assert.strictEqual(parsedCid.code, 297) + + const result = await fetchCid('http://example.com', dagJsonCid) + + // Verify we got the JSON response + assert.deepStrictEqual(result, jsonResponse) + }) + + it('uses DAG-CBOR codec (0x71) to parse response as CBOR', async () => { + const cborData = cbor.encode(jsonResponse) + + // Mock fetch to return ArrayBuffer + globalThis.fetch = mock.fn(() => { + return Promise.resolve(new Response(JSON.stringify({ + ok: true, + status: 200, + json: () => { throw new Error('Should not call json for CBOR') }, + arrayBuffer: () => Promise.resolve(cborData.buffer) + }))) }) - - it('handles invalid multihash in DAG-CBOR gracefully', () => { - // Use a real DAG-CBOR CID - const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' - - const invalidChunk = { - Entries: [new Uint8Array([0, 1, 2, 3])] // Too short to be a valid multihash - } - - // We expect an error when processing this invalid multihash - assert.throws( - () => processEntries(dagCborCid, invalidChunk), - /Unexpected/ - ) + + const parsedCid = CID.parse(dagCborCid) + assert.strictEqual(parsedCid.code, 113) + + const result = await fetchCid('http://example.com', dagCborCid) + + // Verify we got the decoded CBOR data + assert.deepStrictEqual(result, jsonResponse) + }) + + it('throws an error for unknown codec', async () => { + // Mock fetch to return JSON + globalThis.fetch = mock.fn(() => { + return Promise.resolve(new Response(JSON.stringify({ + ok: true, + status: 200, + json: () => { throw new Error('Should not call json for CBOR') }, + arrayBuffer: () => { throw new Error('Should not call arrayBuffer for fallback') } + }))) }) - }) \ No newline at end of file + + // Use a CID with a codec that is neither DAG-JSON (0x0129) nor DAG-CBOR (0x71) + // This is a raw codec (0x55) CID + const unknownCodecCid = 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq' + const parsedCid = CID.parse(unknownCodecCid) + assert.strictEqual(parsedCid.code, 85) + const errorMessage = 'To parse non base32, base36 or base58btc encoded CID multibase decoder must be provided' + try { + await fetchCid('http://example.com', 'testcid') + assert.fail('fetchCid should have thrown an error') + } catch (error) { + // Check the error message + + // @ts-ignore + assert.ok(error.message.includes(errorMessage), `Error message should include: ${errorMessage}`) + } + }) +}) From 264a76d57151f0f3f721766ea25541fbc3bbaa67 Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Sat, 1 Mar 2025 13:54:56 +0100 Subject: [PATCH 3/9] fix tests --- indexer/test/advertisement-walker.test.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/indexer/test/advertisement-walker.test.js b/indexer/test/advertisement-walker.test.js index b1e2a35..63b7595 100644 --- a/indexer/test/advertisement-walker.test.js +++ b/indexer/test/advertisement-walker.test.js @@ -673,13 +673,14 @@ describe('fetchCid', () => { it('uses DAG-JSON codec (0x0129) to parse response as JSON', async () => { // Mock fetch to return JSON + // @ts-ignore globalThis.fetch = mock.fn(() => { - return Promise.resolve(new Response(JSON.stringify({ + return Promise.resolve({ ok: true, status: 200, json: () => Promise.resolve(jsonResponse), arrayBuffer: () => { throw new Error('Should not call arrayBuffer for JSON') } - }))) + }) }) const parsedCid = CID.parse(dagJsonCid) @@ -695,13 +696,14 @@ describe('fetchCid', () => { const cborData = cbor.encode(jsonResponse) // Mock fetch to return ArrayBuffer + // @ts-ignore globalThis.fetch = mock.fn(() => { - return Promise.resolve(new Response(JSON.stringify({ + return Promise.resolve({ ok: true, status: 200, json: () => { throw new Error('Should not call json for CBOR') }, arrayBuffer: () => Promise.resolve(cborData.buffer) - }))) + }) }) const parsedCid = CID.parse(dagCborCid) @@ -715,13 +717,15 @@ describe('fetchCid', () => { it('throws an error for unknown codec', async () => { // Mock fetch to return JSON + + // @ts-ignore globalThis.fetch = mock.fn(() => { - return Promise.resolve(new Response(JSON.stringify({ + return Promise.resolve({ ok: true, status: 200, json: () => { throw new Error('Should not call json for CBOR') }, arrayBuffer: () => { throw new Error('Should not call arrayBuffer for fallback') } - }))) + }) }) // Use a CID with a codec that is neither DAG-JSON (0x0129) nor DAG-CBOR (0x71) From 71f311c244a91a48d9572f69b277423046774ab0 Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Sat, 1 Mar 2025 14:15:35 +0100 Subject: [PATCH 4/9] formatting --- indexer/lib/advertisement-walker.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index 0369d92..69b5ca2 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -351,9 +351,9 @@ export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { case 297: // DAG-JSON: https://github.com/multiformats/multicodec/blob/master/table.csv#L113 return await res.json() - case 113: // DAG-CBOR: https://github.com/multiformats/multicodec/blob/master/table.csv#L46 + case 113: { // DAG-CBOR: https://github.com/multiformats/multicodec/blob/master/table.csv#L46 const buffer = await res.arrayBuffer() - return cbor.decode(new Uint8Array(buffer)) + return cbor.decode(new Uint8Array(buffer)) } default: throw new Error(`Unknown codec ${codec} for CID ${cid}`) @@ -408,7 +408,8 @@ export function processEntries (entriesCid, entriesChunk) { const codec = parsedCid.code let entryBytes switch (codec) { - case 297: // DAG-JSON + case 297: { + // DAG-JSON // For DAG-JSON format, the entry is a base64 encoded string const entry = entriesChunk.Entries[0] // Check that entry is an object with a '/' property @@ -424,12 +425,13 @@ export function processEntries (entriesCid, entriesChunk) { const entryHash = entry['/'].bytes entryBytes = Buffer.from(String(entryHash), 'base64') - break - case 113: // DAG-CBOR + break } + case 113: { + // DAG-CBOR // For DAG-CBOR format, the entry is already a Uint8Array with the multihash entryBytes = entriesChunk.Entries[0] assert(entryBytes instanceof Uint8Array, 'DAG-CBOR entry must be a Uint8Array') - break + break } default: throw new Error(`Unsupported codec ${codec}`) } From 97db5532817e93d2fc4661f8f266ba9fb698150d Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Mon, 3 Mar 2025 12:22:27 +0100 Subject: [PATCH 5/9] add test for fetching cbor cid --- indexer/lib/advertisement-walker.js | 12 ++++++++- indexer/package.json | 2 +- indexer/test/advertisement-walker.test.js | 32 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index 69b5ca2..5968892 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -336,7 +336,17 @@ export async function fetchAdvertisedPayload (providerAddress, advertisementCid, * @returns {Promise} */ export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { - const url = new URL(cid, new URL('/ipni/v1/ad/_cid_placeholder_', providerBaseUrl)) + let url = new URL(providerBaseUrl) + + // Check if the URL already has a path + if (!(url.pathname && url.pathname !== '/')) { + // If no path, add the standard path with a placeholder + url = new URL('/ipni/v1/ad/_cid_placeholder_', providerBaseUrl) + } else { + // If there's already a path, append the additional path + url = new URL(`${url.pathname}/ipni/v1/ad/_cid_placeholder_`, url.origin) + } + url = new URL(cid, url) debug('Fetching %s', url) try { const res = await fetch(url, { signal: AbortSignal.timeout(fetchTimeout ?? 30_000) }) diff --git a/indexer/package.json b/indexer/package.json index e305114..c06c3d0 100644 --- a/indexer/package.json +++ b/indexer/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "node bin/piece-indexer.js", "lint": "standard", - "test": "node --test --test-reporter=spec --test-concurrency=1" + "test": "node --test --test-reporter=spec" }, "devDependencies": { "standard": "^17.1.2" diff --git a/indexer/test/advertisement-walker.test.js b/indexer/test/advertisement-walker.test.js index 63b7595..0646d3e 100644 --- a/indexer/test/advertisement-walker.test.js +++ b/indexer/test/advertisement-walker.test.js @@ -10,6 +10,7 @@ import { , processEntries, fetchCid } from '../lib/advertisement-walker.js' +import pRetry from 'p-retry' import { givenHttpServer } from './helpers/http-server.js' import { FRISBII_ADDRESS, FRISBII_AD_CID } from './helpers/test-data.js' import { assertOkResponse } from '../lib/http-assertions.js' @@ -744,4 +745,35 @@ describe('fetchCid', () => { assert.ok(error.message.includes(errorMessage), `Error message should include: ${errorMessage}`) } }) + it('correctly fetches and processes real DAG-CBOR data from Curio provider', async function () { + // Use a real Curio provider and known DAG-CBOR CID + const curioProviderUrl = 'https://f03303347-market.duckdns.org/ipni-provider/12D3KooWJ91c6xQshrNe7QAXPFAaeRrHWq2UrgXGPf8UmMZMwyZ5' + const dagCborCid = 'baguqeeracgnw2ecmhaa6qkb3irrgjjk5zt5fes7wwwpb4aymoaogzyvvbrma' + // Use the real fetchCid function with the original fetch implementation + globalThis.fetch = originalFetch + /** @type {{ Entries: { [key: string]: string } }} */ + // @ts-ignore + const result = await pRetry( + () => + ( + fetchCid(curioProviderUrl, dagCborCid) + ) + ) + + // Verify the result has the expected structure for DAG-CBOR entries + assert(result, 'Expected a non-null result') + assert(result.Entries, 'Result should have Entries property') + const entriesCid = result.Entries['/'] + /** @type {{Entries: Array}} */ + // @ts-ignore + const entriesChunk = await pRetry( + () => + ( + fetchCid(curioProviderUrl, entriesCid) + ) + ) + const payloadCid = processEntries(entriesCid, entriesChunk) + console.log(payloadCid) + assert.deepStrictEqual(payloadCid, 'bafkreiefrclz7c6w57yl4u7uiq4kvht4z7pits5jpcj3cajbvowik3rvhm') + }) }) From dfa3a0b368597a76e8a9cd0fbc1a7d85a0e963ef Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Mon, 3 Mar 2025 13:03:53 +0100 Subject: [PATCH 6/9] fmt --- indexer/test/advertisement-walker.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/indexer/test/advertisement-walker.test.js b/indexer/test/advertisement-walker.test.js index 0646d3e..ed9a3d5 100644 --- a/indexer/test/advertisement-walker.test.js +++ b/indexer/test/advertisement-walker.test.js @@ -41,7 +41,6 @@ const knownAdvertisement = { payloadCid: 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq', pieceCid: 'baga6ea4seaqlwzed5tgjtyhrugjziutzthx2wrympvsuqhfngwdwqzvosuchmja' } - describe('processNextAdvertisement', () => { it('ignores non-HTTP(s) addresses and explains the problem in the status', async () => { /** @type {ProviderInfo} */ From 85bbb6bc406d03e1171108c70a0684e17537db28 Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl <113891786+NikolasHaimerl@users.noreply.github.com> Date: Tue, 4 Mar 2025 08:42:21 +0100 Subject: [PATCH 7/9] Update indexer/lib/advertisement-walker.js MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Miroslav Bajtoš --- indexer/lib/advertisement-walker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index 5968892..f5c955a 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -412,7 +412,7 @@ export function parseMetadata (meta) { */ export function processEntries (entriesCid, entriesChunk) { if (!entriesChunk.Entries || !entriesChunk.Entries.length) { - throw new Error('No entries found in DAG-CBOR response') + throw new Error(`No entries found in the response for ${entriesCid}`) } const parsedCid = CID.parse(entriesCid) const codec = parsedCid.code From 53575868ee5d5d459a85b5f71af460bb2f71376c Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl <113891786+NikolasHaimerl@users.noreply.github.com> Date: Tue, 4 Mar 2025 08:42:41 +0100 Subject: [PATCH 8/9] Update indexer/lib/advertisement-walker.js MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Miroslav Bajtoš --- indexer/lib/advertisement-walker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index f5c955a..cf5b502 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -430,7 +430,7 @@ export function processEntries (entriesCid, entriesChunk) { // Verify the '/' property is an object with 'bytes' property // In DAG-JSON, CIDs are represented as objects with a '/' property that contains 'bytes' if (!entry['/'] || typeof entry['/'] !== 'object' || !('bytes' in entry['/'])) { - throw new Error('DAG-JSON entry\'s "/" property must be a CID object with a bytes property') + throw new Error('DAG-JSON entry\'s "/" property must be an object with a "bytes" property') } const entryHash = entry['/'].bytes From 4aa9958c4f79666e528094e90883b62a935f49cf Mon Sep 17 00:00:00 2001 From: Nikolas Haimerl Date: Tue, 4 Mar 2025 09:09:53 +0100 Subject: [PATCH 9/9] address pr threads --- indexer/lib/advertisement-walker.js | 18 ++-- indexer/test/advertisement-walker.test.js | 107 ++++++++++------------ 2 files changed, 62 insertions(+), 63 deletions(-) diff --git a/indexer/lib/advertisement-walker.js b/indexer/lib/advertisement-walker.js index 5968892..24f6771 100644 --- a/indexer/lib/advertisement-walker.js +++ b/indexer/lib/advertisement-walker.js @@ -317,7 +317,7 @@ export async function fetchAdvertisedPayload (providerAddress, advertisementCid, } catch (err) { debug('Error processing entries: %s', err) return { - error: /** @type {const} */('ENTRIES_NOT_RETRIEVABLE'), + error: /** @type {const} */('PAYLOAD_CID_NOT_EXTRACTABLE'), previousAdvertisementCid } } @@ -333,9 +333,10 @@ export async function fetchAdvertisedPayload (providerAddress, advertisementCid, * @param {string} cid * @param {object} [options] * @param {number} [options.fetchTimeout] + * @param {typeof fetch} [options.fetchMethod] * @returns {Promise} */ -export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { +export async function fetchCid (providerBaseUrl, cid, { fetchTimeout, fetchMethod } = {}) { let url = new URL(providerBaseUrl) // Check if the URL already has a path @@ -349,7 +350,7 @@ export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { url = new URL(cid, url) debug('Fetching %s', url) try { - const res = await fetch(url, { signal: AbortSignal.timeout(fetchTimeout ?? 30_000) }) + const res = await (fetchMethod ?? fetch)(url, { signal: AbortSignal.timeout(fetchTimeout ?? 30_000) }) debug('Response from %s → %s %o', url, res.status, res.headers) await assertOkResponse(res) @@ -357,11 +358,13 @@ export async function fetchCid (providerBaseUrl, cid, { fetchTimeout } = {}) { const parsedCid = CID.parse(cid) const codec = parsedCid.code + // List of codecs: + // https://github.com/multiformats/multicodec/blob/f18c7ba5f4d3cc74afb51ee79978939abc0e6556/table.csv switch (codec) { - case 297: // DAG-JSON: https://github.com/multiformats/multicodec/blob/master/table.csv#L113 + case 297: // DAG-JSON return await res.json() - case 113: { // DAG-CBOR: https://github.com/multiformats/multicodec/blob/master/table.csv#L46 + case 113: { // DAG-CBOR const buffer = await res.arrayBuffer() return cbor.decode(new Uint8Array(buffer)) } @@ -434,7 +437,10 @@ export function processEntries (entriesCid, entriesChunk) { } const entryHash = entry['/'].bytes - entryBytes = Buffer.from(String(entryHash), 'base64') + if (typeof entryHash !== 'string') { + throw new Error('DAG-JSON entry\'s ["/"]["bytes"] property must be a string') + } + entryBytes = Buffer.from(entryHash, 'base64') break } case 113: { // DAG-CBOR diff --git a/indexer/test/advertisement-walker.test.js b/indexer/test/advertisement-walker.test.js index ed9a3d5..ce4c8d8 100644 --- a/indexer/test/advertisement-walker.test.js +++ b/indexer/test/advertisement-walker.test.js @@ -1,7 +1,7 @@ import { RedisRepository } from '@filecoin-station/spark-piece-indexer-repository' import { Redis } from 'ioredis' import assert from 'node:assert' -import { after, afterEach, before, beforeEach, describe, it, mock } from 'node:test' +import { after, before, beforeEach, describe, it, mock } from 'node:test' import { setTimeout } from 'node:timers/promises' import { fetchAdvertisedPayload, @@ -648,98 +648,81 @@ describe('processEntries', () => { }) describe('fetchCid', () => { - // Store the original fetch function before each test - /** - * @type {{ (input: string | URL | globalThis.Request, init?: RequestInit): Promise; (input: string | URL | globalThis.Request, init?: RequestInit): Promise; }} - */ - let originalFetch // Use a real DAG-JSON CID that will naturally have codec 0x0129 (297) // CIDs that start with 'bagu' are DAG-JSON encoded const dagJsonCid = 'baguqeerayzpbdctxk4iyps45uldgibsvy6zro33vpfbehggivhcxcq5suaia' // Sample JSON response - const jsonResponse = { test: 'value' } + const testResponse = { test: 'value' } // Use a real DAG-CBOR CID that will naturally have codec 0x71 (113) // CIDs that start with 'bafy' are DAG-CBOR encoded const dagCborCid = 'bafyreictdikh363qfxsmjp63i6kup6aukjqfpd4r6wbhbiz2ctuji4bofm' - beforeEach(() => { - originalFetch = globalThis.fetch - }) - - afterEach(() => { - // Restore the original fetch function after each test - globalThis.fetch = originalFetch - }) - it('uses DAG-JSON codec (0x0129) to parse response as JSON', async () => { // Mock fetch to return JSON // @ts-ignore - globalThis.fetch = mock.fn(() => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => Promise.resolve(jsonResponse), - arrayBuffer: () => { throw new Error('Should not call arrayBuffer for JSON') } - }) - }) + const mockFetch = mock.fn(() => Promise.resolve({ + ok: true, + status: 200, + json: () => Promise.resolve(testResponse), + arrayBuffer: () => { throw new Error('Should not call arrayBuffer for JSON') } + })) const parsedCid = CID.parse(dagJsonCid) assert.strictEqual(parsedCid.code, 297) - const result = await fetchCid('http://example.com', dagJsonCid) + // @ts-ignore + const result = await fetchCid('http://example.com', dagJsonCid, { fetchMethod: mockFetch }) // Verify we got the JSON response - assert.deepStrictEqual(result, jsonResponse) + assert.deepStrictEqual(result, testResponse) }) it('uses DAG-CBOR codec (0x71) to parse response as CBOR', async () => { - const cborData = cbor.encode(jsonResponse) + const cborData = cbor.encode(testResponse) // Mock fetch to return ArrayBuffer // @ts-ignore - globalThis.fetch = mock.fn(() => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => { throw new Error('Should not call json for CBOR') }, - arrayBuffer: () => Promise.resolve(cborData.buffer) - }) + const mockFetch = mock.fn(() => Promise.resolve({ + ok: true, + status: 200, + json: () => { throw new Error('Should not call json for CBOR') }, + arrayBuffer: () => Promise.resolve(cborData.buffer) }) + ) const parsedCid = CID.parse(dagCborCid) assert.strictEqual(parsedCid.code, 113) - const result = await fetchCid('http://example.com', dagCborCid) + // @ts-ignore + const result = await fetchCid('http://example.com', dagCborCid, { fetchMethod: mockFetch }) // Verify we got the decoded CBOR data - assert.deepStrictEqual(result, jsonResponse) + assert.deepStrictEqual(result, testResponse) }) it('throws an error for unknown codec', async () => { // Mock fetch to return JSON // @ts-ignore - globalThis.fetch = mock.fn(() => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => { throw new Error('Should not call json for CBOR') }, - arrayBuffer: () => { throw new Error('Should not call arrayBuffer for fallback') } - }) + const mockFetch = mock.fn(() => Promise.resolve({ + ok: true, + status: 200, + json: () => { throw new Error('Should not call json for CBOR') }, + arrayBuffer: () => { throw new Error('Should not call arrayBuffer for fallback') } }) + ) // Use a CID with a codec that is neither DAG-JSON (0x0129) nor DAG-CBOR (0x71) // This is a raw codec (0x55) CID const unknownCodecCid = 'bafkreigrnnl64xuevvkhknbhrcqzbdvvmqnchp7ae2a4ulninsjoc5svoq' const parsedCid = CID.parse(unknownCodecCid) assert.strictEqual(parsedCid.code, 85) - const errorMessage = 'To parse non base32, base36 or base58btc encoded CID multibase decoder must be provided' + const errorMessage = 'Unknown codec 85' try { - await fetchCid('http://example.com', 'testcid') + // @ts-ignore + await fetchCid('http://example.com', unknownCodecCid, { fetchMethod: mockFetch }) assert.fail('fetchCid should have thrown an error') } catch (error) { - // Check the error message - // @ts-ignore assert.ok(error.message.includes(errorMessage), `Error message should include: ${errorMessage}`) } @@ -748,11 +731,8 @@ describe('fetchCid', () => { // Use a real Curio provider and known DAG-CBOR CID const curioProviderUrl = 'https://f03303347-market.duckdns.org/ipni-provider/12D3KooWJ91c6xQshrNe7QAXPFAaeRrHWq2UrgXGPf8UmMZMwyZ5' const dagCborCid = 'baguqeeracgnw2ecmhaa6qkb3irrgjjk5zt5fes7wwwpb4aymoaogzyvvbrma' - // Use the real fetchCid function with the original fetch implementation - globalThis.fetch = originalFetch - /** @type {{ Entries: { [key: string]: string } }} */ - // @ts-ignore - const result = await pRetry( + /** @type {unknown} */ + let result = await pRetry( () => ( fetchCid(curioProviderUrl, dagCborCid) @@ -761,16 +741,29 @@ describe('fetchCid', () => { // Verify the result has the expected structure for DAG-CBOR entries assert(result, 'Expected a non-null result') - assert(result.Entries, 'Result should have Entries property') - const entriesCid = result.Entries['/'] - /** @type {{Entries: Array}} */ - // @ts-ignore - const entriesChunk = await pRetry( + assert(typeof result === 'object' && result !== null, 'Result should be an object') + + /** @type {Record} */ + const resultObj = /** @type {Record} */ (result) + assert('Entries' in resultObj, 'Result should have Entries property') + assert(typeof resultObj.Entries === 'object' && resultObj.Entries !== null, 'Entries should be an object') + + /** @type {Record} */ + const entries = /** @type {Record} */ (resultObj.Entries) + assert('/' in entries, 'Entries should have a "/" property') + + const entriesCid = entries['/'] + assert(typeof entriesCid === 'string', 'Entries CID should be a string') + + /** @type {unknown} */ + result = await pRetry( () => ( fetchCid(curioProviderUrl, entriesCid) ) ) + /** @type {{ Entries: unknown[]; }} */ + const entriesChunk = /** @type {{ Entries: unknown[]; }} */ (result) const payloadCid = processEntries(entriesCid, entriesChunk) console.log(payloadCid) assert.deepStrictEqual(payloadCid, 'bafkreiefrclz7c6w57yl4u7uiq4kvht4z7pits5jpcj3cajbvowik3rvhm')