From 038d4b7a27b9ad679cf97ddd5fd9a18549912551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Thu, 1 Feb 2024 11:17:02 +0100 Subject: [PATCH] feat: resolve the retrieval provider using IPNI (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miroslav Bajtoš --- .github/workflows/ci.yml | 2 +- deps.ts | 9 ++++ lib/deno-encoding-hex.js | 93 ---------------------------------------- lib/ipni-client.js | 75 ++++++++++++++++++++++++++++++++ lib/spark.js | 52 ++++++++++++++-------- test.js | 1 + test/integration.js | 26 ++++++++++- test/ipni-client.test.js | 16 +++++++ vendor/deno-deps.js | 67 +++++++++++++++++++++++++++++ 9 files changed, 229 insertions(+), 112 deletions(-) create mode 100644 deps.ts delete mode 100644 lib/deno-encoding-hex.js create mode 100644 lib/ipni-client.js create mode 100644 test/ipni-client.test.js create mode 100644 vendor/deno-deps.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc34494..96f1094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - run: curl -L https://github.com/filecoin-station/zinnia/releases/download/v0.14.0/zinnia-linux-x64.tar.gz | tar -xz + - run: curl -L https://github.com/filecoin-station/zinnia/releases/download/v0.16.0/zinnia-linux-x64.tar.gz | tar -xz - uses: actions/setup-node@v3 - run: npx standard - run: ./zinnia run test.js diff --git a/deps.ts b/deps.ts new file mode 100644 index 0000000..2586022 --- /dev/null +++ b/deps.ts @@ -0,0 +1,9 @@ +// 3rd-party dependencies from Denoland +// +// Run the following script after making change in this file: +// deno bundle deps.ts vendor/deno-deps.js +// + +export { encodeHex } from 'https://deno.land/std@0.203.0/encoding/hex.ts' +export { decodeBase64 } from 'https://deno.land/std@0.203.0/encoding/base64.ts' +export { decode as decodeVarint } from 'https://deno.land/x/varint@v2.0.0/varint.ts' diff --git a/lib/deno-encoding-hex.js b/lib/deno-encoding-hex.js deleted file mode 100644 index 10aba7a..0000000 --- a/lib/deno-encoding-hex.js +++ /dev/null @@ -1,93 +0,0 @@ -// deno-fmt-ignore-file -// deno-lint-ignore-file -// This code was bundled using `deno bundle` and it's not recommended to edit it manually -// -// You can re-create this file by running the following command: -// deno bundle "https://deno.land/std@0.203.0/encoding/hex.ts" > lib/deno-encoding-hex.js - -const encoder = new TextEncoder() -function getTypeName (value) { - const type = typeof value - if (type !== 'object') { - return type - } else if (value === null) { - return 'null' - } else { - return value?.constructor?.name ?? 'object' - } -} -function validateBinaryLike (source) { - if (typeof source === 'string') { - return encoder.encode(source) - } else if (source instanceof Uint8Array) { - return source - } else if (source instanceof ArrayBuffer) { - return new Uint8Array(source) - } - throw new TypeError(`The input must be a Uint8Array, a string, or an ArrayBuffer. Received a value of the type ${getTypeName(source)}.`) -} -const hexTable = new TextEncoder().encode('0123456789abcdef') -const textEncoder = new TextEncoder() -const textDecoder = new TextDecoder() -function errInvalidByte (__byte) { - return new TypeError(`Invalid byte '${String.fromCharCode(__byte)}'`) -} -function errLength () { - return new RangeError('Odd length hex string') -} -function fromHexChar (__byte) { - if (__byte >= 48 && __byte <= 57) return __byte - 48 - if (__byte >= 97 && __byte <= 102) return __byte - 97 + 10 - if (__byte >= 65 && __byte <= 70) return __byte - 65 + 10 - throw errInvalidByte(__byte) -} -function encode (src) { - const dst = new Uint8Array(src.length * 2) - for (let i = 0; i < dst.length; i++) { - const v = src[i] - dst[i * 2] = hexTable[v >> 4] - dst[i * 2 + 1] = hexTable[v & 0x0f] - } - return dst -} -function encodeHex (src) { - const u8 = validateBinaryLike(src) - const dst = new Uint8Array(u8.length * 2) - for (let i = 0; i < dst.length; i++) { - const v = u8[i] - dst[i * 2] = hexTable[v >> 4] - dst[i * 2 + 1] = hexTable[v & 0x0f] - } - return textDecoder.decode(dst) -} -function decode (src) { - const dst = new Uint8Array(src.length / 2) - for (let i = 0; i < dst.length; i++) { - const a = fromHexChar(src[i * 2]) - const b = fromHexChar(src[i * 2 + 1]) - dst[i] = a << 4 | b - } - if (src.length % 2 === 1) { - fromHexChar(src[dst.length * 2]) - throw errLength() - } - return dst -} -function decodeHex (src) { - const u8 = textEncoder.encode(src) - const dst = new Uint8Array(u8.length / 2) - for (let i = 0; i < dst.length; i++) { - const a = fromHexChar(u8[i * 2]) - const b = fromHexChar(u8[i * 2 + 1]) - dst[i] = a << 4 | b - } - if (u8.length % 2 === 1) { - fromHexChar(u8[dst.length * 2]) - throw errLength() - } - return dst -} -export { encode } -export { encodeHex } -export { decode } -export { decodeHex } diff --git a/lib/ipni-client.js b/lib/ipni-client.js new file mode 100644 index 0000000..59152ae --- /dev/null +++ b/lib/ipni-client.js @@ -0,0 +1,75 @@ +import { decodeBase64, decodeVarint } from '../vendor/deno-deps.js' + +/** + * + * @param {string} cid + * @returns {Promise<{ + * indexerResult: string; + * provider?: { address: string; protocol: string }; + * }>} + */ +export async function queryTheIndex (cid) { + const url = `https://cid.contact/cid/${encodeURIComponent(cid)}` + + let providerResults + try { + const res = await fetch(url) + if (!res.ok) { + console.error('IPNI query failed, HTTP response: %s %s', res.status, await res.text()) + return { indexerResult: `ERROR_${res.status}` } + } + + const result = await res.json() + providerResults = result.MultihashResults.flatMap(r => r.ProviderResults) + console.log('IPNI returned %s provider results', providerResults.length) + } catch (err) { + console.error('IPNI query failed.', err) + return { indexerResult: 'ERROR_FETCH' } + } + + let graphsyncProvider + for (const p of providerResults) { + // TODO: find only the contact advertised by the SP handling this deal + // See https://filecoinproject.slack.com/archives/C048DLT4LAF/p1699958601915269?thread_ts=1699956597.137929&cid=C048DLT4LAF + // bytes of CID of dag-cbor encoded DealProposal + // https://github.com/filecoin-project/boost/blob/main/indexprovider/wrapper.go#L168-L172 + // https://github.com/filecoin-project/boost/blob/main/indexprovider/wrapper.go#L195 + + const [protocolCode] = decodeVarint(decodeBase64(p.Metadata)) + const protocol = { + 0x900: 'bitswap', + 0x910: 'graphsync', + 0x0920: 'http', + 4128768: 'graphsync' + }[protocolCode] + + const address = p.Provider.Addrs[0] + if (!address) continue + + switch (protocol) { + case 'http': + return { + indexerResult: 'OK', + provider: { address, protocol } + } + + case 'graphsync': + if (!graphsyncProvider) { + graphsyncProvider = { + address: `${address}/p2p/${p.Provider.ID}`, + protocol + } + } + } + } + if (graphsyncProvider) { + console.log('HTTP protocol is not advertised, falling back to Graphsync.') + return { + indexerResult: 'HTTP_NOT_ADVERTISED', + provider: graphsyncProvider + } + } + + console.log('All advertisements are for unsupported protocols.') + return { indexerResult: 'NO_VALID_ADVERTISEMENT' } +} diff --git a/lib/spark.js b/lib/spark.js index 0d04013..e187e2d 100644 --- a/lib/spark.js +++ b/lib/spark.js @@ -2,7 +2,10 @@ import { ActivityState } from './activity-state.js' import { SPARK_VERSION, MAX_CAR_SIZE, APPROX_ROUND_LENGTH_IN_MS } from './constants.js' -import { encodeHex } from './deno-encoding-hex.js' +import { queryTheIndex } from './ipni-client.js' +import { + encodeHex +} from '../vendor/deno-deps.js' const sleep = dt => new Promise(resolve => setTimeout(resolve, dt)) @@ -35,8 +38,34 @@ export default class Spark { return retrieval } + async executeRetrievalCheck (retrieval, stats) { + console.log(`Querying IPNI to find retrieval providers for ${retrieval.cid}`) + const { indexerResult, provider } = await queryTheIndex(retrieval.cid) + stats.indexerResult = indexerResult + + if (indexerResult !== 'OK') return + + stats.protocol = provider.protocol + stats.providerAddress = provider.address + + const searchParams = new URLSearchParams({ + // See https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md#dag-scope-request-query-parameter + // Only the root block at the end of the path is returned after blocks required to verify the specified path segments. + 'dag-scope': 'block', + protocols: provider.protocol, + providers: provider.address + }) + const url = `ipfs://${retrieval.cid}?${searchParams.toString()}` + try { + await this.fetchCAR(url, stats) + } catch (err) { + console.error(`Failed to fetch ${url}`) + console.error(err) + } + } + async fetchCAR (url, stats) { - console.log(`Fetching ${url}...`) + console.log(`Fetching: ${url}`) // Abort if no progress was made for 60 seconds const controller = new AbortController() @@ -140,23 +169,12 @@ export default class Spark { carTooLarge: false, byteLength: 0, carChecksum: null, - statusCode: null - } - const searchParams = new URLSearchParams({ - // See https://github.com/filecoin-project/lassie/blob/main/docs/HTTP_SPEC.md#dag-scope-request-query-parameter - // Only the root block at the end of the path is returned after blocks required to verify the specified path segments. - 'dag-scope': 'block', - protocols: retrieval.protocol, - providers: retrieval.providerAddress - }) - const url = `ipfs://${retrieval.cid}?${searchParams.toString()}` - try { - await this.fetchCAR(url, stats) - } catch (err) { - console.error(`Failed to fetch ${url}`) - console.error(err) + statusCode: null, + indexerResult: null } + await this.executeRetrievalCheck(retrieval, stats) + const measurementId = await this.submitMeasurement(retrieval, { ...stats }) Zinnia.jobCompleted() return measurementId diff --git a/test.js b/test.js index 4274d80..afa26af 100644 --- a/test.js +++ b/test.js @@ -1,2 +1,3 @@ +import './test/ipni-client.test.js' import './test/integration.js' import './test/spark.js' diff --git a/test/integration.js b/test/integration.js index 21ed586..3dc555c 100644 --- a/test/integration.js +++ b/test/integration.js @@ -1,6 +1,8 @@ import Spark from '../lib/spark.js' import { test } from 'zinnia:test' -import { assert } from 'zinnia:assert' +import { assert, assertEquals } from 'zinnia:assert' + +const KNOWN_CID = 'bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq' test('integration', async () => { const spark = new Spark() @@ -11,3 +13,25 @@ test('integration', async () => { assert(retrieval.startAt) assert(retrieval.finishedAt) }) + +test('retrieval check for our CID', async () => { + const spark = new Spark() + spark.getRetrieval = async () => ({ cid: KNOWN_CID }) + const measurementId = await spark.nextRetrieval() + const res = await fetch(`https://api.filspark.com/measurements/${measurementId}`) + assert(res.ok) + const m = await res.json() + const assertProp = (prop, expectedValue) => assertEquals(m[prop], expectedValue, prop) + + assertProp('cid', KNOWN_CID) + // TODO - spark-api does not record this field yet + // assertProp('indexerResult', 'OK') + assertProp('providerAddress', '/dns/frisbii.fly.dev/tcp/443/https') + assertProp('protocol', 'http') + assertProp('timeout', false) + assertProp('statusCode', 200) + assertProp('byteLength', 200) + assertProp('carTooLarge', false) + // TODO - spark-api does not record this field yet + // assertProp('carChecksum', '122069f03061f7ad4c14a5691b7e96d3ddd109023a6539a0b4230ea3dc92050e7136') +}) diff --git a/test/ipni-client.test.js b/test/ipni-client.test.js new file mode 100644 index 0000000..f9567cd --- /dev/null +++ b/test/ipni-client.test.js @@ -0,0 +1,16 @@ +import { test } from 'zinnia:test' +import { assertEquals } from 'zinnia:assert' +import { queryTheIndex } from '../lib/ipni-client.js' + +const KNOWN_CID = 'bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq' + +test('query advertised CID', async () => { + const result = await queryTheIndex(KNOWN_CID) + assertEquals(result, { + indexerResult: 'OK', + provider: { + address: '/dns/frisbii.fly.dev/tcp/443/https', + protocol: 'http' + } + }) +}) diff --git a/vendor/deno-deps.js b/vendor/deno-deps.js new file mode 100644 index 0000000..d9b3ff0 --- /dev/null +++ b/vendor/deno-deps.js @@ -0,0 +1,67 @@ +// deno-fmt-ignore-file +// deno-lint-ignore-file +// This code was bundled using `deno bundle` and it's not recommended to edit it manually + +const encoder = new TextEncoder(); +function getTypeName(value) { + const type = typeof value; + if (type !== "object") { + return type; + } else if (value === null) { + return "null"; + } else { + return value?.constructor?.name ?? "object"; + } +} +function validateBinaryLike(source) { + if (typeof source === "string") { + return encoder.encode(source); + } else if (source instanceof Uint8Array) { + return source; + } else if (source instanceof ArrayBuffer) { + return new Uint8Array(source); + } + throw new TypeError(`The input must be a Uint8Array, a string, or an ArrayBuffer. Received a value of the type ${getTypeName(source)}.`); +} +const hexTable = new TextEncoder().encode("0123456789abcdef"); +new TextEncoder(); +const textDecoder = new TextDecoder(); +function encodeHex(src) { + const u8 = validateBinaryLike(src); + const dst = new Uint8Array(u8.length * 2); + for(let i = 0; i < dst.length; i++){ + const v = u8[i]; + dst[i * 2] = hexTable[v >> 4]; + dst[i * 2 + 1] = hexTable[v & 0x0f]; + } + return textDecoder.decode(dst); +} +function decodeBase64(b64) { + const binString = atob(b64); + const size = binString.length; + const bytes = new Uint8Array(size); + for(let i = 0; i < size; i++){ + bytes[i] = binString.charCodeAt(i); + } + return bytes; +} +const MaxUInt64 = 18446744073709551615n; +const REST = 0x7f; +const SHIFT = 7; +function decode(buf, offset = 0) { + for(let i = offset, len = Math.min(buf.length, offset + 10), shift = 0, decoded = 0n; i < len; i += 1, shift += SHIFT){ + let __byte = buf[i]; + decoded += BigInt((__byte & REST) * Math.pow(2, shift)); + if (!(__byte & 0x80) && decoded > MaxUInt64) { + throw new RangeError("overflow varint"); + } + if (!(__byte & 0x80)) return [ + decoded, + i + 1 + ]; + } + throw new RangeError("malformed or overflow varint"); +} +export { encodeHex as encodeHex }; +export { decodeBase64 as decodeBase64 }; +export { decode as decodeVarint };