diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..8db0e95 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +* @bajtos @NikolasHaimerl +*/package.json +package-lock.json \ No newline at end of file diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..9014abd --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,31 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/" + versioning-strategy: increase + schedule: + interval: "daily" + time: "09:00" + timezone: "Europe/Berlin" + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" + groups: + sentry: + patterns: + - "@sentry/*" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + time: "09:00" + timezone: "Europe/Berlin" + commit-message: + prefix: "ci" + - package-ecosystem: "docker" + directories: + - "/" + schedule: + interval: "daily" + time: "15:00" + timezone: "Europe/Berlin" \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..36b75b7 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,30 @@ +name: CI +on: + push: + branches: [main] + pull_request: + +jobs: + build-retrieval: + runs-on: ubuntu-latest + env: + NPM_CONFIG_WORKSPACE: retrieval + GLIF_TOKEN: ${{ secrets.GLIF_TOKEN }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: npm ci + - run: npm test + + lint-all: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: npm ci + - run: npm run lint + - run: npm run test:types \ No newline at end of file diff --git a/.github/workflows/dependabot-auto-approve-minor.yml b/.github/workflows/dependabot-auto-approve-minor.yml new file mode 100644 index 0000000..3dad470 --- /dev/null +++ b/.github/workflows/dependabot-auto-approve-minor.yml @@ -0,0 +1,32 @@ +name: Dependabot auto-approve minor updates +on: pull_request + +permissions: + pull-requests: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + strategy: + matrix: + dependencyStartsWith: + - debug + - ethers + - typescript + - '@types/' + - standard + - prettier + - multiformats + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v2 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Approve a PR + if: ${{startsWith(steps.metadata.outputs.dependency-names, matrix.dependencyStartsWith) && (steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor')}} + run: gh pr review --approve "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} \ No newline at end of file diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml new file mode 100644 index 0000000..845555a --- /dev/null +++ b/.github/workflows/dependabot-auto-merge.yml @@ -0,0 +1,18 @@ +name: Dependabot auto-merge +on: pull_request + +permissions: + contents: write + pull-requests: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - name: Authenticate cli with a PAT + run: echo "${{ secrets.DEPENDABOT_TOKEN }}" | gh auth login --with-token + - name: Enable auto-merge for Dependabot PRs + run: gh pr merge --auto --squash "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a7d6d8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,130 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp +.cache + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* \ No newline at end of file diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..5c7531e --- /dev/null +++ b/.prettierignore @@ -0,0 +1,6 @@ +# We are using https://standardjs.com coding style. It is unfortunately incompatible with Prettier; +# see https://github.com/standard/standard/issues/996 and the linked issues. +# Standard does not handle Markdown files though, so we want to use Prettier formatting. +# The goal of the configuration below is to configure Prettier to lint only Markdown files. +**/*.* +!**/*.md \ No newline at end of file diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 0000000..8071210 --- /dev/null +++ b/.prettierrc.yaml @@ -0,0 +1,2 @@ +printWidth: 80 +proseWrap: always \ No newline at end of file diff --git a/README.md b/README.md index 094e1d9..7c020e3 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# pdp-checker \ No newline at end of file +# pdp-checker diff --git a/manualRetrievalCheck.js b/manualRetrievalCheck.js new file mode 100644 index 0000000..690f058 --- /dev/null +++ b/manualRetrievalCheck.js @@ -0,0 +1,13 @@ +import { SERVICE_ADDRESS, VERIFIER_ADDRESS } from './retrieval/lib/config.js' +import { checkRetrieval } from './retrieval/lib/retrieval.js' +import { createVerifierContract } from './retrieval/lib/verifier.js' + +const baseUrl = 'yablu.net' +const pieceCid = 'baga6ea4seaqkzso6gijktpl22dxarxq25iynurceicxpst35yjrcp72uq3ziwpi' + +(async () => { + const verifier = createVerifierContract(VERIFIER_ADDRESS) + const result = await checkRetrieval(baseUrl, pieceCid) + console.log('CID match:', result) + await verifier.submitRetrievalResults(SERVICE_ADDRESS, [pieceCid], [result]) +})() diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..09a32a8 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,170 @@ +{ + "name": "pdp-checker", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "pdp-checker", + "version": "0.1.0", + "license": "(Apache-2.0 AND MIT)", + "workspaces": [ + "retrieval" + ] + }, + "node_modules/@filecoin-station/pdp-checker-retrieval": { + "resolved": "retrieval", + "link": true + }, + "node_modules/@types/retry": { + "version": "0.12.2", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.2.tgz", + "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", + "license": "MIT" + }, + "node_modules/crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==", + "deprecated": "This package is no longer supported. It's now a built-in Node module. If you've depended on crypto, you should switch to the one that's built-in.", + "license": "ISC" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/is-network-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.1.0.tgz", + "integrity": "sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/p-retry": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", + "integrity": "sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.2", + "is-network-error": "^1.0.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "retrieval": { + "name": "@filecoin-station/pdp-checker-retrieval", + "dependencies": { + "crypto": "^1.0.1", + "node-fetch": "^3.3.2", + "p-retry": "^6.2.1" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..0d451a4 --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "pdp-checker", + "private": true, + "version": "0.1.0", + "type": "module", + "workspaces": [ + "retrieval" + ], + "scripts": { + "lint": "prettier --check . && standard", + "lint:fix": "prettier --write . && standard --fix .", + "test:types": "tsc -p .", + "test:unit": "npm test --workspaces --if-present", + "test": "npm run lint && npm run test:types && npm run test:unit" + }, + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/filecoin-station/deal-observer.git" + }, + "author": "Space Meridian ", + "license": "(Apache-2.0 AND MIT)", + "bugs": { + "url": "https://github.com/filecoin-station/deal-observer/issues" + }, + "homepage": "https://github.com/filecoin-station/deal-observer#readme" +} diff --git a/retrieval/lib/config.js b/retrieval/lib/config.js new file mode 100644 index 0000000..c2b2d95 --- /dev/null +++ b/retrieval/lib/config.js @@ -0,0 +1,3 @@ +// Mock constants +export const SERVICE_ADDRESS = '0x0000000000000000000000000000000000000000' +export const VERIFIER_ADDRESS = '0x0000000000000000000000000000000000000000' diff --git a/retrieval/lib/verifier.js b/retrieval/lib/verifier.js new file mode 100644 index 0000000..8a3bee5 --- /dev/null +++ b/retrieval/lib/verifier.js @@ -0,0 +1,19 @@ +/* +* @param {string} address +*/ +export const createVerifierContract = (address) => { + return { + /* + * @param {string} service + * @param {string[]} commps + * @param {boolean[]} retrievable + */ + async submitRetrievalResults (service, commps, retrievable) { + // Mock implementation: just log the call + console.log('submitRetrievalResults called with:') + console.log('Service:', service) + console.log('CommPs:', commps) + console.log('Retrievable:', retrievable) + } + } +} diff --git a/retrieval/package.json b/retrieval/package.json new file mode 100644 index 0000000..664149d --- /dev/null +++ b/retrieval/package.json @@ -0,0 +1,14 @@ +{ + "name": "@filecoin-station/pdp-checker-retrieval", + "type": "module", + "private": true, + "scripts": { + "lint": "standard", + "test": "node --test --test-reporter=spec --test-concurrency=1" + }, + "dependencies": { + "crypto": "^1.0.1", + "node-fetch": "^3.3.2", + "p-retry": "^6.2.1" + } +} diff --git a/retrieval/test/testData.pdf b/retrieval/test/testData.pdf new file mode 100644 index 0000000..2f920cd Binary files /dev/null and b/retrieval/test/testData.pdf differ diff --git a/retrieval/test/tests.js b/retrieval/test/tests.js new file mode 100644 index 0000000..4fd830b --- /dev/null +++ b/retrieval/test/tests.js @@ -0,0 +1,38 @@ +import test, { describe } from 'node:test' +import assert from 'node:assert/strict' +import { checkRetrieval } from '../lib/retrieval.js' +import fs from 'fs/promises' +import { createHash } from 'crypto' + +const { Response } = await import('node-fetch') + +describe('checkRetrieval integration and unit tests', () => { + test('should download and save actual PDF from live URL', async () => { + const sampleCid = 'baga6ea4seaqkzso6gijktpl22dxarxq25iynurceicxpst35yjrcp72uq3ziwpi' + const baseUrl = 'yablu.net' + const result = await checkRetrieval(baseUrl, sampleCid) + assert.equal(result, true, 'checkRetrieval should return true on success') + }) + + test('should return file with expected hash', async () => { + const sampleCid = 'baga6ea4seaqkzso6gijktpl22dxarxq25iynurceicxpst35yjrcp72uq3ziwpi' + const baseUrl = 'yablu.net' + const downloadedData = await retrieval(baseUrl, sampleCid) + // Check the hash of the testData against the fetched data + const testData = await fs.readFile('./testData.pdf') + const expectedHash = createHash('sha256').update(testData).digest('hex') + const actualHash = createHash('sha256') + .update(downloadedData) + .digest('hex') + assert.equal( + actualHash, + expectedHash, + 'Downloaded data hash should match the expected hash' + ) + }) + + test('should return false on fetch failure', async (t) => { + const result = await checkRetrieval('invalid.url', 'fakecid') + assert.equal(result, false, 'checkRetrieval should fail and return false') + }) +}) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..307408c --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "skipLibCheck": true, + "lib": ["es2022"], + "target": "es2022", + "module": "Node16", + "moduleResolution": "node16", + + "moduleDetection": "force", + "isolatedModules": true, + "verbatimModuleSyntax": true, + + "strict": true, + "forceConsistentCasingInFileNames": true, + + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "declarationMap": true, + "resolveJsonModule": true + }, + "include": [ + + ], + "exclude": [ + "dist/**/*" + ] +} \ No newline at end of file