From ebc270b336fc462fa8e7b1b24033e34512fccfad Mon Sep 17 00:00:00 2001 From: JGiter Date: Tue, 10 Jan 2023 15:13:53 +0200 Subject: [PATCH 01/24] feat: add ipfs module --- .env.dev | 16 ++-- .gitignore | 3 +- README.md | 10 +- devops/dev/values.yaml | 2 +- e2e/app.e2e.spec.ts | 18 ++-- e2e/ipfs/ipfs.testSuite.ts | 112 +++++++++++++++++++++++ e2e/setup-ipfs.ts | 23 ----- e2e/setupIpfsCluster/docker-compose.yml | 116 ++++++++++++++++++++++++ e2e/setupIpfsCluster/index.ts | 53 +++++++++++ e2e/setupIpfsCluster/nginx.conf | 17 ++++ src/common/cid.pipe.ts | 9 ++ src/env-vars-validation-schema.ts | 15 +-- src/modules/auth/login.strategy.ts | 12 +++ src/modules/did/did.module.ts | 13 +-- src/modules/did/did.processor.ts | 1 - src/modules/did/did.service.spec.ts | 4 +- src/modules/did/did.service.ts | 12 +-- src/modules/ipfs/ipfs.controller.ts | 46 ++++++++++ src/modules/ipfs/ipfs.module.ts | 114 ++++++++++++++++++----- src/modules/ipfs/ipfs.service.ts | 83 ++++++++++++++++- src/modules/ipfs/ipfs.types.ts | 26 ++++-- src/modules/ipfs/pins.processor.ts | 36 ++++++++ 22 files changed, 635 insertions(+), 106 deletions(-) create mode 100644 e2e/ipfs/ipfs.testSuite.ts delete mode 100644 e2e/setup-ipfs.ts create mode 100644 e2e/setupIpfsCluster/docker-compose.yml create mode 100644 e2e/setupIpfsCluster/index.ts create mode 100644 e2e/setupIpfsCluster/nginx.conf create mode 100644 src/common/cid.pipe.ts create mode 100644 src/modules/ipfs/ipfs.controller.ts create mode 100644 src/modules/ipfs/pins.processor.ts diff --git a/.env.dev b/.env.dev index d8148885..afc77a0a 100644 --- a/.env.dev +++ b/.env.dev @@ -41,13 +41,15 @@ ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS=21 ASSETS_SYNC_ENABLED=true # IPFS -IPFS_CLUSTER_ROOT=$IPFS_CLUSTER_ROOT -IPFS_CLUSTER_USER=$IPFS_CLUSTER_USER -IPFS_CLUSTER_PASSWORD=$IPFS_CLUSTER_PASSWORD -IPFS_CLIENT_HOST= -IPFS_CLIENT_PORT= -IPFS_CLIENT_PROJECT_ID= -IPFS_CLIENT_PROJECT_SECRET= +IPFS_CLUSTER_ROOT_URL # Cluster can be deployed locally with e2e/setupIpfsCluster/index.ts +IPFS_CLUSTER_USER # Not needed locally +IPFS_CLUSTER_PASSWORD # Not needed locally +IPFS_CLIENT_URL # Can be public gateway https://ipfs.github.io/public-gateway-checker/, Inura gateway or IPFS cluster. When cluster used as IPFS gateway it supports only GET +IPFS_CLIENT_PROTO # These can be +IPFS_CLIENT_HOST # used instead of +IPFS_CLIENT_PORT # IPFS_CLIENT_URL +IPFS_CLIENT_USER # User of cluster or Id of Infura project +IPFS_CLIENT_PASSWORD # Password to cluster or secret of Infura project # INTERVALS DIDDOC_SYNC_INTERVAL_IN_HOURS=1 diff --git a/.gitignore b/.gitignore index 337bee7a..d80db6ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ /dist /node_modules /src/ethers -.env # Logs logs @@ -20,6 +19,7 @@ ormlogs.log # Tests /coverage /.nyc_output +/e2e/setupIpfsCluster/ipfs0/ # IDEs and editors /.idea @@ -41,6 +41,7 @@ ormlogs.log # Using either dev or prod docker-compose file docker-compose.yml +!/e2e/setupIpfsCluster/docker-compose.yml db_dumps private.pem public.pem diff --git a/README.md b/README.md index a9f62051..cf03e6b0 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,7 @@ $ cp .env.dev .env $ cp docker-compose.dev.yml docker-compose.yml ``` -Set the following values in your `.env`: - -```text -ENS_URL= -IPFS_CLIENT_HOST= -IPFS_CLIENT_PORT= -IPFS_CLIENT_PROJECT_ID= -IPFS_CLIENT_PROJECT_SECRET= -``` +Fill in configuration values in your `.env`. For reference look at `.env.dev` ### Production ```bash diff --git a/devops/dev/values.yaml b/devops/dev/values.yaml index f5dfb231..a844ccb0 100644 --- a/devops/dev/values.yaml +++ b/devops/dev/values.yaml @@ -144,6 +144,6 @@ iam-cache-server-helm: STRATEGY_PRIVATE_KEY: AgBM4BIbZIvZ9ErUDLWmmgw0UMEGk4k9hO54/tos4Rf9bUeewlDyAOrnz+gcl8uXuwFiGazQYNia2JFOjCapKnVt0h3LUjvubN4G2cVoQQl9E2+BDzSb7Ucji2UsdleN1ScOwd6XOADdw7Pa1LUXdpf32rf5YADhdYLGDiYy6hqpJ0OPd+61s92HgtyKHioTSP3fJ4YyFqgXQ4wORRR15rdwJdvzSxUrzGnft/ceuB+ilGJc0qSu/jT+SifQ1411zZ6/vsvhfvSq7q4wXEibn2OWSy7MxxNdYafVC7PbyFgc9i6t8V2YKJzUigIGV53ds9GuMK8i8CeGVg2R3e4iHyuOuWv6LeNjbdUd6QeAynOFVos9rBWwS/L7g+Rv0aUUnD016GYSoPwLGPRwvpyhk3jRtJ94uj4FAkUwWGWKagTA+F0uZyIjq2emO4opE8J5FwbPJOA3SKxfo0yPxgQF7S7uTAGlslQdxSrBk9ICUb5FuMVZ32Bxv3fbJ4R3xIS6D8g9hfPG3/P+zpvTxMzPlOjn7bgXHpbV6qOtt47OjTb9A+ELEVnwF/hOY6efSNa/MYlDU0wjF8CGveKFmYkSo9QHnTx7ruGu1s18yXMQZGK47WEERFR3WS0JqB7iQKv2KhPlMJLQapbROLP3MEYGWpEExDdfQRksmvCpQvQhM/7rtBpjobktxKfP6FVsiCTiL0A8Lkbo+i48oVti3SK6vNF2kDBhy4b6uLWwztYNvz3XL1ZalmcWSwe6iMtscD8Vcjms4FVe3hBRPZbXK5/wOpMq ENS_URL: AgCaICh+WJ7pQ2MsRfznNhqFAqauOls7BMQeKb79yyrrFE9PbGRfVaFjL/R1HPzFbdfPzD2OT1kvcJDN/R6Jc4CH9+Vg1z6QTaknXhHsZtBiN+BRCmRwfyno0QFjsWTLVZ83NTSKP2EvwqU5h4/peKavCx7pSkDb31xqf0Mq9V9Kapqw7YvKjEjfvRzdbtAxBtLt3v2UoyOyCldnfXcnenDyHmSgBn6uKS6U8pK/5RlGlsegvCokzR20vw4l8bf60uR/36fX+5ttoYzF+BWI3FWgrry2S/EChv4umk0M8Nr8vYeNc40cLuVJPlxFTr/eHAb+uqKe/bhavJqAvzJFW714/QTNDDKyWqmZMVj6H6BSLSVHHmh6M0Cp2YOH07xecHCgPWqTWzJq1HCfMAuB9hH5wQYVxLhYoIze6XzD8AAzJEFUJ+n+TXmaxH7kuqhVJGbgTtXxbk+QvKeZmX36IcBOl6BxkUlm9VGvj91zCip9zC/HiBbl8zL6nTd6JpFYgJylaLBeOLvMjoyk25vfs+viehC9HbEtsgcLtNevSAzA1sCBFF1c2x4vkyi4yFI2OIp1+FY6xPBkaR5rqMlwiPXS8Al9lgTATHoZgr+tkdcjFwg2wEokmyYPTzOcLc82NU0bZeBclUCVEkxeWCHcNwBRW2heNyujRLWl35klbQPbFlQjQNNOnj0wpp1K1+KFetqjKm7s4Xqjk7UAiCjAX59VIkPpvvTc7iOFHEWqvfdLHciJDxYNuN7nPZk+bVtj/7KNSdCf2Q== IPFS_CLIENT_PROJECT_SECRET: AgCD3q5miwVO6GjALhdNMsbLQul35EQybREj9bV3IQJtC3ij7b65vpySapMAG1NBx6UMm8cqBRKySPxs85ICAevdtPXp9QtDLndTb/HiuEKC945Pw4R7g2V3RJaKgnKO/WZb2HRnaFq3ol63A4eIC2gcxAcU/M2Cinf3mQ8+Ha4D3ZwRTZBp+Vl7aD0vmDF8DrAaczBYs5qX0e4QYwCMlTviHGw/XNNKoQidl+jL+QGWE0aklOMrS9EBgcGtggYaBaNl5eqPAvhNllfsLIKAe4yt5QTQ2hCCih12/8HifOkAQAQw4l6xtzl3idfCSstwsMwMuAeBS/WUd40Y6tD1gR40AxSYiRC/FfMgDhW/ftqZD5wQq67YM7fD2BjJaM2UPDmzcyCdDqRGtHQNeCc9wf6iksGUv/4e68HRlrDQEw+DYqudeKcDs8jbQhJiecCDLGUr3Bb0Ww7Wi+tvs/rqtGrJzLyEJ5kM3XlEeuvA5qaGoJpEOa/0AnHLysfL3KnK+uA6t9KQTNUlRasmATIiHO0X0xSVJj0w30qgN6My83SbnIcbyno2rzlLktBxc+ZeB7pLb2ZvdFYYNgh05F0BXUMmqNXMAYerLpfZvAG4Y7JYcaPVkXnR16rG0vl0Qv+6/mVXbHV/Udfa8wjUrkbzl17mPGKZjmYdF8GoeGbjHsFM4JWceHsxbrX8d8m9NqPqYDRD420a6cqCFZUkpoq7RTa1hZUJ7HX5hlGyg75DuomDNQ== - IPFS_CLUSTER_ROOT: AgCK8khzwUAWcFtB9tO58fCOxUuE9AQqKFP1tO/0zHWLesuiz2N5cXwiySCeokJ3uo2lFfNdclBitV9Mew+mtbKWEJi1s9bXNN83P+5ctJLwXp75MuqROZJxrnigVrb/3BLwgjo8C1KYnH4BHjYvnsN32TmBneIbtdRqsivvs+KT5Xbe6SFj6RlUH7Du9FIfUq5dmf4Bc4hH6Bopwl0KxBZspSECb66ydQnJVJnImKBNpJwySVdwJA3cqYzH/7KE5NR1nByHpVh6Lv0OO6aikq/XGYrBz+pny1NFcLBNKt8qP280SYa8jSRxc+Gd+632LuDsYVnOJLVgzbJllwnTtGUki9EwNAOsIfU5TblJfDJbZ9nLGsispYPrh/DoZHvt4Y0DxljEZomUR+uj1h0x8WCtEX9eRCaVjXW86gcYQ8ssLjKnXTY1n5axQY0IZtc1QyxKp9LEVUGe/UJRTzzR7J+SJTCeeHI9+5ED8wJrAC2SfWuUHYPczB/z4BHRYRv4enO+AWpr3NaMFVD3uQsGRRq4vgw+z8h9O46T1T3uzrIKyL8WdUuzu9nEsC9MZmtlTYcUqAeZZq/rJ9bsgIrRmo1D/3N6aHWU9MNnv+ZPlJeYzRAIqmQEcrWm6pxhzWGzRUOwpD9FYl9xEgmDn5szk4+ihwa776qUXiRkp/6630Y0hkBsext0Jefpe2iXuLroIhSCCYKswWBmMtEpVYHuhet8tbmjmiNpkXo6ysjVSTZ1BvxuTXXS4+dLzgTR6g== + IPFS_CLUSTER_ROOT_URL: AgCK8khzwUAWcFtB9tO58fCOxUuE9AQqKFP1tO/0zHWLesuiz2N5cXwiySCeokJ3uo2lFfNdclBitV9Mew+mtbKWEJi1s9bXNN83P+5ctJLwXp75MuqROZJxrnigVrb/3BLwgjo8C1KYnH4BHjYvnsN32TmBneIbtdRqsivvs+KT5Xbe6SFj6RlUH7Du9FIfUq5dmf4Bc4hH6Bopwl0KxBZspSECb66ydQnJVJnImKBNpJwySVdwJA3cqYzH/7KE5NR1nByHpVh6Lv0OO6aikq/XGYrBz+pny1NFcLBNKt8qP280SYa8jSRxc+Gd+632LuDsYVnOJLVgzbJllwnTtGUki9EwNAOsIfU5TblJfDJbZ9nLGsispYPrh/DoZHvt4Y0DxljEZomUR+uj1h0x8WCtEX9eRCaVjXW86gcYQ8ssLjKnXTY1n5axQY0IZtc1QyxKp9LEVUGe/UJRTzzR7J+SJTCeeHI9+5ED8wJrAC2SfWuUHYPczB/z4BHRYRv4enO+AWpr3NaMFVD3uQsGRRq4vgw+z8h9O46T1T3uzrIKyL8WdUuzu9nEsC9MZmtlTYcUqAeZZq/rJ9bsgIrRmo1D/3N6aHWU9MNnv+ZPlJeYzRAIqmQEcrWm6pxhzWGzRUOwpD9FYl9xEgmDn5szk4+ihwa776qUXiRkp/6630Y0hkBsext0Jefpe2iXuLroIhSCCYKswWBmMtEpVYHuhet8tbmjmiNpkXo6ysjVSTZ1BvxuTXXS4+dLzgTR6g== IPFS_CLUSTER_USER: AgBwOgRhIkI20NFyEWwNyEhHELvOVLajzvu+Q3LhfjIWbyVSfP18Hx33RpPc6KR7gx5gJI0UswESjzqxIIgv2FjSl9LHx8ZXa/go0iVq6y/hJBk6ersWD3a9nqiafMqo691C19CRHIAZqpUK8NIzgcGjRo9/gXu95MuOac3I1Qs1ILkTRsfwGAbQMIUup2CO0bbpsc582i3phj/OguTrI8i+vfoNOSoJRk6CNNHH9qzIb5U6A0fQkic94IkcGIi05rPmUoU3WFB3ebgs7iD1ZRNV5syMNNjvMC7YV9stYok5TbH1Fqnq+OVlu0bfDKnQ2o8SPx4+lDhKVrEnFKH0AeTNOXqM1ZARMaOeWSell4MQAcnSggAXn3SexSy0D1xNy6MrH7uydqO8r8y52BXRVMc0lWkuxcTFmzjPuxG9IEHzGUQ55Rh39FI5/rizEXWV5VYHTcnl+IAa7LqqzfN4HC0FzxYgQZ4ZBsWX0VhL5ZqJyh1Yo856u1vxnPCCaHGlFyAHKueFxh8iMv0F6LxiSXgP5sJyQ8py7piisDbEJUTXsLWwqmhs/5T1eaOiH3cPUcCJZsY8Z362vwguzz9TAXzDRDhSddAEhbZ5NA7aBPxSo3YKmLLXpDvObW82TelZhbRW5epK0xwPNWVmjnOdcm4V6fj76+aiUPP8Iu74ImFq4EEtEpwB30BI9y2iJI2jdxL21n17bZYke3WatAcCUg== IPFS_CLUSTER_PASSWORD: AgAhHW1lN0qi1JitHn10qC4fF8vdh2J2mBbA9MVs5vMHHSdOhOX/s+JAtFAMVplIqaxBUjM4VsWFDqs4LZLN3I06pTDe4srDenOCvdlT2Pcupo6a/ftlVQ/MidOA68Qz5i0HFBlirOG9ZtdfdTEq41CqgvaP73oXbG7pkgkQqiydVglvKGmWOr8QWwqVLYqI1Jlrmz9OKcyC6AKaix7+FToVw2IJqH4dp73ciuQ5qwlOIOHIcsuGXgnvHaGYcbcOt1FYjzF/NTaKq3q69WvffM6UMCQq1m+RHBVpTAMQInGe2nfLnijAUShWJoozgLbGgU9knStwEQAVDPr2Shp+bjFhAVip3F7ZUQ4GmCkKQPVNccMsgHUib/157kvy2ET6OLVDppG6v5QIrXP5JM/8AIYz8zpwvuqkeumVqR94VRsI8Ryy+vSVjxYsEDuyG892cQ4wMf+dl7SOJINJIFxL0vd5f761lu4aUz0G5aQsbzzSafPlGgEQitm35kAoCbRyVX/2pquXiRsiigdxa7Nl+OJBm/EoNbbISi88eqWlbEOVq5EA1Tu0HC6FU0AW3Xg7G8vGrbHKrXxX5oxTRzNB6crBniAgRAdrPOc/G1ORZhkZQD6wEPUJ9Ny6GPikR9MlZ4v5sMFboapZqGD9IIwxjP+FJlrThMBfXcsqHrqGJWYJx8j5WrJpLUmunI4FBE1NcyHCSQ7rHCBfM46bd/URq19q6XCh0w== diff --git a/e2e/app.e2e.spec.ts b/e2e/app.e2e.spec.ts index 1869fcd1..09d69e4d 100644 --- a/e2e/app.e2e.spec.ts +++ b/e2e/app.e2e.spec.ts @@ -8,11 +8,13 @@ import { appConfig } from '../src/common/test.utils'; import { authTestSuite } from './auth'; import { claimTestSuite } from './claim'; import { statusList2021TestSuite } from './status-list'; -import { shutDownIpfsDaemon, spawnIpfsDaemon } from './setup-ipfs'; +import { shutdownIpfsCluster, spawnIpfsCluster } from './setupIpfsCluster'; import { EthereumDIDRegistry } from '../src/ethers/EthereumDIDRegistry'; import { EthereumDIDRegistry__factory } from '../src/ethers/factories/EthereumDIDRegistry__factory'; import { didModuleTestSuite } from './did/did-service'; import { Provider } from '../src/common/provider'; +import { ipfsModuleTestSuite } from './ipfs/ipfs.testSuite'; +import { ChildProcess } from 'child_process'; export let app: INestApplication; @@ -23,6 +25,7 @@ describe('iam-cache-server E2E tests', () => { let didRegistry: EthereumDIDRegistry; network.config.chainId = 73799; let consoleLogSpy: jest.SpyInstance; + let cluster: ChildProcess; async function deployDidRegistry() { const didRegistry = await new EthereumDIDRegistry__factory() @@ -36,17 +39,17 @@ describe('iam-cache-server E2E tests', () => { didRegistry = await loadFixture(deployDidRegistry); process.env.DID_REGISTRY_ADDRESS = didRegistry.address; - process.env.IPFS_CLUSTER_ROOT = 'http://localhost:8080'; - process.env.IPFS_CLUSTER_USER = 'not-required-locally'; - process.env.IPFS_CLUSTER_PASSWORD = 'not-required-locally'; + + cluster = await spawnIpfsCluster(); + process.env.IPFS_CLUSTER_ROOT_URL = 'http://localhost:8080'; + + process.env.IPFS_CLIENT_URL = 'http://mocked'; // CID resolved incorrectly through gateway exposed on cluster. TODO: instead of gateway try to expose IPFS API // have to import dynamically to have opportunity to deploy DID registry before environment configuration validation const { AppModule } = await import('../src/app.module'); const testingModule = await Test.createTestingModule({ imports: [AppModule], }) - .overrideProvider('IPFSClientConfig') - .useValue(await spawnIpfsDaemon()) .overrideProvider(Provider) .useValue(provider) .compile(); @@ -60,7 +63,7 @@ describe('iam-cache-server E2E tests', () => { expect.stringMatching(/^error \[.+\] : .+/) ); await app.close(); - await shutDownIpfsDaemon(); + shutdownIpfsCluster(cluster); }, 60_000); // 1min describe('Modules v1', () => { @@ -68,5 +71,6 @@ describe('iam-cache-server E2E tests', () => { describe('Claim module', claimTestSuite); describe('StatusList2021 module', statusList2021TestSuite); describe('Did module', didModuleTestSuite); + describe('Ipfs module', ipfsModuleTestSuite); }); }); diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts new file mode 100644 index 00000000..88e41bc6 --- /dev/null +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -0,0 +1,112 @@ +import { HttpStatus } from '@nestjs/common'; +import { getQueueToken } from '@nestjs/bull'; +import { Queue } from 'bull'; +import request from 'supertest'; +import { Connection, EntityManager, QueryRunner } from 'typeorm'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { app } from '../app.e2e.spec'; +import { randomUser } from '../utils'; + +export const ipfsModuleTestSuite = () => { + let queryRunner: QueryRunner; + let didStoreCluster: DidStoreCluster; + let didStoreInfura: DidStoreGateway; + let pinsQueue: Queue; + const notPinned = { claimType: 'hello world notpinned' }; + const notPinnedCid = + 'bafkreigj4mi6cnegeh6hh6rxdjb63l4d7dowjcxxeyakgnijvoues3svii'; + const notPersistedCid = + 'bafkreih5pe7r3ucfdebiu7wjx3jr35qpbxqkxm5eneb2s2zu6t7yfqllci'; // CID of { claimType: 'hello world not persisted bafybeicg2rebjoofv4kbyovkw7af3rpiitvnl6i7ckcywaq6xjcxnc2mby' } + + beforeEach(async () => { + jest.restoreAllMocks(); + + didStoreCluster = app.get(DidStoreCluster); + didStoreInfura = app.get(DidStoreGateway); + pinsQueue = app.get(getQueueToken('pins')); + + const manager = app.get(EntityManager); + const dbConnection = app.get(Connection); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + queryRunner = manager.queryRunner = + dbConnection.createQueryRunner('master'); + await queryRunner.startTransaction(); + }); + + afterEach(async () => { + await queryRunner.rollbackTransaction(); + await queryRunner.release(); + }); + + it('should be able to post claim', async () => { + const claimData = { + claimType: 'claim type', + claimTypeVersion: 1, + fields: [], + issuerFields: [], + }; + const requester = await randomUser(); + + const { text: cid } = await request(app.getHttpServer()) + .post(`/v1/ipfs/`) + .set('Cookie', requester.cookies) + .send(claimData) + .expect(HttpStatus.CREATED); + + const get = jest.spyOn(didStoreCluster, 'get'); + const { text: stored } = await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies) + .expect(HttpStatus.OK); + expect(get).toBeCalledTimes(1); + + expect(JSON.parse(stored)).toStrictEqual(claimData); + }); + + it('should return 404 if claim was not persisted in IPFS', async () => { + const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); + const requester = await randomUser(); + + const cid = notPersistedCid; + didStoreInfuraGet.mockRejectedValueOnce({ response: { status: 504 } }); + await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies) + .expect(HttpStatus.NOT_FOUND); + }); + + it('claim persisted in IPFS should be pinned in cluster', async () => { + const didStoreClusterGet = jest.spyOn(didStoreCluster, 'get'); + const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); + + const requester = await randomUser(); + + const claim = JSON.stringify(notPinned); + const cid = notPinnedCid; + + const claimPinned = new Promise((resolve) => { + pinsQueue.on('completed', () => { + resolve(); + }); + }); + didStoreInfuraGet.mockResolvedValueOnce(claim); + await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies); + + await claimPinned; + + expect(didStoreClusterGet).toBeCalledTimes(0); + expect(didStoreInfuraGet).toBeCalledTimes(1); + + await request(app.getHttpServer()) + .get(`/v1/ipfs/${cid}`) + .set('Cookie', requester.cookies); + + expect(didStoreClusterGet).toBeCalledTimes(1); + expect(didStoreInfuraGet).toBeCalledTimes(1); + }); +}; diff --git a/e2e/setup-ipfs.ts b/e2e/setup-ipfs.ts deleted file mode 100644 index a7f807f5..00000000 --- a/e2e/setup-ipfs.ts +++ /dev/null @@ -1,23 +0,0 @@ -import Ctl from 'ipfsd-ctl'; -import path from 'path'; -import ipfsHttpModule from 'ipfs-http-client'; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -let ipfsd: any; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export async function spawnIpfsDaemon(): Promise { - const ipfsBin = path.resolve(__dirname, '../', 'node_modules/.bin', 'jsipfs'); - ipfsd = await Ctl.createController({ - type: 'js', - disposable: true, - test: true, - ipfsBin, - ipfsHttpModule, - }); - return ipfsd.apiAddr; -} - -export async function shutDownIpfsDaemon(): Promise { - return ipfsd && ipfsd.stop(); -} diff --git a/e2e/setupIpfsCluster/docker-compose.yml b/e2e/setupIpfsCluster/docker-compose.yml new file mode 100644 index 00000000..53e06dec --- /dev/null +++ b/e2e/setupIpfsCluster/docker-compose.yml @@ -0,0 +1,116 @@ +# This configuration is based on https://github.com/ipfs-cluster/ipfs-cluster/blob/master/docker-compose.yml + +version: '3.4' + +services: + cluster_proxy: + container_name: cluster_proxy + image: nginx:alpine + ports: + - 8080:8080 + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + depends_on: + - cluster0 + - ipfs0 + + ################################################################################## + ## Cluster PEER 0 ################################################################ + ################################################################################## + + ipfs0: + container_name: ipfs0 + image: ipfs/go-ipfs:latest + expose: + - '8080' + # ports: + # - "4001:4001" # ipfs swarm - expose if needed/wanted + # - "5001:5001" # ipfs api - expose if needed/wanted + # - "8080:8080" # ipfs gateway - expose if needed/wanted + volumes: + - ipfs0:/data/ipfs + + cluster0: + container_name: cluster0 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs0 + environment: + CLUSTER_PEERNAME: cluster0 + CLUSTER_SECRET: ${CLUSTER_SECRET} # From shell variable if set + CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs0/tcp/5001 + CLUSTER_CRDT_TRUSTEDPEERS: '*' # Trust all peers in Cluster + CLUSTER_RESTAPI_HTTPLISTENMULTIADDRESS: /ip4/0.0.0.0/tcp/9094 # Expose API + CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery + expose: + - '9094' + # ports: + # Open API port (allows ipfs-cluster-ctl usage on host) + # - "127.0.0.1:9094:9094" + # The cluster swarm port would need to be exposed if this container + # was to connect to cluster peers on other hosts. + # But this is just a testing cluster. + # - "9095:9095" # Cluster IPFS Proxy endpoint + # - "9096:9096" # Cluster swarm endpoint + volumes: + - cluster0:/data/ipfs-cluster + + ################################################################################## + ## Cluster PEER 1 ################################################################ + ################################################################################## + + # See Cluster PEER 0 for comments (all removed here and below) + ipfs1: + container_name: ipfs1 + image: ipfs/go-ipfs:latest + volumes: + - ipfs1:/data/ipfs + + cluster1: + container_name: cluster1 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs1 + environment: + CLUSTER_PEERNAME: cluster1 + CLUSTER_SECRET: ${CLUSTER_SECRET} + CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs1/tcp/5001 + CLUSTER_CRDT_TRUSTEDPEERS: '*' + CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery + volumes: + - cluster1:/data/ipfs-cluster + + ################################################################################## + ## Cluster PEER 2 ################################################################ + ################################################################################## + + # See Cluster PEER 0 for comments (all removed here and below) + ipfs2: + container_name: ipfs2 + image: ipfs/go-ipfs:latest + volumes: + - ipfs2:/data/ipfs + + cluster2: + container_name: cluster2 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs2 + environment: + CLUSTER_PEERNAME: cluster2 + CLUSTER_SECRET: ${CLUSTER_SECRET} + CLUSTER_IPFSHTTP_NODEMULTIADDRESS: /dns4/ipfs2/tcp/5001 + CLUSTER_CRDT_TRUSTEDPEERS: '*' + CLUSTER_MONITORPINGINTERVAL: 2s # Speed up peer discovery + volumes: + - cluster2:/data/ipfs-cluster + # For adding more peers, copy PEER 1 and rename things to ipfs2, cluster2. + # Keep bootstrapping to cluster0. + +volumes: + ipfs0: + ipfs1: + ipfs2: + cluster0: + cluster1: + cluster2: diff --git a/e2e/setupIpfsCluster/index.ts b/e2e/setupIpfsCluster/index.ts new file mode 100644 index 00000000..ab23ba9b --- /dev/null +++ b/e2e/setupIpfsCluster/index.ts @@ -0,0 +1,53 @@ +import { execFile, execFileSync, ChildProcess } from 'child_process'; +import path from 'path'; +import waitOn from 'wait-on'; + +/** + * Spawn local ipfs cluster + * + * @returns uri of the cluster root + */ +export async function spawnIpfsCluster() { + shutdownNodes(); + + const cluster = execFile( + 'docker', + ['compose', 'up', '--force-recreate', 'cluster_proxy'], + { cwd: path.join(__dirname) }, + (error) => { + if (error) { + throw error; + } + } + ); + + try { + await waitOn({ + // https://ipfscluster.io/documentation/reference/api/ + resources: ['http-get://localhost:8080/id'], + delay: 5000, + // if your internet connection is slow try to increase timeout to be able to download all images + timeout: 30000, + }); + + return cluster; + } catch (error) { + shutdownIpfsCluster(cluster); + throw error; + } +} + +export function shutdownIpfsCluster(cluster?: ChildProcess) { + shutdownNodes(); + + if (cluster !== undefined) { + cluster.kill(); + } +} + +function shutdownNodes() { + execFileSync('docker', ['compose', 'down', '--volumes'], { + cwd: path.join(__dirname), + stdio: 'pipe', + }); +} diff --git a/e2e/setupIpfsCluster/nginx.conf b/e2e/setupIpfsCluster/nginx.conf new file mode 100644 index 00000000..e40c0073 --- /dev/null +++ b/e2e/setupIpfsCluster/nginx.conf @@ -0,0 +1,17 @@ +events { } + +http { + client_max_body_size 100M; + + server { + listen 8080; + + location / { + proxy_pass http://cluster0:9094; + } + + location /ipfs { + proxy_pass http://ipfs0:8080; + } + } +} \ No newline at end of file diff --git a/src/common/cid.pipe.ts b/src/common/cid.pipe.ts new file mode 100644 index 00000000..7cd8ec61 --- /dev/null +++ b/src/common/cid.pipe.ts @@ -0,0 +1,9 @@ +import { PipeTransform, Injectable } from '@nestjs/common'; +import { CID } from 'multiformats/cid'; + +@Injectable() +export class CIDPipe implements PipeTransform { + transform(cid: string) { + return CID.parse(cid); + } +} diff --git a/src/env-vars-validation-schema.ts b/src/env-vars-validation-schema.ts index a83655cc..574a4051 100644 --- a/src/env-vars-validation-schema.ts +++ b/src/env-vars-validation-schema.ts @@ -89,14 +89,15 @@ export const envVarsValidationSchema = Joi.object({ ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS: Joi.number().positive().required(), ASSETS_SYNC_ENABLED: Joi.boolean().required(), - IPFS_CLIENT_HOST: Joi.string().hostname(), + IPFS_CLIENT_URL: Joi.string().uri(), + IPFS_CLIENT_PROTO: Joi.string(), + IPFS_CLIENT_HOST: Joi.string().uri(), IPFS_CLIENT_PORT: Joi.number().port(), - IPFS_CLIENT_PROJECT_ID: Joi.string(), - IPFS_CLIENT_PROJECT_SECRET: Joi.string(), - IPFS_CLUSTER_ROOT: Joi.string(), + IPFS_CLIENT_USER: Joi.string(), + IPFS_CLIENT_PASSWORD: Joi.string(), + IPFS_CLUSTER_ROOT_URL: Joi.string().required(), IPFS_CLUSTER_USER: Joi.string(), IPFS_CLUSTER_PASSWORD: Joi.string(), - DID_SYNC_MODE_FULL: Joi.boolean().required(), DID_SYNC_ENABLED: Joi.boolean().required(), DIDDOC_SYNC_INTERVAL_IN_HOURS: Joi.number().positive().required(), @@ -127,4 +128,6 @@ export const envVarsValidationSchema = Joi.object({ STATUS_LIST_DOMAIN: Joi.string().uri().required(), DISABLE_GET_DIDS_BY_ROLE: Joi.bool().default(false), -}); +}) + .or('IPFS_CLIENT_URL', 'IPFS_CLIENT_HOST') + .with('IPFS_CLIENT_HOST', ['IPFS_CLIENT_PROTO', 'IPFS_CLIENT_PORT']); diff --git a/src/modules/auth/login.strategy.ts b/src/modules/auth/login.strategy.ts index e2b0aab2..8485e100 100644 --- a/src/modules/auth/login.strategy.ts +++ b/src/modules/auth/login.strategy.ts @@ -7,6 +7,7 @@ import { URL } from 'url'; import { RoleIssuerResolver } from '../claim/resolvers/issuer.resolver'; import { RoleRevokerResolver } from '../claim/resolvers/revoker.resolver'; import { RoleCredentialResolver } from '../claim/resolvers/credential.resolver'; +import { IpfsGatewayConfig, IPFSGatewayConfigToken } from '../ipfs/ipfs.types'; import { LoginStrategyOptions } from 'passport-did-auth/dist/lib/LoginStrategy'; @Injectable() @@ -23,6 +24,9 @@ export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { new URL(configService.get('STRATEGY_CACHE_SERVER')).origin ).href; const loginStrategyOptions: LoginStrategyOptions = { + @Inject(IPFSGatewayConfigToken) ipfsConfig: IpfsGatewayConfig + ) { + let loginStrategyParams: Omit = { name: 'login', rpcUrl: configService.get('ENS_URL'), cacheServerUrl: configService.get('STRATEGY_CACHE_SERVER'), @@ -44,6 +48,14 @@ export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { ); if (numberOfBlocksBack) { loginStrategyOptions.numberOfBlocksBack = numberOfBlocksBack; + ipfsUrl: ipfsConfig.url, + }; + const numBlocksBack = configService.get('STRATEGY_NUM_BLOCKS_BACK'); + if (numBlocksBack) { + loginStrategyParams = { + ...loginStrategyParams, + ...{ numberOfBlocksBack: parseInt(numBlocksBack) }, + }; } super(...loginStrategyParams); } diff --git a/src/modules/did/did.module.ts b/src/modules/did/did.module.ts index 242e5786..bc4243e4 100644 --- a/src/modules/did/did.module.ts +++ b/src/modules/did/did.module.ts @@ -11,10 +11,6 @@ import { DIDResolver } from './did.resolver'; import { DIDService } from './did.service'; import { ethrReg } from '@ew-did-registry/did-ethr-resolver'; import { ConfigService } from '@nestjs/config'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; -import { IpfsConfig } from '../ipfs/ipfs.types'; -import { PIN_CLAIM_QUEUE_NAME, UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; -import { PinProcessor } from './pin.processor'; const RegistrySettingsProvider = { provide: 'RegistrySettings', @@ -45,14 +41,7 @@ const RegistrySettingsProvider = { DIDResolver, Provider, RegistrySettingsProvider, - { - provide: DidStoreInfura, - useFactory: (ipfsConfig: IpfsConfig) => { - return new DidStoreInfura(ipfsConfig); - }, - inject: [{ token: 'IPFSClientConfig', optional: false }], - }, ], - exports: [DIDService, RegistrySettingsProvider, DidStoreInfura], + exports: [DIDService, RegistrySettingsProvider], }) export class DIDModule {} diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index b49fdd9f..f93c81bf 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -10,7 +10,6 @@ import { import { ConfigService } from '@nestjs/config'; import { Job, Queue } from 'bull'; import { Logger } from '../logger/logger.service'; -import { DIDDocumentEntity } from './did.entity'; import { DIDService } from './did.service'; import { ADD_DID_DOC_JOB_NAME, diff --git a/src/modules/did/did.service.spec.ts b/src/modules/did/did.service.spec.ts index 4b106681..63a4e1e6 100644 --- a/src/modules/did/did.service.spec.ts +++ b/src/modules/did/did.service.spec.ts @@ -2,7 +2,6 @@ import { IDIDDocument } from '@ew-did-registry/did-resolver-interface'; import { addressOf, ethrReg } from '@ew-did-registry/did-ethr-resolver'; import { Methods, Chain } from '@ew-did-registry/did'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; import { getQueueToken } from '@nestjs/bull'; import { HttpService } from '@nestjs/axios'; import { ConfigService } from '@nestjs/config'; @@ -18,6 +17,7 @@ import { Logger } from '../logger/logger.service'; import { SentryTracingService } from '../sentry/sentry-tracing.service'; import { EthereumDIDRegistry } from '../../ethers/EthereumDIDRegistry'; import { PIN_CLAIM_QUEUE_NAME, UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; +import { IPFSService } from '../ipfs/ipfs.service'; const { formatBytes32String } = utils; @@ -119,7 +119,7 @@ describe('DidDocumentService', () => { }), inject: [ConfigService], }, - { provide: DidStoreInfura, useValue: MockObject }, + { provide: IPFSService, useValue: MockObject }, ], }).compile(); await module.init(); diff --git a/src/modules/did/did.service.ts b/src/modules/did/did.service.ts index 8c9351d2..64313fbc 100644 --- a/src/modules/did/did.service.ts +++ b/src/modules/did/did.service.ts @@ -26,7 +26,6 @@ import { DidEventNames, RegistrySettings, } from '@ew-did-registry/did-resolver-interface'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; import { documentFromLogs, Resolver, @@ -64,7 +63,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { private readonly provider: Provider, private readonly sentryTracingService: SentryTracingService, @Inject('RegistrySettings') registrySettings: RegistrySettings, - private readonly didStore: DidStoreInfura + private readonly ipfsService: IPFSService ) { this.logger.setContext(DIDService.name); @@ -327,15 +326,12 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { * @param did DID of the document service endpoints */ public async resolveServiceEndpoints(did: string) { - if (!this.didStore) { - throw new Error(`resolveServiceEndpoints: DIDStore is undefined`); - } const { service } = await this.getById(did); return Promise.all( service .map(({ serviceEndpoint }) => serviceEndpoint) .filter((endpoint) => IPFSService.isCID(endpoint)) - .map((cid) => this.didStore.get(cid)) + .map((cid) => this.ipfsService.get(cid)) ); } @@ -424,7 +420,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { return logs; } - private resolveNotCachedClaims( + private async resolveNotCachedClaims( services: IServiceEndpoint[], cachedServices: IClaim[] = [] ): Promise { @@ -442,7 +438,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { return { serviceEndpoint, ...rest }; } - const token = await this.didStore.get(serviceEndpoint); + const token = await this.ipfsService.get(serviceEndpoint); if (isJWT(token)) { const decodedData = jwt.decode(token) as { diff --git a/src/modules/ipfs/ipfs.controller.ts b/src/modules/ipfs/ipfs.controller.ts new file mode 100644 index 00000000..fd2c953c --- /dev/null +++ b/src/modules/ipfs/ipfs.controller.ts @@ -0,0 +1,46 @@ +import { + Body, + Controller, + Get, + Param, + Post, + UseInterceptors, +} from '@nestjs/common'; +import { ApiBody, ApiOperation, ApiParam, ApiTags } from '@nestjs/swagger'; +import { CID } from 'multiformats/cid'; +import { CIDPipe } from '../../common/cid.pipe'; +import { Auth } from '../auth/auth.decorator'; +import { SentryErrorInterceptor } from '../interceptors/sentry-error-interceptor'; +import { IPFSService } from './ipfs.service'; + +@Auth() +@UseInterceptors(SentryErrorInterceptor) +@Controller({ path: 'ipfs', version: '1' }) +export class IPFSController { + constructor(private ipfsService: IPFSService) {} + + @Get('/:cid') + @ApiTags('IPFS') + @ApiOperation({ + summary: 'Returns credential from IPFS Store', + description: 'Returns credential represented by service in DID document.', + }) + @ApiParam({ name: 'cid', type: 'string', required: true }) + public async get(@Param('cid', CIDPipe) cid: CID): Promise { + return this.ipfsService.get(cid.toString()); + } + + @Post() + @ApiTags('IPFS') + @ApiBody({ + type: 'string', + description: 'Stringified credential', + }) + @ApiOperation({ + summary: 'Saves credential in IPFS', + description: 'Saves credential on IPFS and returns its CID', + }) + public async save(@Body() credential: Record) { + return this.ipfsService.save(JSON.stringify(credential)); + } +} diff --git a/src/modules/ipfs/ipfs.module.ts b/src/modules/ipfs/ipfs.module.ts index 722e76af..2c7b8501 100644 --- a/src/modules/ipfs/ipfs.module.ts +++ b/src/modules/ipfs/ipfs.module.ts @@ -1,38 +1,108 @@ +import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; import { Module, Global } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { IPFSController } from './ipfs.controller'; import { IPFSService } from './ipfs.service'; -import { IpfsConfig } from './ipfs.types'; +import { + IpfsClusterConfig, + IPFSClusterConfigToken, + IpfsGatewayConfig, + IPFSGatewayConfigToken, + PINS_QUEUE, +} from './ipfs.types'; +import { BullModule } from '@nestjs/bull'; +import { PinsProcessor } from './pins.processor'; -const IPFSClientConfigProvider = { - provide: 'IPFSClientConfig', - useFactory: (config: ConfigService): IpfsConfig => { - const IPFS_CLIENT_PORT = config.get('IPFS_CLIENT_PORT'); +const IPFSGatewayConfigProvider = { + provide: IPFSGatewayConfigToken, + useFactory: (config: ConfigService): IpfsGatewayConfig => { + const IPFS_CLIENT_URL = config.get('IPFS_CLIENT_URL'); + const IPFS_CLIENT_PROTO = config.get('IPFS_CLIENT_PROTO'); const IPFS_CLIENT_HOST = config.get('IPFS_CLIENT_HOST'); - const IPFS_CLIENT_PROJECT_SECRET = config.get( - 'IPFS_CLIENT_PROJECT_SECRET' - ); - const IPFS_CLIENT_PROJECT_ID = config.get('IPFS_CLIENT_PROJECT_ID'); - // https://community.infura.io/t/how-to-add-internet-content-from-a-url-using-ipfs-http-client/5188 - const authorization = - 'Basic ' + - Buffer.from( - `${IPFS_CLIENT_PROJECT_ID}:${IPFS_CLIENT_PROJECT_SECRET}` - ).toString('base64'); - return { + const IPFS_CLIENT_PORT = config.get('IPFS_CLIENT_PORT'); + const ipfsConfig: IpfsGatewayConfig = { + url: IPFS_CLIENT_URL, + protocol: IPFS_CLIENT_PROTO, host: IPFS_CLIENT_HOST, port: parseInt(IPFS_CLIENT_PORT), - protocol: 'https', - headers: { - authorization, - }, }; + const IPFS_CLIENT_PASSWORD = config.get('IPFS_CLIENT_PASSWORD'); + const IPFS_CLIENT_USER = config.get('IPFS_CLIENT_USER'); + // https://community.infura.io/t/how-to-add-internet-content-from-a-url-using-ipfs-http-client/5188 + if (IPFS_CLIENT_USER && IPFS_CLIENT_PASSWORD) { + ipfsConfig.headers = { + Authorization: + 'Basic ' + + Buffer.from(`${IPFS_CLIENT_USER}:${IPFS_CLIENT_PASSWORD}`).toString( + 'base64' + ), + // Authorization: 'Basic WHpOY1NoVGF1R1NTQkk6Y0dnMVZ2dld0dkI1QTF5UW84SE8=', + }; + } + return ipfsConfig; + }, + inject: [ConfigService], +}; + +const IPFSClusterConfigProvider = { + provide: IPFSClusterConfigToken, + useFactory: (config: ConfigService): IpfsClusterConfig => { + const IPFS_CLUSTER_ROOT_URL = config.get('IPFS_CLUSTER_ROOT_URL'); + const IPFS_CLUSTER_USER = config.get('IPFS_CLUSTER_USER'); + const IPFS_CLUSTER_PASSWORD = config.get('IPFS_CLUSTER_PASSWORD'); + let ipfsConfig: IpfsClusterConfig; + if (IPFS_CLUSTER_USER && IPFS_CLUSTER_PASSWORD) { + const headers = { + authorization: + 'Basic ' + + Buffer.from(`${IPFS_CLUSTER_USER}:${IPFS_CLUSTER_PASSWORD}`).toString( + 'base64' + ), + }; + ipfsConfig = [IPFS_CLUSTER_ROOT_URL, headers]; + } else { + ipfsConfig = [IPFS_CLUSTER_ROOT_URL]; + } + return ipfsConfig; }, inject: [ConfigService], }; @Global() @Module({ - providers: [IPFSClientConfigProvider, IPFSService], - exports: [IPFSClientConfigProvider, IPFSService], + imports: [ + BullModule.registerQueue({ + name: PINS_QUEUE, + }), + ], + providers: [ + IPFSClusterConfigProvider, + IPFSGatewayConfigProvider, + IPFSService, + { + provide: DidStoreCluster, + useFactory: (ipfsConfig: IpfsClusterConfig) => { + return new DidStoreCluster(...ipfsConfig); + }, + inject: [{ token: IPFSClusterConfigToken, optional: false }], + }, + { + provide: DidStoreGateway, + useFactory: (ipfsConfig: IpfsGatewayConfig) => { + return new DidStoreGateway(ipfsConfig); + }, + inject: [{ token: IPFSGatewayConfigToken, optional: false }], + }, + PinsProcessor, + ], + controllers: [IPFSController], + exports: [ + IPFSClusterConfigProvider, + IPFSGatewayConfigProvider, + IPFSService, + DidStoreCluster, + DidStoreGateway, + ], }) export class IPFSModule {} diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index 97884e1c..78a9aa66 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -1,6 +1,44 @@ +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { + HttpException, + HttpStatus, + Injectable, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; import { CID } from 'multiformats/cid'; +import { InjectQueue } from '@nestjs/bull'; +import { Queue } from 'bull'; +import { inspect } from 'util'; +import { PINS_QUEUE, PIN_CLAIM } from './ipfs.types'; +import { Logger } from '../logger/logger.service'; + +@Injectable() +export class IPFSService implements OnModuleInit, OnModuleDestroy { + constructor( + private didStoreCluster: DidStoreCluster, + private didStoreGateway: DidStoreGateway, + @InjectQueue(PINS_QUEUE) private readonly pinsQueue: Queue, + private readonly logger: Logger + ) { + this.logger.setContext(IPFSService.name); + } + + async onModuleInit() { + const jobsCount = await this.pinsQueue.getJobCounts(); + this.logger.info( + `Service endpoints pinning jobs statuses ${inspect(jobsCount, { + depth: 3, + colors: true, + })}` + ); + } + + async onModuleDestroy() { + await this.pinsQueue.close(); + } -export class IPFSService { /** * Check if given value is a valid IPFS CID. * @@ -26,4 +64,47 @@ export class IPFSService { return false; } } + + /** + * Get claim from cluster. If claim isn't found tries to get from gateway + * + * @param cid Content identifier. + * @returns Stringified credential + */ + public async get(cid: string): Promise { + let claim: string; + if (await this.didStoreCluster.isPinned(cid)) { + this.logger.debug(`${cid} was pinned. Getting from cluster`); + claim = await this.didStoreCluster.get(cid); + } else { + this.logger.debug(`${cid} was not pinned. Getting from gateway`); + try { + claim = await this.didStoreGateway.get(cid); + await this.pinsQueue.add(PIN_CLAIM, JSON.stringify({ cid, claim })); + } catch (e) { + // 504 is the expected response code when IPFS gateway is unable to provide content within time limit + // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#504-gateway-timeout + // In other words, this is the expected response code if the traversal of the DHT fails to find the content + if (e?.response?.status === 504 || e?.response?.status === 404) { + throw new HttpException( + `Claim ${cid} not found`, + HttpStatus.NOT_FOUND + ); + } else { + throw e; + } + } + } + return claim; + } + + /** + * Saves credential on cluster + * + * @param credential Credential being persisted + * @returns CID of the persisted credential + */ + public async save(credential: string): Promise { + return this.didStoreCluster.save(credential); + } } diff --git a/src/modules/ipfs/ipfs.types.ts b/src/modules/ipfs/ipfs.types.ts index 807c9379..02e038b7 100644 --- a/src/modules/ipfs/ipfs.types.ts +++ b/src/modules/ipfs/ipfs.types.ts @@ -1,7 +1,21 @@ -export interface IpfsConfig { - host: string; - port?: number; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; + +export type IpfsClusterConfig = ConstructorParameters; + +// copied from https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client#createoptions, because ipfs-http-client isnt' typed +export type IpfsGatewayConfig = { + url?: string; protocol?: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - headers?: Record; -} + host?: string; + port?: number; + path?: string; + headers?: { + Authorization?: string; + }; +}; + +export const IPFSGatewayConfigToken = Symbol.for('IPFSGatewayConfigToken'); +export const IPFSClusterConfigToken = Symbol.for('IPFSClusterConfigToken'); + +export const PIN_CLAIM = 'pinClaim'; +export const PINS_QUEUE = 'pins'; diff --git a/src/modules/ipfs/pins.processor.ts b/src/modules/ipfs/pins.processor.ts new file mode 100644 index 00000000..46fdd53f --- /dev/null +++ b/src/modules/ipfs/pins.processor.ts @@ -0,0 +1,36 @@ +import { OnQueueError, Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { Logger } from '../logger/logger.service'; +import { PINS_QUEUE, PIN_CLAIM } from './ipfs.types'; + +@Processor(PINS_QUEUE) +export class PinsProcessor { + constructor( + private readonly didStoreCluster: DidStoreCluster, + private readonly logger: Logger + ) { + this.logger.setContext(PinsProcessor.name); + } + + @OnQueueError() + onError(error: Error) { + this.logger.error(error); + } + + @Process(PIN_CLAIM) + public async pinClaim({ data }: Job) { + const { cid: cidGateway, claim } = JSON.parse(data); + this.logger.debug(`Pinning ${claim}`); + try { + const cidCluster = await this.didStoreCluster.save(claim); + await this.didStoreCluster.pin(cidGateway); + this.logger.debug(`${cidGateway} saved on cluster as ${cidCluster}`); + if (claim !== (await this.didStoreCluster.get(cidGateway))) { + throw new Error('Cluster content is not resolved by gateway CID'); + } + } catch (e) { + this.logger.error(e.message); + } + } +} From 7127f8fcf16ce23e4fc12559757188ef48f0f9e3 Mon Sep 17 00:00:00 2001 From: nichonien Date: Tue, 11 Jul 2023 11:38:02 +0530 Subject: [PATCH 02/24] feat: add script to delete user credentials --- src/scripts/delete-user-credentials.js | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/scripts/delete-user-credentials.js diff --git a/src/scripts/delete-user-credentials.js b/src/scripts/delete-user-credentials.js new file mode 100644 index 00000000..f40341a5 --- /dev/null +++ b/src/scripts/delete-user-credentials.js @@ -0,0 +1,47 @@ +const { createConnection } = require('typeorm'); +const util = require('util'); +const { DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD } = process.env; +const { RoleClaim } = require('./entities/roleClaim.entity.js'); +(async function () { + const connection = await createConnection({ + // had to copy from orgmconfig because typeorm doesn't detect postgres drive + type: 'postgres', + host: DB_HOST, + port: DB_PORT, + username: DB_USERNAME, + password: DB_PASSWORD, + database: DB_NAME, + entities: ['./**/*.entity.js'], + }); + + const userDID = 'did:ethr:volta:0x7A06aCD185116f48Df7Fdd699c7b09ccDb18CA44'; + const roleClaimRepository = connection.getRepository('RoleClaim'); + + // check if the user has any role claims + const claimsBeforeDeletion = await roleClaimRepository.find({ + where: { + requester: userDID, + }, + }); + console.log( + util.inspect(claimsBeforeDeletion, false, null, true /* enable colors */) + ); + + const claim = await roleClaimRepository + .createQueryBuilder('roleClaim') + .delete() + .from(RoleClaim) + .where('requester = :requester', { + requester: userDID, + }) + .execute(); + + // returns [ ] as all the credentials are deleted + const claims = await roleClaimRepository.find({ + where: { + requester: userDID, + }, + }); + console.log(util.inspect(claims, false, null, true /* enable colors */)); + await connection.close(); +})(); From b80878698f842be2234fcac7dc16a9d089d82663 Mon Sep 17 00:00:00 2001 From: JGiter Date: Tue, 11 Jul 2023 17:47:27 +0300 Subject: [PATCH 03/24] refactor: mv pin to ipfs module --- .env.dev | 6 +- .github/workflows/build-test.yml | 2 +- devops/dev/values.yaml | 2 +- docs/api/README.md | 10 +- docs/api/classes/common_cid_pipe.CIDPipe.md | 43 +++++++ .../modules_did_did_service.DIDService.md | 4 +- .../modules_did_pin_processor.PinProcessor.md | 114 ------------------ ...les_ipfs_ipfs_controller.IPFSController.md | 58 +++++++++ .../modules_ipfs_ipfs_service.IPFSService.md | 72 ++++++++++- ...modules_ipfs_pin_processor.PinProcessor.md | 98 +++++++++++++++ .../modules_ipfs_ipfs_types.IpfsConfig.md | 36 ------ docs/api/modules.md | 5 +- docs/api/modules/common_cid_pipe.md | 7 ++ docs/api/modules/modules_did_did_types.md | 14 --- docs/api/modules/modules_did_pin_processor.md | 7 -- .../modules/modules_ipfs_ipfs_controller.md | 7 ++ docs/api/modules/modules_ipfs_ipfs_types.md | 60 ++++++++- .../api/modules/modules_ipfs_pin_processor.md | 7 ++ docs/api/modules/scripts_truncate_db.md | 1 + e2e/app.e2e.spec.ts | 8 +- e2e/ipfs/ipfs.testSuite.ts | 32 +++-- package-lock.json | 35 +++++- package.json | 6 +- src/env-vars-validation-schema.ts | 8 +- src/modules/auth/login.strategy.ts | 15 +-- src/modules/did/did.module.ts | 5 +- src/modules/did/did.processor.ts | 8 +- src/modules/did/did.service.spec.ts | 6 +- src/modules/did/did.types.ts | 3 - src/modules/did/pin.processor.ts | 86 ------------- src/modules/ipfs/ipfs.module.ts | 51 ++++---- src/modules/ipfs/ipfs.service.ts | 63 +++++----- src/modules/ipfs/ipfs.types.ts | 8 +- src/modules/ipfs/pin.processor.ts | 60 +++++++++ src/modules/ipfs/pins.processor.ts | 36 ------ src/scripts/truncate-db.ts | 31 +++++ 36 files changed, 590 insertions(+), 424 deletions(-) create mode 100644 docs/api/classes/common_cid_pipe.CIDPipe.md delete mode 100644 docs/api/classes/modules_did_pin_processor.PinProcessor.md create mode 100644 docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md create mode 100644 docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md delete mode 100644 docs/api/interfaces/modules_ipfs_ipfs_types.IpfsConfig.md create mode 100644 docs/api/modules/common_cid_pipe.md delete mode 100644 docs/api/modules/modules_did_pin_processor.md create mode 100644 docs/api/modules/modules_ipfs_ipfs_controller.md create mode 100644 docs/api/modules/modules_ipfs_pin_processor.md create mode 100644 docs/api/modules/scripts_truncate_db.md delete mode 100644 src/modules/did/pin.processor.ts create mode 100644 src/modules/ipfs/pin.processor.ts delete mode 100644 src/modules/ipfs/pins.processor.ts create mode 100644 src/scripts/truncate-db.ts diff --git a/.env.dev b/.env.dev index afc77a0a..acfb4d59 100644 --- a/.env.dev +++ b/.env.dev @@ -41,15 +41,15 @@ ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS=21 ASSETS_SYNC_ENABLED=true # IPFS -IPFS_CLUSTER_ROOT_URL # Cluster can be deployed locally with e2e/setupIpfsCluster/index.ts +IPFS_CLUSTER_ROOT # Cluster can be deployed locally with e2e/setupIpfsCluster/index.ts IPFS_CLUSTER_USER # Not needed locally IPFS_CLUSTER_PASSWORD # Not needed locally IPFS_CLIENT_URL # Can be public gateway https://ipfs.github.io/public-gateway-checker/, Inura gateway or IPFS cluster. When cluster used as IPFS gateway it supports only GET IPFS_CLIENT_PROTO # These can be IPFS_CLIENT_HOST # used instead of IPFS_CLIENT_PORT # IPFS_CLIENT_URL -IPFS_CLIENT_USER # User of cluster or Id of Infura project -IPFS_CLIENT_PASSWORD # Password to cluster or secret of Infura project +IPFS_CLIENT_PROJECT_ID # Id of Infura project or user of cluster +IPFS_CLIENT_PROJECT_SECRET # Secret of Infura project or cluster password # INTERVALS DIDDOC_SYNC_INTERVAL_IN_HOURS=1 diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 4b6eade5..2a43969d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -46,7 +46,7 @@ env: ASSETS_SYNC_INTERVAL_IN_HOURS: ${{secrets.ASSETS_SYNC_INTERVAL_IN_HOURS}} ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS: ${{secrets.ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS}} ASSETS_SYNC_ENABLED: ${{secrets.ASSETS_SYNC_ENABLED}} - IPFS_URL: ${{secrets.IPFS_URL}} + IPFS_CLIENT_URL: ${{secrets.IPFS_URL}} NESTJS_PORT: ${{secrets.NESTJS_PORT}} STRATEGY_CACHE_SERVER: ${{secrets.STRATEGY_CACHE_SERVER}} STRATEGY_PRIVATE_KEY: ${{secrets.STRATEGY_PRIVATE_KEY}} diff --git a/devops/dev/values.yaml b/devops/dev/values.yaml index a844ccb0..f5dfb231 100644 --- a/devops/dev/values.yaml +++ b/devops/dev/values.yaml @@ -144,6 +144,6 @@ iam-cache-server-helm: STRATEGY_PRIVATE_KEY: AgBM4BIbZIvZ9ErUDLWmmgw0UMEGk4k9hO54/tos4Rf9bUeewlDyAOrnz+gcl8uXuwFiGazQYNia2JFOjCapKnVt0h3LUjvubN4G2cVoQQl9E2+BDzSb7Ucji2UsdleN1ScOwd6XOADdw7Pa1LUXdpf32rf5YADhdYLGDiYy6hqpJ0OPd+61s92HgtyKHioTSP3fJ4YyFqgXQ4wORRR15rdwJdvzSxUrzGnft/ceuB+ilGJc0qSu/jT+SifQ1411zZ6/vsvhfvSq7q4wXEibn2OWSy7MxxNdYafVC7PbyFgc9i6t8V2YKJzUigIGV53ds9GuMK8i8CeGVg2R3e4iHyuOuWv6LeNjbdUd6QeAynOFVos9rBWwS/L7g+Rv0aUUnD016GYSoPwLGPRwvpyhk3jRtJ94uj4FAkUwWGWKagTA+F0uZyIjq2emO4opE8J5FwbPJOA3SKxfo0yPxgQF7S7uTAGlslQdxSrBk9ICUb5FuMVZ32Bxv3fbJ4R3xIS6D8g9hfPG3/P+zpvTxMzPlOjn7bgXHpbV6qOtt47OjTb9A+ELEVnwF/hOY6efSNa/MYlDU0wjF8CGveKFmYkSo9QHnTx7ruGu1s18yXMQZGK47WEERFR3WS0JqB7iQKv2KhPlMJLQapbROLP3MEYGWpEExDdfQRksmvCpQvQhM/7rtBpjobktxKfP6FVsiCTiL0A8Lkbo+i48oVti3SK6vNF2kDBhy4b6uLWwztYNvz3XL1ZalmcWSwe6iMtscD8Vcjms4FVe3hBRPZbXK5/wOpMq ENS_URL: AgCaICh+WJ7pQ2MsRfznNhqFAqauOls7BMQeKb79yyrrFE9PbGRfVaFjL/R1HPzFbdfPzD2OT1kvcJDN/R6Jc4CH9+Vg1z6QTaknXhHsZtBiN+BRCmRwfyno0QFjsWTLVZ83NTSKP2EvwqU5h4/peKavCx7pSkDb31xqf0Mq9V9Kapqw7YvKjEjfvRzdbtAxBtLt3v2UoyOyCldnfXcnenDyHmSgBn6uKS6U8pK/5RlGlsegvCokzR20vw4l8bf60uR/36fX+5ttoYzF+BWI3FWgrry2S/EChv4umk0M8Nr8vYeNc40cLuVJPlxFTr/eHAb+uqKe/bhavJqAvzJFW714/QTNDDKyWqmZMVj6H6BSLSVHHmh6M0Cp2YOH07xecHCgPWqTWzJq1HCfMAuB9hH5wQYVxLhYoIze6XzD8AAzJEFUJ+n+TXmaxH7kuqhVJGbgTtXxbk+QvKeZmX36IcBOl6BxkUlm9VGvj91zCip9zC/HiBbl8zL6nTd6JpFYgJylaLBeOLvMjoyk25vfs+viehC9HbEtsgcLtNevSAzA1sCBFF1c2x4vkyi4yFI2OIp1+FY6xPBkaR5rqMlwiPXS8Al9lgTATHoZgr+tkdcjFwg2wEokmyYPTzOcLc82NU0bZeBclUCVEkxeWCHcNwBRW2heNyujRLWl35klbQPbFlQjQNNOnj0wpp1K1+KFetqjKm7s4Xqjk7UAiCjAX59VIkPpvvTc7iOFHEWqvfdLHciJDxYNuN7nPZk+bVtj/7KNSdCf2Q== IPFS_CLIENT_PROJECT_SECRET: AgCD3q5miwVO6GjALhdNMsbLQul35EQybREj9bV3IQJtC3ij7b65vpySapMAG1NBx6UMm8cqBRKySPxs85ICAevdtPXp9QtDLndTb/HiuEKC945Pw4R7g2V3RJaKgnKO/WZb2HRnaFq3ol63A4eIC2gcxAcU/M2Cinf3mQ8+Ha4D3ZwRTZBp+Vl7aD0vmDF8DrAaczBYs5qX0e4QYwCMlTviHGw/XNNKoQidl+jL+QGWE0aklOMrS9EBgcGtggYaBaNl5eqPAvhNllfsLIKAe4yt5QTQ2hCCih12/8HifOkAQAQw4l6xtzl3idfCSstwsMwMuAeBS/WUd40Y6tD1gR40AxSYiRC/FfMgDhW/ftqZD5wQq67YM7fD2BjJaM2UPDmzcyCdDqRGtHQNeCc9wf6iksGUv/4e68HRlrDQEw+DYqudeKcDs8jbQhJiecCDLGUr3Bb0Ww7Wi+tvs/rqtGrJzLyEJ5kM3XlEeuvA5qaGoJpEOa/0AnHLysfL3KnK+uA6t9KQTNUlRasmATIiHO0X0xSVJj0w30qgN6My83SbnIcbyno2rzlLktBxc+ZeB7pLb2ZvdFYYNgh05F0BXUMmqNXMAYerLpfZvAG4Y7JYcaPVkXnR16rG0vl0Qv+6/mVXbHV/Udfa8wjUrkbzl17mPGKZjmYdF8GoeGbjHsFM4JWceHsxbrX8d8m9NqPqYDRD420a6cqCFZUkpoq7RTa1hZUJ7HX5hlGyg75DuomDNQ== - IPFS_CLUSTER_ROOT_URL: AgCK8khzwUAWcFtB9tO58fCOxUuE9AQqKFP1tO/0zHWLesuiz2N5cXwiySCeokJ3uo2lFfNdclBitV9Mew+mtbKWEJi1s9bXNN83P+5ctJLwXp75MuqROZJxrnigVrb/3BLwgjo8C1KYnH4BHjYvnsN32TmBneIbtdRqsivvs+KT5Xbe6SFj6RlUH7Du9FIfUq5dmf4Bc4hH6Bopwl0KxBZspSECb66ydQnJVJnImKBNpJwySVdwJA3cqYzH/7KE5NR1nByHpVh6Lv0OO6aikq/XGYrBz+pny1NFcLBNKt8qP280SYa8jSRxc+Gd+632LuDsYVnOJLVgzbJllwnTtGUki9EwNAOsIfU5TblJfDJbZ9nLGsispYPrh/DoZHvt4Y0DxljEZomUR+uj1h0x8WCtEX9eRCaVjXW86gcYQ8ssLjKnXTY1n5axQY0IZtc1QyxKp9LEVUGe/UJRTzzR7J+SJTCeeHI9+5ED8wJrAC2SfWuUHYPczB/z4BHRYRv4enO+AWpr3NaMFVD3uQsGRRq4vgw+z8h9O46T1T3uzrIKyL8WdUuzu9nEsC9MZmtlTYcUqAeZZq/rJ9bsgIrRmo1D/3N6aHWU9MNnv+ZPlJeYzRAIqmQEcrWm6pxhzWGzRUOwpD9FYl9xEgmDn5szk4+ihwa776qUXiRkp/6630Y0hkBsext0Jefpe2iXuLroIhSCCYKswWBmMtEpVYHuhet8tbmjmiNpkXo6ysjVSTZ1BvxuTXXS4+dLzgTR6g== + IPFS_CLUSTER_ROOT: AgCK8khzwUAWcFtB9tO58fCOxUuE9AQqKFP1tO/0zHWLesuiz2N5cXwiySCeokJ3uo2lFfNdclBitV9Mew+mtbKWEJi1s9bXNN83P+5ctJLwXp75MuqROZJxrnigVrb/3BLwgjo8C1KYnH4BHjYvnsN32TmBneIbtdRqsivvs+KT5Xbe6SFj6RlUH7Du9FIfUq5dmf4Bc4hH6Bopwl0KxBZspSECb66ydQnJVJnImKBNpJwySVdwJA3cqYzH/7KE5NR1nByHpVh6Lv0OO6aikq/XGYrBz+pny1NFcLBNKt8qP280SYa8jSRxc+Gd+632LuDsYVnOJLVgzbJllwnTtGUki9EwNAOsIfU5TblJfDJbZ9nLGsispYPrh/DoZHvt4Y0DxljEZomUR+uj1h0x8WCtEX9eRCaVjXW86gcYQ8ssLjKnXTY1n5axQY0IZtc1QyxKp9LEVUGe/UJRTzzR7J+SJTCeeHI9+5ED8wJrAC2SfWuUHYPczB/z4BHRYRv4enO+AWpr3NaMFVD3uQsGRRq4vgw+z8h9O46T1T3uzrIKyL8WdUuzu9nEsC9MZmtlTYcUqAeZZq/rJ9bsgIrRmo1D/3N6aHWU9MNnv+ZPlJeYzRAIqmQEcrWm6pxhzWGzRUOwpD9FYl9xEgmDn5szk4+ihwa776qUXiRkp/6630Y0hkBsext0Jefpe2iXuLroIhSCCYKswWBmMtEpVYHuhet8tbmjmiNpkXo6ysjVSTZ1BvxuTXXS4+dLzgTR6g== IPFS_CLUSTER_USER: AgBwOgRhIkI20NFyEWwNyEhHELvOVLajzvu+Q3LhfjIWbyVSfP18Hx33RpPc6KR7gx5gJI0UswESjzqxIIgv2FjSl9LHx8ZXa/go0iVq6y/hJBk6ersWD3a9nqiafMqo691C19CRHIAZqpUK8NIzgcGjRo9/gXu95MuOac3I1Qs1ILkTRsfwGAbQMIUup2CO0bbpsc582i3phj/OguTrI8i+vfoNOSoJRk6CNNHH9qzIb5U6A0fQkic94IkcGIi05rPmUoU3WFB3ebgs7iD1ZRNV5syMNNjvMC7YV9stYok5TbH1Fqnq+OVlu0bfDKnQ2o8SPx4+lDhKVrEnFKH0AeTNOXqM1ZARMaOeWSell4MQAcnSggAXn3SexSy0D1xNy6MrH7uydqO8r8y52BXRVMc0lWkuxcTFmzjPuxG9IEHzGUQ55Rh39FI5/rizEXWV5VYHTcnl+IAa7LqqzfN4HC0FzxYgQZ4ZBsWX0VhL5ZqJyh1Yo856u1vxnPCCaHGlFyAHKueFxh8iMv0F6LxiSXgP5sJyQ8py7piisDbEJUTXsLWwqmhs/5T1eaOiH3cPUcCJZsY8Z362vwguzz9TAXzDRDhSddAEhbZ5NA7aBPxSo3YKmLLXpDvObW82TelZhbRW5epK0xwPNWVmjnOdcm4V6fj76+aiUPP8Iu74ImFq4EEtEpwB30BI9y2iJI2jdxL21n17bZYke3WatAcCUg== IPFS_CLUSTER_PASSWORD: AgAhHW1lN0qi1JitHn10qC4fF8vdh2J2mBbA9MVs5vMHHSdOhOX/s+JAtFAMVplIqaxBUjM4VsWFDqs4LZLN3I06pTDe4srDenOCvdlT2Pcupo6a/ftlVQ/MidOA68Qz5i0HFBlirOG9ZtdfdTEq41CqgvaP73oXbG7pkgkQqiydVglvKGmWOr8QWwqVLYqI1Jlrmz9OKcyC6AKaix7+FToVw2IJqH4dp73ciuQ5qwlOIOHIcsuGXgnvHaGYcbcOt1FYjzF/NTaKq3q69WvffM6UMCQq1m+RHBVpTAMQInGe2nfLnijAUShWJoozgLbGgU9knStwEQAVDPr2Shp+bjFhAVip3F7ZUQ4GmCkKQPVNccMsgHUib/157kvy2ET6OLVDppG6v5QIrXP5JM/8AIYz8zpwvuqkeumVqR94VRsI8Ryy+vSVjxYsEDuyG892cQ4wMf+dl7SOJINJIFxL0vd5f761lu4aUz0G5aQsbzzSafPlGgEQitm35kAoCbRyVX/2pquXiRsiigdxa7Nl+OJBm/EoNbbISi88eqWlbEOVq5EA1Tu0HC6FU0AW3Xg7G8vGrbHKrXxX5oxTRzNB6crBniAgRAdrPOc/G1ORZhkZQD6wEPUJ9Ny6GPikR9MlZ4v5sMFboapZqGD9IIwxjP+FJlrThMBfXcsqHrqGJWYJx8j5WrJpLUmunI4FBE1NcyHCSQ7rHCBfM46bd/URq19q6XCh0w== diff --git a/docs/api/README.md b/docs/api/README.md index cb010a2c..0ead5735 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -19,15 +19,7 @@ $ cp .env.dev .env $ cp docker-compose.dev.yml docker-compose.yml ``` -Set the following values in your `.env`: - -```text -ENS_URL= -IPFS_CLIENT_HOST= -IPFS_CLIENT_PORT= -IPFS_CLIENT_PROJECT_ID= -IPFS_CLIENT_PROJECT_SECRET= -``` +Fill in configuration values in your `.env`. For reference look at `.env.dev` ### Production ```bash diff --git a/docs/api/classes/common_cid_pipe.CIDPipe.md b/docs/api/classes/common_cid_pipe.CIDPipe.md new file mode 100644 index 00000000..c1a42c17 --- /dev/null +++ b/docs/api/classes/common_cid_pipe.CIDPipe.md @@ -0,0 +1,43 @@ +# Class: CIDPipe + +[common/cid.pipe](../modules/common_cid_pipe.md).CIDPipe + +## Implements + +- `PipeTransform` + +## Table of contents + +### Constructors + +- [constructor](common_cid_pipe.CIDPipe.md#constructor) + +### Methods + +- [transform](common_cid_pipe.CIDPipe.md#transform) + +## Constructors + +### constructor + +• **new CIDPipe**() + +## Methods + +### transform + +▸ **transform**(`cid`): `CID` + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `cid` | `string` | + +#### Returns + +`CID` + +#### Implementation of + +PipeTransform.transform diff --git a/docs/api/classes/modules_did_did_service.DIDService.md b/docs/api/classes/modules_did_did_service.DIDService.md index 04ae6d34..bb7ef485 100644 --- a/docs/api/classes/modules_did_did_service.DIDService.md +++ b/docs/api/classes/modules_did_did_service.DIDService.md @@ -28,7 +28,7 @@ ### constructor -• **new DIDService**(`config`, `schedulerRegistry`, `httpService`, `didQueue`, `logger`, `didRepository`, `provider`, `sentryTracingService`, `registrySettings`, `didStore`) +• **new DIDService**(`config`, `schedulerRegistry`, `httpService`, `didQueue`, `logger`, `didRepository`, `provider`, `sentryTracingService`, `registrySettings`, `ipfsService`) #### Parameters @@ -43,7 +43,7 @@ | `provider` | [`Provider`](common_provider.Provider.md) | | `sentryTracingService` | [`SentryTracingService`](modules_sentry_sentry_tracing_service.SentryTracingService.md) | | `registrySettings` | `RegistrySettings` | -| `didStore` | `DidStore` | +| `ipfsService` | [`IPFSService`](modules_ipfs_ipfs_service.IPFSService.md) | ## Methods diff --git a/docs/api/classes/modules_did_pin_processor.PinProcessor.md b/docs/api/classes/modules_did_pin_processor.PinProcessor.md deleted file mode 100644 index 9102bca2..00000000 --- a/docs/api/classes/modules_did_pin_processor.PinProcessor.md +++ /dev/null @@ -1,114 +0,0 @@ -# Class: PinProcessor - -[modules/did/pin.processor](../modules/modules_did_pin_processor.md).PinProcessor - -## Table of contents - -### Constructors - -- [constructor](modules_did_pin_processor.PinProcessor.md#constructor) - -### Methods - -- [onActive](modules_did_pin_processor.PinProcessor.md#onactive) -- [onError](modules_did_pin_processor.PinProcessor.md#onerror) -- [onFailed](modules_did_pin_processor.PinProcessor.md#onfailed) -- [onStalled](modules_did_pin_processor.PinProcessor.md#onstalled) -- [pinClaims](modules_did_pin_processor.PinProcessor.md#pinclaims) - -## Constructors - -### constructor - -• **new PinProcessor**(`logger`, `configService`, `didInfura`) - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | -| `configService` | `ConfigService`<`Record`<`string`, `unknown`\>, ``false``\> | -| `didInfura` | `DidStore` | - -## Methods - -### onActive - -▸ **onActive**(`job`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `job` | `Job`<`any`\> | - -#### Returns - -`void` - -___ - -### onError - -▸ **onError**(`error`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `error` | `Error` | - -#### Returns - -`void` - -___ - -### onFailed - -▸ **onFailed**(`job`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `job` | `Job`<`any`\> | - -#### Returns - -`void` - -___ - -### onStalled - -▸ **onStalled**(`job`): `void` - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `job` | `Job`<`any`\> | - -#### Returns - -`void` - -___ - -### pinClaims - -▸ **pinClaims**(`doc`): `Promise`<`void`\> - -This method migrates claims by retrieving from one DidStore and pinning to another -It was implemented for EW migration from Infura to EW hosted IPFS - -#### Parameters - -| Name | Type | -| :------ | :------ | -| `doc` | [`DIDDocumentEntity`](modules_did_did_entity.DIDDocumentEntity.md) | - -#### Returns - -`Promise`<`void`\> diff --git a/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md new file mode 100644 index 00000000..f87d1d53 --- /dev/null +++ b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md @@ -0,0 +1,58 @@ +# Class: IPFSController + +[modules/ipfs/ipfs.controller](../modules/modules_ipfs_ipfs_controller.md).IPFSController + +## Table of contents + +### Constructors + +- [constructor](modules_ipfs_ipfs_controller.IPFSController.md#constructor) + +### Methods + +- [get](modules_ipfs_ipfs_controller.IPFSController.md#get) +- [save](modules_ipfs_ipfs_controller.IPFSController.md#save) + +## Constructors + +### constructor + +• **new IPFSController**(`ipfsService`) + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `ipfsService` | [`IPFSService`](modules_ipfs_ipfs_service.IPFSService.md) | + +## Methods + +### get + +▸ **get**(`cid`): `Promise`<`string`\> + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `cid` | `CID` | + +#### Returns + +`Promise`<`string`\> + +___ + +### save + +▸ **save**(`credential`): `Promise`<`string`\> + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `credential` | `Record`<`string`, `unknown`\> | + +#### Returns + +`Promise`<`string`\> diff --git a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md index 9a7347a1..ea1cebf3 100644 --- a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md +++ b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md @@ -2,6 +2,10 @@ [modules/ipfs/ipfs.service](../modules/modules_ipfs_ipfs_service.md).IPFSService +## Implements + +- `OnModuleDestroy` + ## Table of contents ### Constructors @@ -10,16 +14,82 @@ ### Methods +- [get](modules_ipfs_ipfs_service.IPFSService.md#get) +- [onModuleDestroy](modules_ipfs_ipfs_service.IPFSService.md#onmoduledestroy) +- [save](modules_ipfs_ipfs_service.IPFSService.md#save) - [isCID](modules_ipfs_ipfs_service.IPFSService.md#iscid) ## Constructors ### constructor -• **new IPFSService**() +• **new IPFSService**(`didStoreCluster`, `didStoreInfura`, `pinsQueue`, `logger`) + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `didStoreCluster` | `DidStore` | +| `didStoreInfura` | `DidStore` | +| `pinsQueue` | `Queue`<`string`\> | +| `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | ## Methods +### get + +▸ **get**(`cid`): `Promise`<`string`\> + +Get claim from cluster. If claim isn't found tries to get from gateway + +#### Parameters + +| Name | Type | Description | +| :------ | :------ | :------ | +| `cid` | `string` | Content identifier. | + +#### Returns + +`Promise`<`string`\> + +Stringified credential + +___ + +### onModuleDestroy + +▸ **onModuleDestroy**(): `Promise`<`void`\> + +#### Returns + +`Promise`<`void`\> + +#### Implementation of + +OnModuleDestroy.onModuleDestroy + +___ + +### save + +▸ **save**(`credential`): `Promise`<`string`\> + +Saves credential on cluster + +#### Parameters + +| Name | Type | Description | +| :------ | :------ | :------ | +| `credential` | `string` | Credential being persisted | + +#### Returns + +`Promise`<`string`\> + +CID of the persisted credential + +___ + ### isCID ▸ `Static` **isCID**(`hash`): `boolean` diff --git a/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md b/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md new file mode 100644 index 00000000..08d85d6c --- /dev/null +++ b/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md @@ -0,0 +1,98 @@ +# Class: PinProcessor + +[modules/ipfs/pin.processor](../modules/modules_ipfs_pin_processor.md).PinProcessor + +## Table of contents + +### Constructors + +- [constructor](modules_ipfs_pin_processor.PinProcessor.md#constructor) + +### Methods + +- [onError](modules_ipfs_pin_processor.PinProcessor.md#onerror) +- [onFailed](modules_ipfs_pin_processor.PinProcessor.md#onfailed) +- [onStalled](modules_ipfs_pin_processor.PinProcessor.md#onstalled) +- [pin](modules_ipfs_pin_processor.PinProcessor.md#pin) + +## Constructors + +### constructor + +• **new PinProcessor**(`logger`, `didStoreCluster`, `didStoreInfura`) + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `logger` | [`Logger`](modules_logger_logger_service.Logger.md) | +| `didStoreCluster` | `DidStore` | +| `didStoreInfura` | `DidStore` | + +## Methods + +### onError + +▸ **onError**(`error`): `void` + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `error` | `Error` | + +#### Returns + +`void` + +___ + +### onFailed + +▸ **onFailed**(`job`, `err`): `void` + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `job` | `Job`<`any`\> | +| `err` | `Error` | + +#### Returns + +`void` + +___ + +### onStalled + +▸ **onStalled**(`job`): `void` + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `job` | `Job`<`any`\> | + +#### Returns + +`void` + +___ + +### pin + +▸ **pin**(`job`): `Promise`<`void`\> + +This method migrates claims by retrieving from one DidStore and pinning to another +It was implemented for EW migration from Infura to EW hosted IPFS + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `job` | `Job`<`any`\> | + +#### Returns + +`Promise`<`void`\> diff --git a/docs/api/interfaces/modules_ipfs_ipfs_types.IpfsConfig.md b/docs/api/interfaces/modules_ipfs_ipfs_types.IpfsConfig.md deleted file mode 100644 index 5a5a71a8..00000000 --- a/docs/api/interfaces/modules_ipfs_ipfs_types.IpfsConfig.md +++ /dev/null @@ -1,36 +0,0 @@ -# Interface: IpfsConfig - -[modules/ipfs/ipfs.types](../modules/modules_ipfs_ipfs_types.md).IpfsConfig - -## Table of contents - -### Properties - -- [headers](modules_ipfs_ipfs_types.IpfsConfig.md#headers) -- [host](modules_ipfs_ipfs_types.IpfsConfig.md#host) -- [port](modules_ipfs_ipfs_types.IpfsConfig.md#port) -- [protocol](modules_ipfs_ipfs_types.IpfsConfig.md#protocol) - -## Properties - -### headers - -• `Optional` **headers**: `Record`<`string`, `any`\> - -___ - -### host - -• **host**: `string` - -___ - -### port - -• `Optional` **port**: `number` - -___ - -### protocol - -• `Optional` **protocol**: `string` diff --git a/docs/api/modules.md b/docs/api/modules.md index 4dd5ffee..bd1e1af9 100644 --- a/docs/api/modules.md +++ b/docs/api/modules.md @@ -9,6 +9,7 @@ - [common/ENSBaseSchema](modules/common_ENSBaseSchema.md) - [common/boolean.pipe](modules/common_boolean_pipe.md) - [common/bootstrap](modules/common_bootstrap.md) +- [common/cid.pipe](modules/common_cid_pipe.md) - [common/constants](modules/common_constants.md) - [common/cors.config](modules/common_cors_config.md) - [common/json.scalar](modules/common_json_scalar.md) @@ -90,16 +91,17 @@ - [modules/did/did.resolver](modules/modules_did_did_resolver.md) - [modules/did/did.service](modules/modules_did_did_service.md) - [modules/did/did.types](modules/modules_did_did_types.md) -- [modules/did/pin.processor](modules/modules_did_pin_processor.md) - [modules/ens/ens.module](modules/modules_ens_ens_module.md) - [modules/ens/ens.service](modules/modules_ens_ens_service.md) - [modules/interceptors/interceptors.module](modules/modules_interceptors_interceptors_module.md) - [modules/interceptors/not-found.interceptor](modules/modules_interceptors_not_found_interceptor.md) - [modules/interceptors/sentry-error-interceptor](modules/modules_interceptors_sentry_error_interceptor.md) - [modules/interceptors/sentry-tracing.interceptor](modules/modules_interceptors_sentry_tracing_interceptor.md) +- [modules/ipfs/ipfs.controller](modules/modules_ipfs_ipfs_controller.md) - [modules/ipfs/ipfs.module](modules/modules_ipfs_ipfs_module.md) - [modules/ipfs/ipfs.service](modules/modules_ipfs_ipfs_service.md) - [modules/ipfs/ipfs.types](modules/modules_ipfs_ipfs_types.md) +- [modules/ipfs/pin.processor](modules/modules_ipfs_pin_processor.md) - [modules/logger/logger.module](modules/modules_logger_logger_module.md) - [modules/logger/logger.service](modules/modules_logger_logger_service.md) - [modules/nats/nats.module](modules/modules_nats_nats_module.md) @@ -157,4 +159,5 @@ - [modules/status-list/status-list.service](modules/modules_status_list_status_list_service.md) - [scripts/migrate](modules/scripts_migrate.md) - [scripts/purge-extraneous-claims](modules/scripts_purge_extraneous_claims.md) +- [scripts/truncate-db](modules/scripts_truncate_db.md) - [setup-swagger.function](modules/setup_swagger_function.md) diff --git a/docs/api/modules/common_cid_pipe.md b/docs/api/modules/common_cid_pipe.md new file mode 100644 index 00000000..0403ee1c --- /dev/null +++ b/docs/api/modules/common_cid_pipe.md @@ -0,0 +1,7 @@ +# Module: common/cid.pipe + +## Table of contents + +### Classes + +- [CIDPipe](../classes/common_cid_pipe.CIDPipe.md) diff --git a/docs/api/modules/modules_did_did_types.md b/docs/api/modules/modules_did_did_types.md index 68349d3c..65b4899b 100644 --- a/docs/api/modules/modules_did_did_types.md +++ b/docs/api/modules/modules_did_did_types.md @@ -9,8 +9,6 @@ ### Variables - [ADD\_DID\_DOC\_JOB\_NAME](modules_did_did_types.md#add_did_doc_job_name) -- [PIN\_CLAIM\_JOB\_NAME](modules_did_did_types.md#pin_claim_job_name) -- [PIN\_CLAIM\_QUEUE\_NAME](modules_did_did_types.md#pin_claim_queue_name) - [UPDATE\_DID\_DOC\_JOB\_NAME](modules_did_did_types.md#update_did_doc_job_name) - [UPDATE\_DOCUMENT\_QUEUE\_NAME](modules_did_did_types.md#update_document_queue_name) - [didPattern](modules_did_did_types.md#didpattern) @@ -27,18 +25,6 @@ ___ -### PIN\_CLAIM\_JOB\_NAME - -• `Const` **PIN\_CLAIM\_JOB\_NAME**: ``"pinning"`` - -___ - -### PIN\_CLAIM\_QUEUE\_NAME - -• `Const` **PIN\_CLAIM\_QUEUE\_NAME**: ``"pinClaimQueue"`` - -___ - ### UPDATE\_DID\_DOC\_JOB\_NAME • `Const` **UPDATE\_DID\_DOC\_JOB\_NAME**: ``"refreshing"`` diff --git a/docs/api/modules/modules_did_pin_processor.md b/docs/api/modules/modules_did_pin_processor.md deleted file mode 100644 index 0efdec5f..00000000 --- a/docs/api/modules/modules_did_pin_processor.md +++ /dev/null @@ -1,7 +0,0 @@ -# Module: modules/did/pin.processor - -## Table of contents - -### Classes - -- [PinProcessor](../classes/modules_did_pin_processor.PinProcessor.md) diff --git a/docs/api/modules/modules_ipfs_ipfs_controller.md b/docs/api/modules/modules_ipfs_ipfs_controller.md new file mode 100644 index 00000000..2632bc5d --- /dev/null +++ b/docs/api/modules/modules_ipfs_ipfs_controller.md @@ -0,0 +1,7 @@ +# Module: modules/ipfs/ipfs.controller + +## Table of contents + +### Classes + +- [IPFSController](../classes/modules_ipfs_ipfs_controller.IPFSController.md) diff --git a/docs/api/modules/modules_ipfs_ipfs_types.md b/docs/api/modules/modules_ipfs_ipfs_types.md index 59e6cb23..333e8e9a 100644 --- a/docs/api/modules/modules_ipfs_ipfs_types.md +++ b/docs/api/modules/modules_ipfs_ipfs_types.md @@ -2,6 +2,62 @@ ## Table of contents -### Interfaces +### Type Aliases -- [IpfsConfig](../interfaces/modules_ipfs_ipfs_types.IpfsConfig.md) +- [IpfsClusterConfig](modules_ipfs_ipfs_types.md#ipfsclusterconfig) +- [IpfsInfuraConfig](modules_ipfs_ipfs_types.md#ipfsinfuraconfig) + +### Variables + +- [IPFSClusterConfigToken](modules_ipfs_ipfs_types.md#ipfsclusterconfigtoken) +- [IPFSInfuraConfigToken](modules_ipfs_ipfs_types.md#ipfsinfuraconfigtoken) +- [PIN\_CLAIM\_JOB\_NAME](modules_ipfs_ipfs_types.md#pin_claim_job_name) +- [PIN\_CLAIM\_QUEUE\_NAME](modules_ipfs_ipfs_types.md#pin_claim_queue_name) + +## Type Aliases + +### IpfsClusterConfig + +Ƭ **IpfsClusterConfig**: `ConstructorParameters` + +___ + +### IpfsInfuraConfig + +Ƭ **IpfsInfuraConfig**: `Object` + +#### Type declaration + +| Name | Type | +| :------ | :------ | +| `headers?` | { `Authorization?`: `string` } | +| `headers.Authorization?` | `string` | +| `host?` | `string` | +| `path?` | `string` | +| `port?` | `number` | +| `protocol?` | `string` | +| `url?` | `string` | + +## Variables + +### IPFSClusterConfigToken + +• `Const` **IPFSClusterConfigToken**: typeof [`IPFSClusterConfigToken`](modules_ipfs_ipfs_types.md#ipfsclusterconfigtoken) + +___ + +### IPFSInfuraConfigToken + +• `Const` **IPFSInfuraConfigToken**: typeof [`IPFSInfuraConfigToken`](modules_ipfs_ipfs_types.md#ipfsinfuraconfigtoken) + +___ + +### PIN\_CLAIM\_JOB\_NAME + +• `Const` **PIN\_CLAIM\_JOB\_NAME**: ``"pinning"`` + +___ + +### PIN\_CLAIM\_QUEUE\_NAME + +• `Const` **PIN\_CLAIM\_QUEUE\_NAME**: ``"pinClaimQueue"`` diff --git a/docs/api/modules/modules_ipfs_pin_processor.md b/docs/api/modules/modules_ipfs_pin_processor.md new file mode 100644 index 00000000..fdbdef7d --- /dev/null +++ b/docs/api/modules/modules_ipfs_pin_processor.md @@ -0,0 +1,7 @@ +# Module: modules/ipfs/pin.processor + +## Table of contents + +### Classes + +- [PinProcessor](../classes/modules_ipfs_pin_processor.PinProcessor.md) diff --git a/docs/api/modules/scripts_truncate_db.md b/docs/api/modules/scripts_truncate_db.md new file mode 100644 index 00000000..8710577a --- /dev/null +++ b/docs/api/modules/scripts_truncate_db.md @@ -0,0 +1 @@ +# Module: scripts/truncate-db diff --git a/e2e/app.e2e.spec.ts b/e2e/app.e2e.spec.ts index 09d69e4d..4ac711d1 100644 --- a/e2e/app.e2e.spec.ts +++ b/e2e/app.e2e.spec.ts @@ -1,5 +1,6 @@ -import d from 'dotenv'; -d.config(); +import { config } from 'dotenv'; +// although it is called in app.module, it is added here to be able to override DID_REGISTRY_ADDRESS. When config() is called next time it will not change variables alredy presented in env +config(); import { Test } from '@nestjs/testing'; import { INestApplication } from '@nestjs/common'; import { ethers, network } from 'hardhat'; @@ -35,13 +36,14 @@ describe('iam-cache-server E2E tests', () => { } beforeAll(async () => { + await import('../src/scripts/truncate-db'); consoleLogSpy = jest.spyOn(global.console, 'log'); didRegistry = await loadFixture(deployDidRegistry); process.env.DID_REGISTRY_ADDRESS = didRegistry.address; cluster = await spawnIpfsCluster(); - process.env.IPFS_CLUSTER_ROOT_URL = 'http://localhost:8080'; + process.env.IPFS_CLUSTER_ROOT = 'http://localhost:8080'; process.env.IPFS_CLIENT_URL = 'http://mocked'; // CID resolved incorrectly through gateway exposed on cluster. TODO: instead of gateway try to expose IPFS API diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts index 88e41bc6..cfe23f84 100644 --- a/e2e/ipfs/ipfs.testSuite.ts +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -4,9 +4,10 @@ import { Queue } from 'bull'; import request from 'supertest'; import { Connection, EntityManager, QueryRunner } from 'typeorm'; import { DidStore as DidStoreCluster } from 'didStoreCluster'; -import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { DidStore as DidStoreGateway } from 'didStoreInfura'; import { app } from '../app.e2e.spec'; import { randomUser } from '../utils'; +import { PIN_CLAIM_QUEUE_NAME } from '../../src/modules/ipfs/ipfs.types'; export const ipfsModuleTestSuite = () => { let queryRunner: QueryRunner; @@ -24,7 +25,8 @@ export const ipfsModuleTestSuite = () => { didStoreCluster = app.get(DidStoreCluster); didStoreInfura = app.get(DidStoreGateway); - pinsQueue = app.get(getQueueToken('pins')); + pinsQueue = app.get(getQueueToken(PIN_CLAIM_QUEUE_NAME)); + await pinsQueue.empty(); const manager = app.get(EntityManager); const dbConnection = app.get(Connection); @@ -41,7 +43,7 @@ export const ipfsModuleTestSuite = () => { await queryRunner.release(); }); - it('should be able to post claim', async () => { + it('save() should post claim in cluster', async () => { const claimData = { claimType: 'claim type', claimTypeVersion: 1, @@ -56,17 +58,22 @@ export const ipfsModuleTestSuite = () => { .send(claimData) .expect(HttpStatus.CREATED); - const get = jest.spyOn(didStoreCluster, 'get'); + const didStoreClusterGet = jest.spyOn(didStoreCluster, 'get'); + jest + .spyOn(didStoreInfura, 'get') + .mockImplementation( + () => new Promise((resolve) => setTimeout(resolve, 24 * 60 * 60 * 1000)) + ); const { text: stored } = await request(app.getHttpServer()) .get(`/v1/ipfs/${cid}`) .set('Cookie', requester.cookies) .expect(HttpStatus.OK); - expect(get).toBeCalledTimes(1); + expect(didStoreClusterGet).toBeCalledTimes(2); // first in IpfsService.get, second in PinProcessor.pin expect(JSON.parse(stored)).toStrictEqual(claimData); }); - it('should return 404 if claim was not persisted in IPFS', async () => { + it('if claim was not persisted in IPFS, get() should respond with 404 status code', async () => { const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); const requester = await randomUser(); @@ -78,7 +85,7 @@ export const ipfsModuleTestSuite = () => { .expect(HttpStatus.NOT_FOUND); }); - it('claim persisted in IPFS should be pinned in cluster', async () => { + it('if claim is not found in cluster, get() should save claim in cluster', async () => { const didStoreClusterGet = jest.spyOn(didStoreCluster, 'get'); const didStoreInfuraGet = jest.spyOn(didStoreInfura, 'get'); @@ -99,14 +106,19 @@ export const ipfsModuleTestSuite = () => { await claimPinned; - expect(didStoreClusterGet).toBeCalledTimes(0); + expect(didStoreClusterGet).toBeCalledTimes(2); expect(didStoreInfuraGet).toBeCalledTimes(1); + jest + .spyOn(didStoreInfura, 'get') + .mockImplementation( + () => new Promise((resolve) => setTimeout(resolve, 24 * 60 * 60 * 1000)) + ); await request(app.getHttpServer()) .get(`/v1/ipfs/${cid}`) .set('Cookie', requester.cookies); - expect(didStoreClusterGet).toBeCalledTimes(1); - expect(didStoreInfuraGet).toBeCalledTimes(1); + expect(didStoreClusterGet).toBeCalledTimes(4); + expect(didStoreInfuraGet).toBeCalledTimes(2); }); }; diff --git a/package-lock.json b/package-lock.json index 3b0ca425..51e060cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -140,7 +140,8 @@ "tsconfig-paths": "^3.9.0", "typedoc": "^0.23.6", "typedoc-plugin-markdown": "^3.13.3", - "typescript": "4.9.5" + "typescript": "4.9.5", + "wait-on": "^7.0.1" }, "engines": { "node": ">=16.19", @@ -33914,6 +33915,25 @@ "resolved": "https://registry.npmjs.org/wait-for-expect/-/wait-for-expect-3.0.2.tgz", "integrity": "sha512-cfS1+DZxuav1aBYbaO/kE06EOS8yRw7qOFoD3XtjTkYvCvh3zUvNST8DXK/nPaeqIzIv3P3kL3lRJn8iwOiSag==" }, + "node_modules/wait-on": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-7.0.1.tgz", + "integrity": "sha512-9AnJE9qTjRQOlTZIldAaf/da2eW0eSRSgcqq85mXQja/DW3MriHxkpODDSUEg+Gri/rKEcXUZHe+cevvYItaog==", + "dev": true, + "dependencies": { + "axios": "^0.27.2", + "joi": "^17.7.0", + "lodash": "^4.17.21", + "minimist": "^1.2.7", + "rxjs": "^7.8.0" + }, + "bin": { + "wait-on": "bin/wait-on" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", @@ -60819,6 +60839,19 @@ "resolved": "https://registry.npmjs.org/wait-for-expect/-/wait-for-expect-3.0.2.tgz", "integrity": "sha512-cfS1+DZxuav1aBYbaO/kE06EOS8yRw7qOFoD3XtjTkYvCvh3zUvNST8DXK/nPaeqIzIv3P3kL3lRJn8iwOiSag==" }, + "wait-on": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/wait-on/-/wait-on-7.0.1.tgz", + "integrity": "sha512-9AnJE9qTjRQOlTZIldAaf/da2eW0eSRSgcqq85mXQja/DW3MriHxkpODDSUEg+Gri/rKEcXUZHe+cevvYItaog==", + "dev": true, + "requires": { + "axios": "^0.27.2", + "joi": "^17.7.0", + "lodash": "^4.17.21", + "minimist": "^1.2.7", + "rxjs": "^7.8.0" + } + }, "walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", diff --git a/package.json b/package.json index eaa53a1b..5b063350 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,8 @@ "migration:test": "NODE_ENV=test ts-node ./node_modules/typeorm/cli.js migration:run -d src/db/cli.ts", "typeorm:cli": "ts-node ./node_modules/typeorm/cli.js", "release": "standard-version", - "purge:extraneous-claims": "node dist/scripts/purge-extraneous-claims.js" + "purge:extraneous-claims": "node dist/scripts/purge-extraneous-claims.js", + "db:truncate": "ts-node -P tsconfig.json src/scripts/truncate-db.ts" }, "dependencies": { "@energyweb/credential-governance": "^2.2.1-alpha.316.0", @@ -183,7 +184,8 @@ "tsconfig-paths": "^3.9.0", "typedoc": "^0.23.6", "typedoc-plugin-markdown": "^3.13.3", - "typescript": "4.9.5" + "typescript": "4.9.5", + "wait-on": "^7.0.1" }, "jest": { "moduleFileExtensions": [ diff --git a/src/env-vars-validation-schema.ts b/src/env-vars-validation-schema.ts index 574a4051..7a0d2a55 100644 --- a/src/env-vars-validation-schema.ts +++ b/src/env-vars-validation-schema.ts @@ -91,11 +91,11 @@ export const envVarsValidationSchema = Joi.object({ IPFS_CLIENT_URL: Joi.string().uri(), IPFS_CLIENT_PROTO: Joi.string(), - IPFS_CLIENT_HOST: Joi.string().uri(), + IPFS_CLIENT_HOST: Joi.string().hostname(), IPFS_CLIENT_PORT: Joi.number().port(), - IPFS_CLIENT_USER: Joi.string(), - IPFS_CLIENT_PASSWORD: Joi.string(), - IPFS_CLUSTER_ROOT_URL: Joi.string().required(), + IPFS_CLIENT_PROJECT_ID: Joi.string(), + IPFS_CLIENT_PROJECT_SECRET: Joi.string(), + IPFS_CLUSTER_ROOT: Joi.string().required(), IPFS_CLUSTER_USER: Joi.string(), IPFS_CLUSTER_PASSWORD: Joi.string(), DID_SYNC_MODE_FULL: Joi.boolean().required(), diff --git a/src/modules/auth/login.strategy.ts b/src/modules/auth/login.strategy.ts index 8485e100..a1c15f05 100644 --- a/src/modules/auth/login.strategy.ts +++ b/src/modules/auth/login.strategy.ts @@ -7,14 +7,14 @@ import { URL } from 'url'; import { RoleIssuerResolver } from '../claim/resolvers/issuer.resolver'; import { RoleRevokerResolver } from '../claim/resolvers/revoker.resolver'; import { RoleCredentialResolver } from '../claim/resolvers/credential.resolver'; -import { IpfsGatewayConfig, IPFSGatewayConfigToken } from '../ipfs/ipfs.types'; +import { IPFSInfuraConfigToken } from '../ipfs/ipfs.types'; import { LoginStrategyOptions } from 'passport-did-auth/dist/lib/LoginStrategy'; @Injectable() export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { constructor( configService: ConfigService, - @Inject('IPFSClientConfig') ipfsConfig, + @Inject(IPFSInfuraConfigToken) ipfsConfig, issuerResolver: RoleIssuerResolver, revokerResolver: RoleRevokerResolver, credentialResolver: RoleCredentialResolver @@ -24,9 +24,6 @@ export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { new URL(configService.get('STRATEGY_CACHE_SERVER')).origin ).href; const loginStrategyOptions: LoginStrategyOptions = { - @Inject(IPFSGatewayConfigToken) ipfsConfig: IpfsGatewayConfig - ) { - let loginStrategyParams: Omit = { name: 'login', rpcUrl: configService.get('ENS_URL'), cacheServerUrl: configService.get('STRATEGY_CACHE_SERVER'), @@ -48,14 +45,6 @@ export class AuthStrategy extends PassportStrategy(LoginStrategy, 'login') { ); if (numberOfBlocksBack) { loginStrategyOptions.numberOfBlocksBack = numberOfBlocksBack; - ipfsUrl: ipfsConfig.url, - }; - const numBlocksBack = configService.get('STRATEGY_NUM_BLOCKS_BACK'); - if (numBlocksBack) { - loginStrategyParams = { - ...loginStrategyParams, - ...{ numberOfBlocksBack: parseInt(numBlocksBack) }, - }; } super(...loginStrategyParams); } diff --git a/src/modules/did/did.module.ts b/src/modules/did/did.module.ts index bc4243e4..5554f7bf 100644 --- a/src/modules/did/did.module.ts +++ b/src/modules/did/did.module.ts @@ -1,4 +1,3 @@ -import { BullModule } from '@nestjs/bull'; import { Module } from '@nestjs/common'; import { HttpModule } from '@nestjs/axios'; import { TypeOrmModule } from '@nestjs/typeorm'; @@ -11,6 +10,9 @@ import { DIDResolver } from './did.resolver'; import { DIDService } from './did.service'; import { ethrReg } from '@ew-did-registry/did-ethr-resolver'; import { ConfigService } from '@nestjs/config'; +import { BullModule } from '@nestjs/bull'; +import { UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; +import { PIN_CLAIM_QUEUE_NAME } from '../ipfs/ipfs.types'; const RegistrySettingsProvider = { provide: 'RegistrySettings', @@ -37,7 +39,6 @@ const RegistrySettingsProvider = { providers: [ DIDService, DIDProcessor, - PinProcessor, DIDResolver, Provider, RegistrySettingsProvider, diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index f93c81bf..f82db63e 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -9,12 +9,12 @@ import { } from '@nestjs/bull'; import { ConfigService } from '@nestjs/config'; import { Job, Queue } from 'bull'; +import { PIN_CLAIM_JOB_NAME, PIN_CLAIM_QUEUE_NAME } from '../ipfs/ipfs.types'; import { Logger } from '../logger/logger.service'; +import { DIDDocumentEntity } from './did.entity'; import { DIDService } from './did.service'; import { ADD_DID_DOC_JOB_NAME, - PIN_CLAIM_JOB_NAME, - PIN_CLAIM_QUEUE_NAME, UPDATE_DID_DOC_JOB_NAME, UPDATE_DOCUMENT_QUEUE_NAME, } from './did.types'; @@ -55,7 +55,9 @@ export class DIDProcessor { public async processDIDDocumentAddition(job: Job) { const doc = await this.didService.addCachedDocument(job.data); - await this.pinQueue.add(PIN_CLAIM_JOB_NAME, doc); + for (const cid of doc.service.map((s) => s.serviceEndpoint)) { + await this.pinQueue.add(PIN_CLAIM_JOB_NAME, { cid }); + } } @Process(UPDATE_DID_DOC_JOB_NAME) diff --git a/src/modules/did/did.service.spec.ts b/src/modules/did/did.service.spec.ts index 63a4e1e6..f5e6899e 100644 --- a/src/modules/did/did.service.spec.ts +++ b/src/modules/did/did.service.spec.ts @@ -16,7 +16,7 @@ import { DIDService } from './did.service'; import { Logger } from '../logger/logger.service'; import { SentryTracingService } from '../sentry/sentry-tracing.service'; import { EthereumDIDRegistry } from '../../ethers/EthereumDIDRegistry'; -import { PIN_CLAIM_QUEUE_NAME, UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; +import { UPDATE_DOCUMENT_QUEUE_NAME } from './did.types'; import { IPFSService } from '../ipfs/ipfs.service'; const { formatBytes32String } = utils; @@ -100,10 +100,6 @@ describe('DidDocumentService', () => { provide: getQueueToken(UPDATE_DOCUMENT_QUEUE_NAME), useFactory: queueMockFactory, }, - { - provide: getQueueToken(PIN_CLAIM_QUEUE_NAME), - useFactory: queueMockFactory, - }, { provide: getRepositoryToken(DIDDocumentEntity), useFactory: repositoryMockFactory, diff --git a/src/modules/did/did.types.ts b/src/modules/did/did.types.ts index 63dae35a..e87493a3 100644 --- a/src/modules/did/did.types.ts +++ b/src/modules/did/did.types.ts @@ -110,6 +110,3 @@ export const getDIDFromAddress = (address: string) => export const ADD_DID_DOC_JOB_NAME = 'adding'; export const UPDATE_DID_DOC_JOB_NAME = 'refreshing'; export const UPDATE_DOCUMENT_QUEUE_NAME = 'updateDocumentQueue'; - -export const PIN_CLAIM_JOB_NAME = 'pinning'; -export const PIN_CLAIM_QUEUE_NAME = 'pinClaimQueue'; diff --git a/src/modules/did/pin.processor.ts b/src/modules/did/pin.processor.ts deleted file mode 100644 index 6d8f5742..00000000 --- a/src/modules/did/pin.processor.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { - OnQueueActive, - OnQueueError, - OnQueueFailed, - OnQueueStalled, - Process, - Processor, -} from '@nestjs/bull'; -import { ConfigService } from '@nestjs/config'; -import { Job } from 'bull'; -import { DidStore as DidStoreInfura } from 'didStoreInfura'; -import { DidStore as DidStoreCluster } from 'didStoreCluster'; -import { Logger } from '../logger/logger.service'; -import { DIDDocumentEntity } from './did.entity'; -import { PIN_CLAIM_JOB_NAME, PIN_CLAIM_QUEUE_NAME } from './did.types'; - -@Processor(PIN_CLAIM_QUEUE_NAME) -export class PinProcessor { - private didCluster: DidStoreCluster; - constructor( - private readonly logger: Logger, - private readonly configService: ConfigService, - private didInfura: DidStoreInfura - ) { - this.logger.setContext(PinProcessor.name); - const IPFS_CLUSTER_ROOT = this.configService.get('IPFS_CLUSTER_ROOT'); - const IPFS_CLUSTER_USER = this.configService.get('IPFS_CLUSTER_USER'); - const IPFS_CLUSTER_PASSWORD = this.configService.get( - 'IPFS_CLUSTER_PASSWORD' - ); - const Authorization = `Basic ${Buffer.from( - `${IPFS_CLUSTER_USER}:${IPFS_CLUSTER_PASSWORD}` - ).toString('base64')}`; - this.didCluster = new DidStoreCluster(IPFS_CLUSTER_ROOT, { - headers: { Authorization }, - }); - } - - @OnQueueError() - onError(error: Error) { - this.logger.error(`Error pinning claims ${error.message}`); - } - - @OnQueueActive() - onActive(job: Job) { - this.logger.debug( - `Starting ${job.name} claims of document ${ - (job.data as DIDDocumentEntity).id - }` - ); - } - - @OnQueueStalled() - onStalled(job: Job) { - this.logger.debug( - `Stalled ${job.name} claims of document ${ - (job.data as DIDDocumentEntity).id - }` - ); - } - - @OnQueueFailed() - onFailed(job: Job) { - this.logger.debug( - `Failed ${job.name} claims of document ${ - (job.data as DIDDocumentEntity).id - }` - ); - } - - /** - * This method migrates claims by retrieving from one DidStore and pinning to another - * It was implemented for EW migration from Infura to EW hosted IPFS - */ - @Process(PIN_CLAIM_JOB_NAME) - async pinClaims(doc: DIDDocumentEntity) { - for (const cid of doc.service.map((s) => s.serviceEndpoint)) { - try { - await this.didCluster.get(cid); - } catch (e) { - const token = await this.didInfura.get(cid); - await this.didCluster.save(token); - } - } - } -} diff --git a/src/modules/ipfs/ipfs.module.ts b/src/modules/ipfs/ipfs.module.ts index 2c7b8501..2b0c520e 100644 --- a/src/modules/ipfs/ipfs.module.ts +++ b/src/modules/ipfs/ipfs.module.ts @@ -1,4 +1,4 @@ -import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { DidStore as DidStoreGateway } from 'didStoreInfura'; import { DidStore as DidStoreCluster } from 'didStoreCluster'; import { Module, Global } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; @@ -7,37 +7,38 @@ import { IPFSService } from './ipfs.service'; import { IpfsClusterConfig, IPFSClusterConfigToken, - IpfsGatewayConfig, - IPFSGatewayConfigToken, - PINS_QUEUE, + IpfsInfuraConfig, + IPFSInfuraConfigToken, + PIN_CLAIM_QUEUE_NAME, } from './ipfs.types'; import { BullModule } from '@nestjs/bull'; -import { PinsProcessor } from './pins.processor'; +import { PinProcessor } from './pin.processor'; -const IPFSGatewayConfigProvider = { - provide: IPFSGatewayConfigToken, - useFactory: (config: ConfigService): IpfsGatewayConfig => { +const IPFSInfuraConfigProvider = { + provide: IPFSInfuraConfigToken, + useFactory: (config: ConfigService): IpfsInfuraConfig => { const IPFS_CLIENT_URL = config.get('IPFS_CLIENT_URL'); const IPFS_CLIENT_PROTO = config.get('IPFS_CLIENT_PROTO'); const IPFS_CLIENT_HOST = config.get('IPFS_CLIENT_HOST'); const IPFS_CLIENT_PORT = config.get('IPFS_CLIENT_PORT'); - const ipfsConfig: IpfsGatewayConfig = { + const ipfsConfig: IpfsInfuraConfig = { url: IPFS_CLIENT_URL, protocol: IPFS_CLIENT_PROTO, host: IPFS_CLIENT_HOST, port: parseInt(IPFS_CLIENT_PORT), }; - const IPFS_CLIENT_PASSWORD = config.get('IPFS_CLIENT_PASSWORD'); - const IPFS_CLIENT_USER = config.get('IPFS_CLIENT_USER'); + const IPFS_CLIENT_PROJECT_SECRET = config.get( + 'IPFS_CLIENT_PROJECT_SECRET' + ); + const IPFS_CLIENT_PROJECT_ID = config.get('IPFS_CLIENT_PROJECT_ID'); // https://community.infura.io/t/how-to-add-internet-content-from-a-url-using-ipfs-http-client/5188 - if (IPFS_CLIENT_USER && IPFS_CLIENT_PASSWORD) { + if (IPFS_CLIENT_PROJECT_ID && IPFS_CLIENT_PROJECT_SECRET) { ipfsConfig.headers = { Authorization: 'Basic ' + - Buffer.from(`${IPFS_CLIENT_USER}:${IPFS_CLIENT_PASSWORD}`).toString( - 'base64' - ), - // Authorization: 'Basic WHpOY1NoVGF1R1NTQkk6Y0dnMVZ2dld0dkI1QTF5UW84SE8=', + Buffer.from( + `${IPFS_CLIENT_PROJECT_ID}:${IPFS_CLIENT_PROJECT_SECRET}` + ).toString('base64'), }; } return ipfsConfig; @@ -48,7 +49,7 @@ const IPFSGatewayConfigProvider = { const IPFSClusterConfigProvider = { provide: IPFSClusterConfigToken, useFactory: (config: ConfigService): IpfsClusterConfig => { - const IPFS_CLUSTER_ROOT_URL = config.get('IPFS_CLUSTER_ROOT_URL'); + const IPFS_CLUSTER_ROOT = config.get('IPFS_CLUSTER_ROOT'); const IPFS_CLUSTER_USER = config.get('IPFS_CLUSTER_USER'); const IPFS_CLUSTER_PASSWORD = config.get('IPFS_CLUSTER_PASSWORD'); let ipfsConfig: IpfsClusterConfig; @@ -60,9 +61,9 @@ const IPFSClusterConfigProvider = { 'base64' ), }; - ipfsConfig = [IPFS_CLUSTER_ROOT_URL, headers]; + ipfsConfig = [IPFS_CLUSTER_ROOT, { headers }]; } else { - ipfsConfig = [IPFS_CLUSTER_ROOT_URL]; + ipfsConfig = [IPFS_CLUSTER_ROOT]; } return ipfsConfig; }, @@ -73,12 +74,12 @@ const IPFSClusterConfigProvider = { @Module({ imports: [ BullModule.registerQueue({ - name: PINS_QUEUE, + name: PIN_CLAIM_QUEUE_NAME, }), ], providers: [ IPFSClusterConfigProvider, - IPFSGatewayConfigProvider, + IPFSInfuraConfigProvider, IPFSService, { provide: DidStoreCluster, @@ -89,17 +90,17 @@ const IPFSClusterConfigProvider = { }, { provide: DidStoreGateway, - useFactory: (ipfsConfig: IpfsGatewayConfig) => { + useFactory: (ipfsConfig: IpfsInfuraConfig) => { return new DidStoreGateway(ipfsConfig); }, - inject: [{ token: IPFSGatewayConfigToken, optional: false }], + inject: [{ token: IPFSInfuraConfigToken, optional: false }], }, - PinsProcessor, + PinProcessor, ], controllers: [IPFSController], exports: [ IPFSClusterConfigProvider, - IPFSGatewayConfigProvider, + IPFSInfuraConfigProvider, IPFSService, DidStoreCluster, DidStoreGateway, diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index 78a9aa66..4080073b 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -1,41 +1,31 @@ import { DidStore as DidStoreCluster } from 'didStoreCluster'; -import { DidStore as DidStoreGateway } from 'didStoreGateway'; +import { DidStore as DidStoreGateway } from 'didStoreInfura'; import { HttpException, HttpStatus, Injectable, OnModuleDestroy, - OnModuleInit, } from '@nestjs/common'; import { CID } from 'multiformats/cid'; import { InjectQueue } from '@nestjs/bull'; import { Queue } from 'bull'; -import { inspect } from 'util'; -import { PINS_QUEUE, PIN_CLAIM } from './ipfs.types'; +import { PIN_CLAIM_QUEUE_NAME, PIN_CLAIM_JOB_NAME } from './ipfs.types'; import { Logger } from '../logger/logger.service'; @Injectable() -export class IPFSService implements OnModuleInit, OnModuleDestroy { +export class IPFSService implements OnModuleDestroy { constructor( private didStoreCluster: DidStoreCluster, - private didStoreGateway: DidStoreGateway, - @InjectQueue(PINS_QUEUE) private readonly pinsQueue: Queue, + private didStoreInfura: DidStoreGateway, + @InjectQueue(PIN_CLAIM_QUEUE_NAME) + private readonly pinsQueue: Queue, private readonly logger: Logger ) { this.logger.setContext(IPFSService.name); } - async onModuleInit() { - const jobsCount = await this.pinsQueue.getJobCounts(); - this.logger.info( - `Service endpoints pinning jobs statuses ${inspect(jobsCount, { - depth: 3, - colors: true, - })}` - ); - } - async onModuleDestroy() { + await this.pinsQueue.empty(); await this.pinsQueue.close(); } @@ -73,28 +63,22 @@ export class IPFSService implements OnModuleInit, OnModuleDestroy { */ public async get(cid: string): Promise { let claim: string; - if (await this.didStoreCluster.isPinned(cid)) { - this.logger.debug(`${cid} was pinned. Getting from cluster`); - claim = await this.didStoreCluster.get(cid); - } else { - this.logger.debug(`${cid} was not pinned. Getting from gateway`); + const getFromCluster = this.didStoreCluster.get(cid); + const getFromInfura = this.didStoreInfura.get(cid); + try { + claim = await Promise.race([getFromCluster, getFromInfura]); + } catch (e) { try { - claim = await this.didStoreGateway.get(cid); - await this.pinsQueue.add(PIN_CLAIM, JSON.stringify({ cid, claim })); + claim = await getFromInfura; } catch (e) { - // 504 is the expected response code when IPFS gateway is unable to provide content within time limit - // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#504-gateway-timeout - // In other words, this is the expected response code if the traversal of the DHT fails to find the content - if (e?.response?.status === 504 || e?.response?.status === 404) { - throw new HttpException( - `Claim ${cid} not found`, - HttpStatus.NOT_FOUND - ); - } else { - throw e; - } + // TODO: catch this in DidService + throw new HttpException(`Claim ${cid} not found`, HttpStatus.NOT_FOUND); } } + await this.pinsQueue.add( + PIN_CLAIM_JOB_NAME, + JSON.stringify({ cid, claim }) + ); return claim; } @@ -105,6 +89,13 @@ export class IPFSService implements OnModuleInit, OnModuleDestroy { * @returns CID of the persisted credential */ public async save(credential: string): Promise { - return this.didStoreCluster.save(credential); + try { + return this.didStoreCluster.save(credential); + } catch (_) { + this.logger.warn( + `Error saving ${credential} in cluster. Saving in Infura` + ); + return this.didStoreInfura.save(credential); + } } } diff --git a/src/modules/ipfs/ipfs.types.ts b/src/modules/ipfs/ipfs.types.ts index 02e038b7..c0015dba 100644 --- a/src/modules/ipfs/ipfs.types.ts +++ b/src/modules/ipfs/ipfs.types.ts @@ -3,7 +3,7 @@ import { DidStore as DidStoreCluster } from 'didStoreCluster'; export type IpfsClusterConfig = ConstructorParameters; // copied from https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client#createoptions, because ipfs-http-client isnt' typed -export type IpfsGatewayConfig = { +export type IpfsInfuraConfig = { url?: string; protocol?: string; host?: string; @@ -14,8 +14,8 @@ export type IpfsGatewayConfig = { }; }; -export const IPFSGatewayConfigToken = Symbol.for('IPFSGatewayConfigToken'); +export const IPFSInfuraConfigToken = Symbol.for('IPFSInfuraConfigToken'); export const IPFSClusterConfigToken = Symbol.for('IPFSClusterConfigToken'); -export const PIN_CLAIM = 'pinClaim'; -export const PINS_QUEUE = 'pins'; +export const PIN_CLAIM_JOB_NAME = 'pinning'; +export const PIN_CLAIM_QUEUE_NAME = 'pinClaimQueue'; diff --git a/src/modules/ipfs/pin.processor.ts b/src/modules/ipfs/pin.processor.ts new file mode 100644 index 00000000..98369876 --- /dev/null +++ b/src/modules/ipfs/pin.processor.ts @@ -0,0 +1,60 @@ +import { + OnQueueError, + OnQueueFailed, + OnQueueStalled, + Process, + Processor, +} from '@nestjs/bull'; +import { Job } from 'bull'; +import { DidStore as DidStoreInfura } from 'didStoreInfura'; +import { DidStore as DidStoreCluster } from 'didStoreCluster'; +import { Logger } from '../logger/logger.service'; +import { PIN_CLAIM_JOB_NAME, PIN_CLAIM_QUEUE_NAME } from './ipfs.types'; + +@Processor(PIN_CLAIM_QUEUE_NAME) +export class PinProcessor { + constructor( + private readonly logger: Logger, + private didStoreCluster: DidStoreCluster, + private didStoreInfura: DidStoreInfura + ) { + this.logger.setContext(PinProcessor.name); + } + + @OnQueueError() + onError(error: Error) { + this.logger.error(`Error pinning claims ${error.message}`); + } + + @OnQueueStalled() + onStalled(job: Job) { + this.logger.warn(`Stalled ${job.name} claim ${JSON.parse(job.data).cid}`); + } + + @OnQueueFailed() + onFailed(job: Job, err: Error) { + this.logger.error( + `Failed ${job.name} claim ${JSON.parse(job.data).cid}: ${err}` + ); + } + + /** + * This method migrates claims by retrieving from one DidStore and pinning to another + * It was implemented for EW migration from Infura to EW hosted IPFS + */ + @Process(PIN_CLAIM_JOB_NAME) + async pin(job: Job) { + const data = JSON.parse(job.data); + const cid = data.cid; + let claim = data.claim; + try { + await this.didStoreCluster.get(cid); + } catch (_) { + if (!claim) { + claim = await this.didStoreInfura.get(cid); + } + const clusterCid = await this.didStoreCluster.save(claim); + this.logger.debug(`CID ${cid} saved in cluster by CID ${clusterCid}`); + } + } +} diff --git a/src/modules/ipfs/pins.processor.ts b/src/modules/ipfs/pins.processor.ts deleted file mode 100644 index 46fdd53f..00000000 --- a/src/modules/ipfs/pins.processor.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { OnQueueError, Process, Processor } from '@nestjs/bull'; -import { Job } from 'bull'; -import { DidStore as DidStoreCluster } from 'didStoreCluster'; -import { Logger } from '../logger/logger.service'; -import { PINS_QUEUE, PIN_CLAIM } from './ipfs.types'; - -@Processor(PINS_QUEUE) -export class PinsProcessor { - constructor( - private readonly didStoreCluster: DidStoreCluster, - private readonly logger: Logger - ) { - this.logger.setContext(PinsProcessor.name); - } - - @OnQueueError() - onError(error: Error) { - this.logger.error(error); - } - - @Process(PIN_CLAIM) - public async pinClaim({ data }: Job) { - const { cid: cidGateway, claim } = JSON.parse(data); - this.logger.debug(`Pinning ${claim}`); - try { - const cidCluster = await this.didStoreCluster.save(claim); - await this.didStoreCluster.pin(cidGateway); - this.logger.debug(`${cidGateway} saved on cluster as ${cidCluster}`); - if (claim !== (await this.didStoreCluster.get(cidGateway))) { - throw new Error('Cluster content is not resolved by gateway CID'); - } - } catch (e) { - this.logger.error(e.message); - } - } -} diff --git a/src/scripts/truncate-db.ts b/src/scripts/truncate-db.ts new file mode 100644 index 00000000..d382c450 --- /dev/null +++ b/src/scripts/truncate-db.ts @@ -0,0 +1,31 @@ +import { config } from 'dotenv'; +import path from 'path'; +config(); +import { DataSource, DataSourceOptions } from 'typeorm'; + +truncateDatabase().then(() => console.log(`database truncated`)); + +async function truncateDatabase() { + const entities = path.join(__dirname, '../../', 'dist/**/*entity.js'); + const config: DataSourceOptions = { + type: 'postgres', + host: process.env.DB_HOST, + port: +process.env.DB_PORT, + username: process.env.DB_USERNAME, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + entities: [entities], + }; + const dataSource = new DataSource(config); + await dataSource.initialize(); + + const tableNames = dataSource.entityMetadatas.map( + (entity) => `"${entity.tableName}"` + ); + + await dataSource.query( + `TRUNCATE ${tableNames.join(',')} RESTART IDENTITY CASCADE` + ); + + await dataSource.destroy(); +} From b98dd19e52397f166c32175ca1a18f17e86d1f1a Mon Sep 17 00:00:00 2001 From: JGiter Date: Thu, 13 Jul 2023 09:36:58 +0300 Subject: [PATCH 04/24] refactor: rn ipfs proto --- .env.dev | 2 +- devops/dev/values.yaml | 1 + devops/prod/values.yaml | 3 ++- devops/sandbox/values.yaml | 1 + devops/staging/values.yaml | 1 + src/env-vars-validation-schema.ts | 4 ++-- src/modules/ipfs/ipfs.module.ts | 4 ++-- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.env.dev b/.env.dev index acfb4d59..b5dfc285 100644 --- a/.env.dev +++ b/.env.dev @@ -45,7 +45,7 @@ IPFS_CLUSTER_ROOT # Cluster can be deployed locally with e2e/setupIpfs IPFS_CLUSTER_USER # Not needed locally IPFS_CLUSTER_PASSWORD # Not needed locally IPFS_CLIENT_URL # Can be public gateway https://ipfs.github.io/public-gateway-checker/, Inura gateway or IPFS cluster. When cluster used as IPFS gateway it supports only GET -IPFS_CLIENT_PROTO # These can be +IPFS_CLIENT_PROTOCOL # These can be IPFS_CLIENT_HOST # used instead of IPFS_CLIENT_PORT # IPFS_CLIENT_URL IPFS_CLIENT_PROJECT_ID # Id of Infura project or user of cluster diff --git a/devops/dev/values.yaml b/devops/dev/values.yaml index f5dfb231..c8f99cc7 100644 --- a/devops/dev/values.yaml +++ b/devops/dev/values.yaml @@ -109,6 +109,7 @@ iam-cache-server-helm: NESTJS_PORT: 3000 DB_MAXIMUM_CONNECTION_POOL: 56 # https://github.com/energywebfoundation/iam-cache-server/blob/develop/README.md#connection-pooling SENTRY_ENV: dev + IPFS_CLIENT_PROTOCOL: https IPFS_CLIENT_PORT: 5001 IPFS_CLIENT_PROJECT_ID: 2DFOVbJGkFy289ekGv5BjgZngCt IPFS_CLIENT_HOST: ipfs.infura.io diff --git a/devops/prod/values.yaml b/devops/prod/values.yaml index 8b142b09..462aac4f 100644 --- a/devops/prod/values.yaml +++ b/devops/prod/values.yaml @@ -135,6 +135,7 @@ iam-cache-server-helm: LOGS_DIRECTORY: /app/logs NESTJS_PORT: 3000 SENTRY_ENV: prod + IPFS_CLIENT_PROTOCOL: https IPFS_CLIENT_PORT: 5001 IPFS_CLIENT_PROJECT_ID: 2DFVKQS7MfgeN9hdXrPG6HHH02O IPFS_CLIENT_HOST: ipfs.infura.io @@ -182,4 +183,4 @@ iam-cache-server-helm: ENS_URL: AgBoXgF+veIAYSQRVN5mq5Q189utr0bxHXjOVSzAu/cfEmBJyH/q0DdCMCjkKILypNTAcQaLqbsVixZy4qVl14vs6LPYeKyLsgZ9CXrz0dffeogHX88+PJYgcGnfVhhsjBwGvToBz9adgCPv030dVEwz+HYRFhIayz/Mnubs0CLzkAEDdQiOW6u7JBNNyw0q7CMPW1Z0HDroYVR/zNAZJHAmSpsmKSiiHnJw2LO0JMJ6YLkHRZRuNPR1Yu4D05l3vPk+1YNk7t2Eca05fTfiH6n3KlW69Zl6F48uGumpJM2ME4WI36lFclB+c5T22mOWi+lx3ZW9ypJei/pp/sjnTTAgnboKC5GPI6LAbi1/VaZzDcdbWDJ18dOHXuz0MkgTTcSsRBFM2edxzWwSTWe8JJqOTxZq84ljefv3p/7GAXZ/ZCEa1PBA1RDuJ22h1ibMvSO696RuqqYh64H9KHzcTDuFzbjqr0osbP9bovIWDPNcQFDX8vF4SZkbnTHvbWV/KK9k1i+SNKJyQ9FOBFcLKLGD7FI2G1VEKRtzzxuXWEp9OMa9qx7pabxmSKgzaSXhdYw0klNzguZx3enpC1tMv0p+eQ44qa0GA22q3C1oYjJqxrVFQyxnhgPd+WS03iH00kAraqJsR5rHYjcuoIXvbcYVnPm0Zs7/4Km5r0+nnwJTg5W3IDoevbX9mLejnd08e7YmyDqkJlPFL8zeGw8ondQ98aGvfsQan9w3ukBZDeWsgVX/K7Jmnh0= IPFS_CLUSTER_ROOT: AgChfcjVTz3VKph9hnbcLSLYvXG0dUkv+Zw4MrtUCzKCRKRDxtl1hJh0FEPw/eMlHHyiFdeoaJ6W6esxftt3yaT84muWYhXRNoCnzr5Bd+QyI0fVDphfxIAyw+j0SOGO/pIXAw5xHA9AJt4JhL3YJg1wlT7FUVI7keKfWWIofe0froVtUVG8rc0sL3D63ktfrjNUb/R/w2VcFLQmFF9JzqDFFPDDkD1LPiEe4xYI3Pcyh1APKH2gmNJsAcwoPzQ/QUVBaFNuMm2sLS24IAmZ6ghWehlRJtDtBSLGJ4SOJ6vxwAnICfFt1k/+VXuRQWgQcNzrAPFdhxyuymjyfVtLHsJojqqARqdmX9WWARIroi9kmP/Vdt2jqy1lZ5eTKedYYX13pDAITxW880PHrl4NlM1DWpXFrcvZs7X31zoMhdGDbcVI2K8LThG2V75swFT0GfnfPQLSsf6y5dvFOwVZGgU4Um0v48GeYuvb9x5RWHhVn0Y4dm1rcebDeiKojSjJtNQOUz2UV9HaOMBsWBgUg1Xd7uwRPUnycvTm8fe8JqQ46ewo6cyQ77/4c4kAj9hbBpSRgMpResIlZJbMu1MhR0+wajvtI2su30ZOl8FxJu02ORShu8RXUMq42lgPPCUw4Huy43qzxFIQg3r1UTcHuweBB4vZq/abDnx3/UMs96G6UqM86FWTKzmqhTmtqY7qGM6ZIUcoSGuKR5sRnkCO9j8ftIFMVn3pykbFUXYu87cwoIc6qvkOhjsemwU= IPFS_CLUSTER_USER: AgBUy6GBiFTMwz4Q1jjlEzy7lt4yfyMpYjhs3SeGA+X0Qo34ERmy+uxTlkraCK/uq7M5mcLsH609wa4ubEx2aV5jQCaVZ5rhnEW71KCRfkGUIcip1WbdVTvrtESADPUadFf7pEXm+OvJMQPNPMjQ4uyS57M/Zl16/oNltlsffLRanAgZohLMPXcLi4XC4ZUTSI1erHWMuL2pR9s6+edHEtd+7PEa7jbFdfZdeDPdnh0Xgf+3XXM6wXqykaaCSG/lCWIpiqrklSrKf+jt8HC2vJ/R+iK2RefXjH1LlR8JiwEzUaa6vbLBx0UVGAe9ncShWLXZtzKl+aqChqpiYHCCuD4h5bEKZK4JTpPsojQuoi8TutGTQPlislakzN+20DcAqdZ7soNmOVvhKyNwcLB5ycbK7wW5XB0k2MjsOxHBjLF28dFiroCTjskJgUxot1EPo8wSdssBlGxPYu9HsVG7jH0iw/RCDoepJdx47t74TPOpdaIVvBYYu6TJy4ohN1mED7c4kvXGEk1YOcZXCqXf/Grn3/rLUoAFUOrLRUFdOgXcz5FCbF50azES/Oo59LCCxHCNWx+xcJl3+QmCme+X7h6RdLDUU5TFlRoZrAoVtBQGrkQKVWZzuRYmjHm3ZVXJJfzPqzc+7pvQ1l1dIokG6NHzUJ2iIRv3qkagyH7BlQnoj0tOsSnWuReW1QHIboOWAL96qyfmadIxZJu94hEv/bIM - IPFS_CLUSTER_PASSWORD: AgCncDxtjXgBmdaMT+a7NN8PIZQaNH7PJ/C5RmGkcXwYUsS9JONjWqBny+7OEPM9Y82DaYGrsH3LwBrCG2wJ8ja5zelHfrkl4ue9HnplpbhnG9eigMRer0pFHbwdrqagGZNI5EOZXEQNZb3uy/BUFbvB+dWvEbYnNz4UYHU/3hQAAbuQ81VniTRmmSb6kbNN58mHVHCwbC1kjQsssQ1cVE5aTSLZFfxgb1zIvgrhZKKWmG0ygZDV78XMHZ8fCvBACcIlt9Suy9JdpJMZG8omvtYwF6wvGj7w1DiZwr67Pmt0i2959MgFA6nIYxo8mJbEaDaAnTYAheiNHWnTXxVXCQDECN5xNkd81tNfj0wMQa8CqgH0KbH/lJCFSyruKA4Up/7h7xhU/SG9nx8DyzV1g0phxkggHcXBeFGeupwHS6d5O80N9ThxVMLznrHgpyqdz4ZZORtGjvPoH+3ehLg22CbZasPhEPzkfT4wg0NBRNqf7YUqEz+WS41CQVHL/PqON39YzEkqy0MZhAdhSWKjSopdQBfZxji5DujbofX17qSMD3IWsTpF3IV3pRCZAqAmH77R93FxJWBtrJiUztGf4Po8fiYScusNiEORhFru0x1+avH8aHxbNvDfsPGN5NOAsoQ8AD6SAvPDwTacl8YydccAv71To+sNQYP+qgmKk4ir6Nn5sB/tAXDLzvfx9QHU+j6jgY48abU4q2ZKA2GKG6po/lu+jawugA== \ No newline at end of file + IPFS_CLUSTER_PASSWORD: AgCncDxtjXgBmdaMT+a7NN8PIZQaNH7PJ/C5RmGkcXwYUsS9JONjWqBny+7OEPM9Y82DaYGrsH3LwBrCG2wJ8ja5zelHfrkl4ue9HnplpbhnG9eigMRer0pFHbwdrqagGZNI5EOZXEQNZb3uy/BUFbvB+dWvEbYnNz4UYHU/3hQAAbuQ81VniTRmmSb6kbNN58mHVHCwbC1kjQsssQ1cVE5aTSLZFfxgb1zIvgrhZKKWmG0ygZDV78XMHZ8fCvBACcIlt9Suy9JdpJMZG8omvtYwF6wvGj7w1DiZwr67Pmt0i2959MgFA6nIYxo8mJbEaDaAnTYAheiNHWnTXxVXCQDECN5xNkd81tNfj0wMQa8CqgH0KbH/lJCFSyruKA4Up/7h7xhU/SG9nx8DyzV1g0phxkggHcXBeFGeupwHS6d5O80N9ThxVMLznrHgpyqdz4ZZORtGjvPoH+3ehLg22CbZasPhEPzkfT4wg0NBRNqf7YUqEz+WS41CQVHL/PqON39YzEkqy0MZhAdhSWKjSopdQBfZxji5DujbofX17qSMD3IWsTpF3IV3pRCZAqAmH77R93FxJWBtrJiUztGf4Po8fiYScusNiEORhFru0x1+avH8aHxbNvDfsPGN5NOAsoQ8AD6SAvPDwTacl8YydccAv71To+sNQYP+qgmKk4ir6Nn5sB/tAXDLzvfx9QHU+j6jgY48abU4q2ZKA2GKG6po/lu+jawugA== diff --git a/devops/sandbox/values.yaml b/devops/sandbox/values.yaml index be5a51a7..76f82e49 100644 --- a/devops/sandbox/values.yaml +++ b/devops/sandbox/values.yaml @@ -110,6 +110,7 @@ iam-cache-server-helm: NATS_CLIENTS_URL: iam-cache-server-nats.ics.svc.cluster.local:4222 LOGS_DIRECTORY: /app/logs NESTJS_PORT: 3000 + IPFS_CLIENT_PROTOCOL: https IPFS_CLIENT_PORT: 5001 IPFS_CLIENT_PROJECT_SECRET: 9506198a91243f03417aaed60e5eb654 IPFS_CLIENT_PROJECT_ID: 2DBFBP5zsjfImw1wBPn1Vx08Yd5 diff --git a/devops/staging/values.yaml b/devops/staging/values.yaml index 7352a408..dd1e0e58 100644 --- a/devops/staging/values.yaml +++ b/devops/staging/values.yaml @@ -138,6 +138,7 @@ iam-cache-server-helm: NESTJS_PORT: 3000 DB_MAXIMUM_CONNECTION_POOL: 150 # 1024^3 * 4 / 9531392 = 450 . 450/3 = 150. https://github.com/energywebfoundation/iam-cache-server/blob/develop/README.md#connection-pooling SENTRY_ENV: staging + IPFS_CLIENT_PROTOCOL: https IPFS_CLIENT_PORT: 5001 IPFS_CLIENT_PROJECT_ID: 2DFOoYU78LsICBTNhrwRGctWE7i IPFS_CLIENT_HOST: ipfs.infura.io diff --git a/src/env-vars-validation-schema.ts b/src/env-vars-validation-schema.ts index 7a0d2a55..2cecae5e 100644 --- a/src/env-vars-validation-schema.ts +++ b/src/env-vars-validation-schema.ts @@ -90,7 +90,7 @@ export const envVarsValidationSchema = Joi.object({ ASSETS_SYNC_ENABLED: Joi.boolean().required(), IPFS_CLIENT_URL: Joi.string().uri(), - IPFS_CLIENT_PROTO: Joi.string(), + IPFS_CLIENT_PROTOCOL: Joi.string(), IPFS_CLIENT_HOST: Joi.string().hostname(), IPFS_CLIENT_PORT: Joi.number().port(), IPFS_CLIENT_PROJECT_ID: Joi.string(), @@ -130,4 +130,4 @@ export const envVarsValidationSchema = Joi.object({ DISABLE_GET_DIDS_BY_ROLE: Joi.bool().default(false), }) .or('IPFS_CLIENT_URL', 'IPFS_CLIENT_HOST') - .with('IPFS_CLIENT_HOST', ['IPFS_CLIENT_PROTO', 'IPFS_CLIENT_PORT']); + .with('IPFS_CLIENT_HOST', ['IPFS_CLIENT_PROTOCOL', 'IPFS_CLIENT_PORT']); diff --git a/src/modules/ipfs/ipfs.module.ts b/src/modules/ipfs/ipfs.module.ts index 2b0c520e..105ab047 100644 --- a/src/modules/ipfs/ipfs.module.ts +++ b/src/modules/ipfs/ipfs.module.ts @@ -18,12 +18,12 @@ const IPFSInfuraConfigProvider = { provide: IPFSInfuraConfigToken, useFactory: (config: ConfigService): IpfsInfuraConfig => { const IPFS_CLIENT_URL = config.get('IPFS_CLIENT_URL'); - const IPFS_CLIENT_PROTO = config.get('IPFS_CLIENT_PROTO'); + const IPFS_CLIENT_PROTOCOL = config.get('IPFS_CLIENT_PROTOCOL'); const IPFS_CLIENT_HOST = config.get('IPFS_CLIENT_HOST'); const IPFS_CLIENT_PORT = config.get('IPFS_CLIENT_PORT'); const ipfsConfig: IpfsInfuraConfig = { url: IPFS_CLIENT_URL, - protocol: IPFS_CLIENT_PROTO, + protocol: IPFS_CLIENT_PROTOCOL, host: IPFS_CLIENT_HOST, port: parseInt(IPFS_CLIENT_PORT), }; From ddabc539a434d3f00d1ee55da6cd27ee3f09cf71 Mon Sep 17 00:00:00 2001 From: JGiter Date: Thu, 13 Jul 2023 10:09:35 +0300 Subject: [PATCH 05/24] chore: await for pinning in test --- e2e/ipfs/ipfs.testSuite.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts index cfe23f84..db0975ca 100644 --- a/e2e/ipfs/ipfs.testSuite.ts +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -58,6 +58,11 @@ export const ipfsModuleTestSuite = () => { .send(claimData) .expect(HttpStatus.CREATED); + const claimPinned = new Promise((resolve) => { + pinsQueue.on('completed', () => { + resolve(); + }); + }); const didStoreClusterGet = jest.spyOn(didStoreCluster, 'get'); jest .spyOn(didStoreInfura, 'get') @@ -68,7 +73,8 @@ export const ipfsModuleTestSuite = () => { .get(`/v1/ipfs/${cid}`) .set('Cookie', requester.cookies) .expect(HttpStatus.OK); - expect(didStoreClusterGet).toBeCalledTimes(2); // first in IpfsService.get, second in PinProcessor.pin + await claimPinned; + expect(didStoreClusterGet).toBeCalledTimes(2); // first in IpfsService.get, second in PinProcessor.pin to check that claim if claim is pinned expect(JSON.parse(stored)).toStrictEqual(claimData); }); From 468f58b37308db911bc5ec7b0add4a8d56a7085e Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 17 Jul 2023 08:20:58 +0300 Subject: [PATCH 06/24] refactor: resolve from ipfs with any --- src/modules/ipfs/ipfs.service.ts | 11 ++++------- tsconfig.json | 4 +++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index 4080073b..c05d30e7 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -66,15 +66,12 @@ export class IPFSService implements OnModuleDestroy { const getFromCluster = this.didStoreCluster.get(cid); const getFromInfura = this.didStoreInfura.get(cid); try { - claim = await Promise.race([getFromCluster, getFromInfura]); + claim = await Promise.any([getFromCluster, getFromInfura]); } catch (e) { - try { - claim = await getFromInfura; - } catch (e) { - // TODO: catch this in DidService - throw new HttpException(`Claim ${cid} not found`, HttpStatus.NOT_FOUND); - } + // TODO: catch this in DidService + throw new HttpException(`Claim ${cid} not found`, HttpStatus.NOT_FOUND); } + await this.pinsQueue.add( PIN_CLAIM_JOB_NAME, JSON.stringify({ cid, claim }) diff --git a/tsconfig.json b/tsconfig.json index f13cb716..64cec5ef 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,9 @@ "resolveJsonModule": true, "moduleResolution": "node", "esModuleInterop": true, - "lib": ["ES2020"], + "lib": [ + "ES2021" + ], "noUnusedLocals": true, "skipLibCheck": true } From ed565072eb0edaef4658718cd5432505e983c4c6 Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 17 Jul 2023 08:23:53 +0300 Subject: [PATCH 07/24] chore: organize gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d80db6ee..15eaf115 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ ormlogs.log /coverage /.nyc_output /e2e/setupIpfsCluster/ipfs0/ +!/e2e/setupIpfsCluster/docker-compose.yml # IDEs and editors /.idea @@ -41,7 +42,6 @@ ormlogs.log # Using either dev or prod docker-compose file docker-compose.yml -!/e2e/setupIpfsCluster/docker-compose.yml db_dumps private.pem public.pem From c603217aa4c3888b7f822f736c45f853885a181e Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 17 Jul 2023 08:33:39 +0300 Subject: [PATCH 08/24] chore: discard rm endlin in prod values From a18d3b838c51ac1e84a2ba6b20d8f645e52fa557 Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 17 Jul 2023 08:50:25 +0300 Subject: [PATCH 09/24] chore: set eol --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6313b56c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf From f8c08d06cf4414aa4107081434b753b2915352b9 Mon Sep 17 00:00:00 2001 From: Dmitry <44746858+JGiter@users.noreply.github.com> Date: Mon, 17 Jul 2023 09:15:52 +0300 Subject: [PATCH 10/24] docs: ipfs get Co-authored-by: John Henderson --- src/modules/ipfs/ipfs.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/ipfs/ipfs.controller.ts b/src/modules/ipfs/ipfs.controller.ts index fd2c953c..0e2c2f7c 100644 --- a/src/modules/ipfs/ipfs.controller.ts +++ b/src/modules/ipfs/ipfs.controller.ts @@ -22,8 +22,8 @@ export class IPFSController { @Get('/:cid') @ApiTags('IPFS') @ApiOperation({ - summary: 'Returns credential from IPFS Store', - description: 'Returns credential represented by service in DID document.', + summary: 'Returns content from IPFS Store', + description: 'Returns data identified by the provided content identifier (cid). This can be used, for example, to resolve credentials linked in the service endpoint of a DID document.', }) @ApiParam({ name: 'cid', type: 'string', required: true }) public async get(@Param('cid', CIDPipe) cid: CID): Promise { From 14149b14db880317c2ec36e174d234e226d1a1e4 Mon Sep 17 00:00:00 2001 From: Dmitry <44746858+JGiter@users.noreply.github.com> Date: Mon, 17 Jul 2023 09:16:59 +0300 Subject: [PATCH 11/24] docs: ipfs save Co-authored-by: John Henderson --- src/modules/ipfs/ipfs.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/ipfs/ipfs.controller.ts b/src/modules/ipfs/ipfs.controller.ts index 0e2c2f7c..474bec2b 100644 --- a/src/modules/ipfs/ipfs.controller.ts +++ b/src/modules/ipfs/ipfs.controller.ts @@ -37,8 +37,8 @@ export class IPFSController { description: 'Stringified credential', }) @ApiOperation({ - summary: 'Saves credential in IPFS', - description: 'Saves credential on IPFS and returns its CID', + summary: 'Saves content in IPFS', + description: 'Saves content in IPFS and returns its CID', }) public async save(@Body() credential: Record) { return this.ipfsService.save(JSON.stringify(credential)); From 930ab1c4d414f66cdc71b8ed9c6527d053612f4d Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 17 Jul 2023 09:30:06 +0300 Subject: [PATCH 12/24] refactor: post to ipfs as string --- .../modules_ipfs_ipfs_controller.IPFSController.md | 2 +- e2e/ipfs/ipfs.testSuite.ts | 2 +- src/modules/ipfs/ipfs.controller.ts | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md index f87d1d53..b468695b 100644 --- a/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md +++ b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md @@ -51,7 +51,7 @@ ___ | Name | Type | | :------ | :------ | -| `credential` | `Record`<`string`, `unknown`\> | +| `credential` | `string` | #### Returns diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts index db0975ca..e8bb3df0 100644 --- a/e2e/ipfs/ipfs.testSuite.ts +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -55,7 +55,7 @@ export const ipfsModuleTestSuite = () => { const { text: cid } = await request(app.getHttpServer()) .post(`/v1/ipfs/`) .set('Cookie', requester.cookies) - .send(claimData) + .send(JSON.stringify(claimData)) .expect(HttpStatus.CREATED); const claimPinned = new Promise((resolve) => { diff --git a/src/modules/ipfs/ipfs.controller.ts b/src/modules/ipfs/ipfs.controller.ts index 474bec2b..7143b094 100644 --- a/src/modules/ipfs/ipfs.controller.ts +++ b/src/modules/ipfs/ipfs.controller.ts @@ -23,7 +23,8 @@ export class IPFSController { @ApiTags('IPFS') @ApiOperation({ summary: 'Returns content from IPFS Store', - description: 'Returns data identified by the provided content identifier (cid). This can be used, for example, to resolve credentials linked in the service endpoint of a DID document.', + description: + 'Returns data identified by the provided content identifier (cid). This can be used, for example, to resolve credentials linked in the service endpoint of a DID document.', }) @ApiParam({ name: 'cid', type: 'string', required: true }) public async get(@Param('cid', CIDPipe) cid: CID): Promise { @@ -34,13 +35,13 @@ export class IPFSController { @ApiTags('IPFS') @ApiBody({ type: 'string', - description: 'Stringified credential', + description: 'Stringified content', }) @ApiOperation({ summary: 'Saves content in IPFS', description: 'Saves content in IPFS and returns its CID', }) - public async save(@Body() credential: Record) { - return this.ipfsService.save(JSON.stringify(credential)); + public async save(@Body() credential: string) { + return this.ipfsService.save(credential); } } From 729f1a2c8fa720c149de3546762d50a8324e444c Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 17 Jul 2023 10:53:46 +0300 Subject: [PATCH 13/24] fix: stringify ipfs objects --- .../classes/modules_ipfs_ipfs_controller.IPFSController.md | 2 +- e2e/ipfs/ipfs.testSuite.ts | 3 +-- src/modules/ipfs/ipfs.controller.ts | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md index b468695b..1b101c70 100644 --- a/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md +++ b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md @@ -51,7 +51,7 @@ ___ | Name | Type | | :------ | :------ | -| `credential` | `string` | +| `credential` | `string` \| `object` | #### Returns diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts index e8bb3df0..a625df09 100644 --- a/e2e/ipfs/ipfs.testSuite.ts +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -55,7 +55,7 @@ export const ipfsModuleTestSuite = () => { const { text: cid } = await request(app.getHttpServer()) .post(`/v1/ipfs/`) .set('Cookie', requester.cookies) - .send(JSON.stringify(claimData)) + .send(claimData) .expect(HttpStatus.CREATED); const claimPinned = new Promise((resolve) => { @@ -75,7 +75,6 @@ export const ipfsModuleTestSuite = () => { .expect(HttpStatus.OK); await claimPinned; expect(didStoreClusterGet).toBeCalledTimes(2); // first in IpfsService.get, second in PinProcessor.pin to check that claim if claim is pinned - expect(JSON.parse(stored)).toStrictEqual(claimData); }); diff --git a/src/modules/ipfs/ipfs.controller.ts b/src/modules/ipfs/ipfs.controller.ts index 7143b094..663b780f 100644 --- a/src/modules/ipfs/ipfs.controller.ts +++ b/src/modules/ipfs/ipfs.controller.ts @@ -41,7 +41,9 @@ export class IPFSController { summary: 'Saves content in IPFS', description: 'Saves content in IPFS and returns its CID', }) - public async save(@Body() credential: string) { - return this.ipfsService.save(credential); + public async save(@Body() credential: string | object) { + return this.ipfsService.save( + typeof credential === 'string' ? credential : JSON.stringify(credential) + ); } } From d700a673e0395be999eccc94428dc5e1ca9e69f5 Mon Sep 17 00:00:00 2001 From: JGiter Date: Tue, 18 Jul 2023 11:39:43 +0300 Subject: [PATCH 14/24] fix: save in Infura regardless of save in cluster --- src/modules/ipfs/ipfs.service.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index c05d30e7..48b1b562 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -86,13 +86,21 @@ export class IPFSService implements OnModuleDestroy { * @returns CID of the persisted credential */ public async save(credential: string): Promise { - try { - return this.didStoreCluster.save(credential); - } catch (_) { + const [clusterCID, infuraCID] = await Promise.allSettled([ + this.didStoreCluster.save(credential), + this.didStoreInfura.save(credential), + ]); + if (clusterCID.status === 'fulfilled') { this.logger.warn( `Error saving ${credential} in cluster. Saving in Infura` ); - return this.didStoreInfura.save(credential); + return clusterCID.value; + } else if (infuraCID.status === 'fulfilled') { + return infuraCID.value; + } else { + throw new Error( + `Error saving ${credential} in Infura: ${infuraCID.reason}` + ); } } } From 9355e229ee1e46ef8f68aa523a3fae2467288784 Mon Sep 17 00:00:00 2001 From: JGiter Date: Tue, 18 Jul 2023 11:58:52 +0300 Subject: [PATCH 15/24] fix: await for pin in test ipfs get --- e2e/ipfs/ipfs.testSuite.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/e2e/ipfs/ipfs.testSuite.ts b/e2e/ipfs/ipfs.testSuite.ts index a625df09..e0dd3cd2 100644 --- a/e2e/ipfs/ipfs.testSuite.ts +++ b/e2e/ipfs/ipfs.testSuite.ts @@ -99,7 +99,7 @@ export const ipfsModuleTestSuite = () => { const claim = JSON.stringify(notPinned); const cid = notPinnedCid; - const claimPinned = new Promise((resolve) => { + let claimPinned = new Promise((resolve) => { pinsQueue.on('completed', () => { resolve(); }); @@ -114,6 +114,12 @@ export const ipfsModuleTestSuite = () => { expect(didStoreClusterGet).toBeCalledTimes(2); expect(didStoreInfuraGet).toBeCalledTimes(1); + claimPinned = new Promise((resolve) => { + pinsQueue.on('completed', () => { + resolve(); + }); + }); + jest .spyOn(didStoreInfura, 'get') .mockImplementation( @@ -123,6 +129,8 @@ export const ipfsModuleTestSuite = () => { .get(`/v1/ipfs/${cid}`) .set('Cookie', requester.cookies); + await claimPinned; + expect(didStoreClusterGet).toBeCalledTimes(4); expect(didStoreInfuraGet).toBeCalledTimes(2); }); From 2aeec50d524d58615d35dc838d045783abda6588 Mon Sep 17 00:00:00 2001 From: JGiter Date: Wed, 19 Jul 2023 10:45:50 +0300 Subject: [PATCH 16/24] fix: log save in infura --- src/modules/ipfs/ipfs.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index 48b1b562..bd5e2b5e 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -91,11 +91,11 @@ export class IPFSService implements OnModuleDestroy { this.didStoreInfura.save(credential), ]); if (clusterCID.status === 'fulfilled') { + return clusterCID.value; + } else if (infuraCID.status === 'fulfilled') { this.logger.warn( `Error saving ${credential} in cluster. Saving in Infura` ); - return clusterCID.value; - } else if (infuraCID.status === 'fulfilled') { return infuraCID.value; } else { throw new Error( From 7566785853448cb9e467e42dc40a4452ea8b9cf8 Mon Sep 17 00:00:00 2001 From: JGiter Date: Wed, 19 Jul 2023 11:07:09 +0300 Subject: [PATCH 17/24] refactor: cancel pins queue destroy --- .../modules_ipfs_ipfs_service.IPFSService.md | 19 ------------------- src/modules/ipfs/ipfs.service.ts | 14 ++------------ 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md index ea1cebf3..d8395feb 100644 --- a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md +++ b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md @@ -2,10 +2,6 @@ [modules/ipfs/ipfs.service](../modules/modules_ipfs_ipfs_service.md).IPFSService -## Implements - -- `OnModuleDestroy` - ## Table of contents ### Constructors @@ -15,7 +11,6 @@ ### Methods - [get](modules_ipfs_ipfs_service.IPFSService.md#get) -- [onModuleDestroy](modules_ipfs_ipfs_service.IPFSService.md#onmoduledestroy) - [save](modules_ipfs_ipfs_service.IPFSService.md#save) - [isCID](modules_ipfs_ipfs_service.IPFSService.md#iscid) @@ -56,20 +51,6 @@ Stringified credential ___ -### onModuleDestroy - -▸ **onModuleDestroy**(): `Promise`<`void`\> - -#### Returns - -`Promise`<`void`\> - -#### Implementation of - -OnModuleDestroy.onModuleDestroy - -___ - ### save ▸ **save**(`credential`): `Promise`<`string`\> diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index bd5e2b5e..15be50fb 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -1,11 +1,6 @@ import { DidStore as DidStoreCluster } from 'didStoreCluster'; import { DidStore as DidStoreGateway } from 'didStoreInfura'; -import { - HttpException, - HttpStatus, - Injectable, - OnModuleDestroy, -} from '@nestjs/common'; +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { CID } from 'multiformats/cid'; import { InjectQueue } from '@nestjs/bull'; import { Queue } from 'bull'; @@ -13,7 +8,7 @@ import { PIN_CLAIM_QUEUE_NAME, PIN_CLAIM_JOB_NAME } from './ipfs.types'; import { Logger } from '../logger/logger.service'; @Injectable() -export class IPFSService implements OnModuleDestroy { +export class IPFSService { constructor( private didStoreCluster: DidStoreCluster, private didStoreInfura: DidStoreGateway, @@ -24,11 +19,6 @@ export class IPFSService implements OnModuleDestroy { this.logger.setContext(IPFSService.name); } - async onModuleDestroy() { - await this.pinsQueue.empty(); - await this.pinsQueue.close(); - } - /** * Check if given value is a valid IPFS CID. * From 57753319908ae5b40508c445235cd672c04226ae Mon Sep 17 00:00:00 2001 From: Dmitry <44746858+JGiter@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:13:47 +0300 Subject: [PATCH 18/24] docs: log error save in cluster Co-authored-by: John Henderson --- src/modules/ipfs/ipfs.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index 15be50fb..e9d4b699 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -84,7 +84,7 @@ export class IPFSService { return clusterCID.value; } else if (infuraCID.status === 'fulfilled') { this.logger.warn( - `Error saving ${credential} in cluster. Saving in Infura` + `Error saving ${credential} in cluster. Was saved to Infura as backup` ); return infuraCID.value; } else { From 40015bd45181e7dc054418a261ce0d53a61cd076 Mon Sep 17 00:00:00 2001 From: JGiter Date: Wed, 26 Jul 2023 12:19:03 +0300 Subject: [PATCH 19/24] fix: pin claims of updated doc --- src/modules/did/did.processor.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index f82db63e..aad4911a 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -69,6 +69,10 @@ export class DIDProcessor { doc = await this.didService.incrementalRefreshCachedDocument(job.data); } - await this.pinQueue.add(PIN_CLAIM_JOB_NAME, doc); + for (const service of doc.service) { + await this.pinQueue.add(PIN_CLAIM_JOB_NAME, { + cid: service.serviceEndpoint, + }); + } } } From f9a5f1349900d92aa0ca9165576edb52243e4a30 Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 31 Jul 2023 15:22:05 +0300 Subject: [PATCH 20/24] refactor: pin claims in promise.all --- src/modules/did/did.processor.ts | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index aad4911a..93bb1aab 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -55,9 +55,13 @@ export class DIDProcessor { public async processDIDDocumentAddition(job: Job) { const doc = await this.didService.addCachedDocument(job.data); - for (const cid of doc.service.map((s) => s.serviceEndpoint)) { - await this.pinQueue.add(PIN_CLAIM_JOB_NAME, { cid }); - } + await Promise.all( + doc.service.map(({ serviceEndpoint }) => { + this.pinQueue.add(PIN_CLAIM_JOB_NAME, { + cid: serviceEndpoint, + }); + }) + ); } @Process(UPDATE_DID_DOC_JOB_NAME) @@ -69,10 +73,12 @@ export class DIDProcessor { doc = await this.didService.incrementalRefreshCachedDocument(job.data); } - for (const service of doc.service) { - await this.pinQueue.add(PIN_CLAIM_JOB_NAME, { - cid: service.serviceEndpoint, - }); - } + await Promise.all( + doc.service.map(({ serviceEndpoint }) => { + this.pinQueue.add(PIN_CLAIM_JOB_NAME, { + cid: serviceEndpoint, + }); + }) + ); } } From 4a57c65acbe4f40c99271bc94616046ddcf4a9ad Mon Sep 17 00:00:00 2001 From: JGiter Date: Fri, 4 Aug 2023 11:34:23 +0300 Subject: [PATCH 21/24] chore: log waiting job --- .../modules_did_did_processor.DIDProcessor.md | 17 +++++++++++++++++ .../modules_ipfs_pin_processor.PinProcessor.md | 17 +++++++++++++++++ src/modules/did/did.processor.ts | 6 ++++++ src/modules/ipfs/pin.processor.ts | 6 ++++++ 4 files changed, 46 insertions(+) diff --git a/docs/api/classes/modules_did_did_processor.DIDProcessor.md b/docs/api/classes/modules_did_did_processor.DIDProcessor.md index 72c5bc04..1575ee54 100644 --- a/docs/api/classes/modules_did_did_processor.DIDProcessor.md +++ b/docs/api/classes/modules_did_did_processor.DIDProcessor.md @@ -10,6 +10,7 @@ ### Methods +- [OnQueueWaiting](modules_did_did_processor.DIDProcessor.md#onqueuewaiting) - [onActive](modules_did_did_processor.DIDProcessor.md#onactive) - [onError](modules_did_did_processor.DIDProcessor.md#onerror) - [onFailed](modules_did_did_processor.DIDProcessor.md#onfailed) @@ -34,6 +35,22 @@ ## Methods +### OnQueueWaiting + +▸ **OnQueueWaiting**(`job`): `Promise`<`void`\> + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `job` | `Job`<`any`\> | + +#### Returns + +`Promise`<`void`\> + +___ + ### onActive ▸ **onActive**(`job`): `void` diff --git a/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md b/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md index 08d85d6c..a40ea1fa 100644 --- a/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md +++ b/docs/api/classes/modules_ipfs_pin_processor.PinProcessor.md @@ -10,6 +10,7 @@ ### Methods +- [OnQueueWaiting](modules_ipfs_pin_processor.PinProcessor.md#onqueuewaiting) - [onError](modules_ipfs_pin_processor.PinProcessor.md#onerror) - [onFailed](modules_ipfs_pin_processor.PinProcessor.md#onfailed) - [onStalled](modules_ipfs_pin_processor.PinProcessor.md#onstalled) @@ -31,6 +32,22 @@ ## Methods +### OnQueueWaiting + +▸ **OnQueueWaiting**(`job`): `Promise`<`void`\> + +#### Parameters + +| Name | Type | +| :------ | :------ | +| `job` | `Job`<`any`\> | + +#### Returns + +`Promise`<`void`\> + +___ + ### onError ▸ **onError**(`error`): `void` diff --git a/src/modules/did/did.processor.ts b/src/modules/did/did.processor.ts index 93bb1aab..93ae15ac 100644 --- a/src/modules/did/did.processor.ts +++ b/src/modules/did/did.processor.ts @@ -4,6 +4,7 @@ import { OnQueueError, OnQueueFailed, OnQueueStalled, + OnQueueWaiting, Process, Processor, } from '@nestjs/bull'; @@ -51,6 +52,11 @@ export class DIDProcessor { this.logger.debug(`Failed ${job.name} document ${job.data}`); } + @OnQueueWaiting() + async OnQueueWaiting(job: Job) { + this.logger.debug(`Waiting ${job.name} document ${job.data}`); + } + @Process(ADD_DID_DOC_JOB_NAME) public async processDIDDocumentAddition(job: Job) { const doc = await this.didService.addCachedDocument(job.data); diff --git a/src/modules/ipfs/pin.processor.ts b/src/modules/ipfs/pin.processor.ts index 98369876..1012e787 100644 --- a/src/modules/ipfs/pin.processor.ts +++ b/src/modules/ipfs/pin.processor.ts @@ -2,6 +2,7 @@ import { OnQueueError, OnQueueFailed, OnQueueStalled, + OnQueueWaiting, Process, Processor, } from '@nestjs/bull'; @@ -31,6 +32,11 @@ export class PinProcessor { this.logger.warn(`Stalled ${job.name} claim ${JSON.parse(job.data).cid}`); } + @OnQueueWaiting() + async OnQueueWaiting(job: Job) { + this.logger.debug(`Waiting ${job.name} claim ${job.data}`); + } + @OnQueueFailed() onFailed(job: Job, err: Error) { this.logger.error( From f63eb1cbb942cbeeae16f402e7b47ffa10e68130 Mon Sep 17 00:00:00 2001 From: JGiter Date: Fri, 4 Aug 2023 11:42:51 +0300 Subject: [PATCH 22/24] fix: rm wait jobs on redis oom --- src/modules/did/did.service.ts | 23 +++++++++++++++++++++-- src/modules/ipfs/ipfs.service.ts | 21 +++++++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/modules/did/did.service.ts b/src/modules/did/did.service.ts index 64313fbc..981fa1cb 100644 --- a/src/modules/did/did.service.ts +++ b/src/modules/did/did.service.ts @@ -44,6 +44,7 @@ import { Provider } from '../../common/provider'; import { SentryTracingService } from '../sentry/sentry-tracing.service'; import { isVerifiableCredential } from '@ew-did-registry/credentials-interface'; import { IPFSService } from '../ipfs/ipfs.service'; +import { inspect } from 'util'; @Injectable() export class DIDService implements OnModuleInit, OnModuleDestroy { @@ -350,7 +351,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { // Only refreshing a DID that is already cached. // Otherwise, cache could grow too large with DID Docs that aren't relevant to Switchboard if (didDocEntity) { - await this.didQueue.add(UPDATE_DID_DOC_JOB_NAME, did); + await this.pinDocument(did); } }); } @@ -359,7 +360,7 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { this.logger.debug(`Beginning sync of DID Documents`); const cachedDIDs = await this.didRepository.find({ select: ['id'] }); cachedDIDs.forEach(async (did) => { - await this.didQueue.add(UPDATE_DID_DOC_JOB_NAME, did.id); + await this.pinDocument(did.id); }); } @@ -482,4 +483,22 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { }) ); } + + private async pinDocument(did: string): Promise { + try { + await this.didQueue.add(UPDATE_DID_DOC_JOB_NAME, did); + } catch (e) { + this.logger.warn( + `Error to add DID synchronization job for document ${did}: ${e}` + ); + const jobsCounts = await this.didQueue.getJobCounts(); + this.logger.debug(inspect(jobsCounts, { depth: 2, colors: true })); + if (/OOM/.test(String(e))) { + this.logger.warn( + `Redis exceeded memory limit. Removing waiting jobs from DID queue` + ); + await this.didQueue.empty(); + } + } + } } diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index e9d4b699..c3df2150 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -6,6 +6,7 @@ import { InjectQueue } from '@nestjs/bull'; import { Queue } from 'bull'; import { PIN_CLAIM_QUEUE_NAME, PIN_CLAIM_JOB_NAME } from './ipfs.types'; import { Logger } from '../logger/logger.service'; +import { inspect } from 'util'; @Injectable() export class IPFSService { @@ -62,10 +63,22 @@ export class IPFSService { throw new HttpException(`Claim ${cid} not found`, HttpStatus.NOT_FOUND); } - await this.pinsQueue.add( - PIN_CLAIM_JOB_NAME, - JSON.stringify({ cid, claim }) - ); + try { + await this.pinsQueue.add( + PIN_CLAIM_JOB_NAME, + JSON.stringify({ cid, claim }) + ); + } catch (e) { + this.logger.debug(`Error to add pin job for cid ${cid}: ${e}`); + const jobsCounts = await this.pinsQueue.getJobCounts(); + this.logger.debug(inspect(jobsCounts, { depth: 2, colors: true })); + if (/OOM/.test(String(e))) { + this.logger.warn( + `Redis exceeded memory limit. Removing waiting jobs from PIN queue` + ); + await this.pinsQueue.empty(); + } + } return claim; } From 0d2e978bdb232349eb2df46c043734a9422d9b1f Mon Sep 17 00:00:00 2001 From: JGiter Date: Fri, 4 Aug 2023 14:42:19 +0300 Subject: [PATCH 23/24] build: skip updating of didstoreinfura --- .github/renovate.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index e9e6e70d..85b030e4 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,5 +1,6 @@ { "extends": ["github>energywebfoundation/shared-configs"], "assignees": ["Harasz", "JGiter"], - "reviewers": ["Harasz", "jrhender", "JGiter"] + "reviewers": ["Harasz", "jrhender", "JGiter"], + "ignoreDeps": ["didStoreInfura"] } From 4ff6281fede97df25539e1bdf417663ebb7dcf3d Mon Sep 17 00:00:00 2001 From: JGiter Date: Mon, 7 Aug 2023 16:32:55 +0300 Subject: [PATCH 24/24] chore: cancel empty queue on error --- src/modules/did/did.service.ts | 6 ------ src/modules/ipfs/ipfs.service.ts | 6 ------ 2 files changed, 12 deletions(-) diff --git a/src/modules/did/did.service.ts b/src/modules/did/did.service.ts index 981fa1cb..0180e8b8 100644 --- a/src/modules/did/did.service.ts +++ b/src/modules/did/did.service.ts @@ -493,12 +493,6 @@ export class DIDService implements OnModuleInit, OnModuleDestroy { ); const jobsCounts = await this.didQueue.getJobCounts(); this.logger.debug(inspect(jobsCounts, { depth: 2, colors: true })); - if (/OOM/.test(String(e))) { - this.logger.warn( - `Redis exceeded memory limit. Removing waiting jobs from DID queue` - ); - await this.didQueue.empty(); - } } } } diff --git a/src/modules/ipfs/ipfs.service.ts b/src/modules/ipfs/ipfs.service.ts index c3df2150..50e64756 100644 --- a/src/modules/ipfs/ipfs.service.ts +++ b/src/modules/ipfs/ipfs.service.ts @@ -72,12 +72,6 @@ export class IPFSService { this.logger.debug(`Error to add pin job for cid ${cid}: ${e}`); const jobsCounts = await this.pinsQueue.getJobCounts(); this.logger.debug(inspect(jobsCounts, { depth: 2, colors: true })); - if (/OOM/.test(String(e))) { - this.logger.warn( - `Redis exceeded memory limit. Removing waiting jobs from PIN queue` - ); - await this.pinsQueue.empty(); - } } return claim; }