Merge pull request #662 from energywebfoundation/develop

Updating prod with develop
energywebfoundation · Sep 20, 2023 · 894b7eb · 894b7eb
2 parents 1d7cae0 + dca0e34
commit 894b7eb
Show file tree

Hide file tree

Showing 51 changed files with 1,116 additions and 298 deletions.
diff --git a/.env.dev b/.env.dev
@@ -41,13 +41,15 @@ ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS=21
 ASSETS_SYNC_ENABLED=true
 
 # IPFS
-IPFS_CLUSTER_ROOT=$IPFS_CLUSTER_ROOT
-IPFS_CLUSTER_USER=$IPFS_CLUSTER_USER
-IPFS_CLUSTER_PASSWORD=$IPFS_CLUSTER_PASSWORD
-IPFS_CLIENT_HOST=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PORT=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PROJECT_ID=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PROJECT_SECRET=<PUT_YOUR_VALUE_HERE>
+IPFS_CLUSTER_ROOT           # Cluster can be deployed locally with e2e/setupIpfsCluster/index.ts 
+IPFS_CLUSTER_USER           # Not needed locally
+IPFS_CLUSTER_PASSWORD       # Not needed locally
+IPFS_CLIENT_URL             # Can be public gateway https://ipfs.github.io/public-gateway-checker/, Inura gateway or IPFS cluster. When cluster used as IPFS gateway it supports only GET
+IPFS_CLIENT_PROTOCOL           # These can be
+IPFS_CLIENT_HOST            # used instead of 
+IPFS_CLIENT_PORT            # IPFS_CLIENT_URL
+IPFS_CLIENT_PROJECT_ID      # Id of Infura project or user of cluster
+IPFS_CLIENT_PROJECT_SECRET  # Secret of Infura project or cluster password
 
 # INTERVALS
 DIDDOC_SYNC_INTERVAL_IN_HOURS=1

diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+* text=auto eol=lf
diff --git a/.github/renovate.json b/.github/renovate.json
@@ -1,5 +1,6 @@
 {
   "extends": ["github>energywebfoundation/shared-configs"],
   "assignees": ["Harasz", "JGiter"],
-  "reviewers": ["Harasz", "jrhender", "JGiter"]
+  "reviewers": ["Harasz", "jrhender", "JGiter"],
+  "ignoreDeps": ["didStoreInfura"]
 }
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -46,7 +46,7 @@ env:
   ASSETS_SYNC_INTERVAL_IN_HOURS: ${{secrets.ASSETS_SYNC_INTERVAL_IN_HOURS}}
   ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS: ${{secrets.ASSETS_SYNC_HISTORY_INTERVAL_IN_HOURS}}
   ASSETS_SYNC_ENABLED: ${{secrets.ASSETS_SYNC_ENABLED}}
-  IPFS_URL: ${{secrets.IPFS_URL}}
+  IPFS_CLIENT_URL: ${{secrets.IPFS_URL}}
   NESTJS_PORT: ${{secrets.NESTJS_PORT}}
   STRATEGY_CACHE_SERVER: ${{secrets.STRATEGY_CACHE_SERVER}}
   STRATEGY_PRIVATE_KEY: ${{secrets.STRATEGY_PRIVATE_KEY}}

diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,6 @@
 /dist
 /node_modules
 /src/ethers
-.env
 
 # Logs
 logs
@@ -20,6 +19,8 @@ ormlogs.log
 # Tests
 /coverage
 /.nyc_output
+/e2e/setupIpfsCluster/ipfs0/
+!/e2e/setupIpfsCluster/docker-compose.yml
 
 # IDEs and editors
 /.idea

diff --git a/README.md b/README.md
@@ -19,15 +19,7 @@ $ cp .env.dev .env
 $ cp docker-compose.dev.yml docker-compose.yml
 ```
 
-Set the following values in your `.env`:
-
-```text
-ENS_URL=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_HOST=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PORT=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PROJECT_ID=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PROJECT_SECRET=<PUT_YOUR_VALUE_HERE>
-```
+Fill in configuration values in your `.env`. For reference look at `.env.dev`
 
 ### Production
 ```bash

diff --git a/devops/dev/values.yaml b/devops/dev/values.yaml
@@ -109,6 +109,7 @@ iam-cache-server-helm:
     NESTJS_PORT: 3000
     DB_MAXIMUM_CONNECTION_POOL: 56 # https://github.com/energywebfoundation/iam-cache-server/blob/develop/README.md#connection-pooling
     SENTRY_ENV: dev
+    IPFS_CLIENT_PROTOCOL: https
     IPFS_CLIENT_PORT: 5001
     IPFS_CLIENT_PROJECT_ID: 2DFOVbJGkFy289ekGv5BjgZngCt
     IPFS_CLIENT_HOST: ipfs.infura.io

diff --git a/devops/prod/values.yaml b/devops/prod/values.yaml
@@ -139,6 +139,7 @@ iam-cache-server-helm:
     LOGS_DIRECTORY: /app/logs
     NESTJS_PORT: 3000
     SENTRY_ENV: prod
+    IPFS_CLIENT_PROTOCOL: https
     IPFS_CLIENT_PORT: 5001
     IPFS_CLIENT_PROJECT_ID: 2DFVKQS7MfgeN9hdXrPG6HHH02O
     IPFS_CLIENT_HOST: ipfs.infura.io
@@ -186,4 +187,4 @@ iam-cache-server-helm:
       ENS_URL: AgBoXgF+veIAYSQRVN5mq5Q189utr0bxHXjOVSzAu/cfEmBJyH/q0DdCMCjkKILypNTAcQaLqbsVixZy4qVl14vs6LPYeKyLsgZ9CXrz0dffeogHX88+PJYgcGnfVhhsjBwGvToBz9adgCPv030dVEwz+HYRFhIayz/Mnubs0CLzkAEDdQiOW6u7JBNNyw0q7CMPW1Z0HDroYVR/zNAZJHAmSpsmKSiiHnJw2LO0JMJ6YLkHRZRuNPR1Yu4D05l3vPk+1YNk7t2Eca05fTfiH6n3KlW69Zl6F48uGumpJM2ME4WI36lFclB+c5T22mOWi+lx3ZW9ypJei/pp/sjnTTAgnboKC5GPI6LAbi1/VaZzDcdbWDJ18dOHXuz0MkgTTcSsRBFM2edxzWwSTWe8JJqOTxZq84ljefv3p/7GAXZ/ZCEa1PBA1RDuJ22h1ibMvSO696RuqqYh64H9KHzcTDuFzbjqr0osbP9bovIWDPNcQFDX8vF4SZkbnTHvbWV/KK9k1i+SNKJyQ9FOBFcLKLGD7FI2G1VEKRtzzxuXWEp9OMa9qx7pabxmSKgzaSXhdYw0klNzguZx3enpC1tMv0p+eQ44qa0GA22q3C1oYjJqxrVFQyxnhgPd+WS03iH00kAraqJsR5rHYjcuoIXvbcYVnPm0Zs7/4Km5r0+nnwJTg5W3IDoevbX9mLejnd08e7YmyDqkJlPFL8zeGw8ondQ98aGvfsQan9w3ukBZDeWsgVX/K7Jmnh0=
       IPFS_CLUSTER_ROOT: AgChfcjVTz3VKph9hnbcLSLYvXG0dUkv+Zw4MrtUCzKCRKRDxtl1hJh0FEPw/eMlHHyiFdeoaJ6W6esxftt3yaT84muWYhXRNoCnzr5Bd+QyI0fVDphfxIAyw+j0SOGO/pIXAw5xHA9AJt4JhL3YJg1wlT7FUVI7keKfWWIofe0froVtUVG8rc0sL3D63ktfrjNUb/R/w2VcFLQmFF9JzqDFFPDDkD1LPiEe4xYI3Pcyh1APKH2gmNJsAcwoPzQ/QUVBaFNuMm2sLS24IAmZ6ghWehlRJtDtBSLGJ4SOJ6vxwAnICfFt1k/+VXuRQWgQcNzrAPFdhxyuymjyfVtLHsJojqqARqdmX9WWARIroi9kmP/Vdt2jqy1lZ5eTKedYYX13pDAITxW880PHrl4NlM1DWpXFrcvZs7X31zoMhdGDbcVI2K8LThG2V75swFT0GfnfPQLSsf6y5dvFOwVZGgU4Um0v48GeYuvb9x5RWHhVn0Y4dm1rcebDeiKojSjJtNQOUz2UV9HaOMBsWBgUg1Xd7uwRPUnycvTm8fe8JqQ46ewo6cyQ77/4c4kAj9hbBpSRgMpResIlZJbMu1MhR0+wajvtI2su30ZOl8FxJu02ORShu8RXUMq42lgPPCUw4Huy43qzxFIQg3r1UTcHuweBB4vZq/abDnx3/UMs96G6UqM86FWTKzmqhTmtqY7qGM6ZIUcoSGuKR5sRnkCO9j8ftIFMVn3pykbFUXYu87cwoIc6qvkOhjsemwU=
       IPFS_CLUSTER_USER: AgBUy6GBiFTMwz4Q1jjlEzy7lt4yfyMpYjhs3SeGA+X0Qo34ERmy+uxTlkraCK/uq7M5mcLsH609wa4ubEx2aV5jQCaVZ5rhnEW71KCRfkGUIcip1WbdVTvrtESADPUadFf7pEXm+OvJMQPNPMjQ4uyS57M/Zl16/oNltlsffLRanAgZohLMPXcLi4XC4ZUTSI1erHWMuL2pR9s6+edHEtd+7PEa7jbFdfZdeDPdnh0Xgf+3XXM6wXqykaaCSG/lCWIpiqrklSrKf+jt8HC2vJ/R+iK2RefXjH1LlR8JiwEzUaa6vbLBx0UVGAe9ncShWLXZtzKl+aqChqpiYHCCuD4h5bEKZK4JTpPsojQuoi8TutGTQPlislakzN+20DcAqdZ7soNmOVvhKyNwcLB5ycbK7wW5XB0k2MjsOxHBjLF28dFiroCTjskJgUxot1EPo8wSdssBlGxPYu9HsVG7jH0iw/RCDoepJdx47t74TPOpdaIVvBYYu6TJy4ohN1mED7c4kvXGEk1YOcZXCqXf/Grn3/rLUoAFUOrLRUFdOgXcz5FCbF50azES/Oo59LCCxHCNWx+xcJl3+QmCme+X7h6RdLDUU5TFlRoZrAoVtBQGrkQKVWZzuRYmjHm3ZVXJJfzPqzc+7pvQ1l1dIokG6NHzUJ2iIRv3qkagyH7BlQnoj0tOsSnWuReW1QHIboOWAL96qyfmadIxZJu94hEv/bIM
-      IPFS_CLUSTER_PASSWORD: AgCncDxtjXgBmdaMT+a7NN8PIZQaNH7PJ/C5RmGkcXwYUsS9JONjWqBny+7OEPM9Y82DaYGrsH3LwBrCG2wJ8ja5zelHfrkl4ue9HnplpbhnG9eigMRer0pFHbwdrqagGZNI5EOZXEQNZb3uy/BUFbvB+dWvEbYnNz4UYHU/3hQAAbuQ81VniTRmmSb6kbNN58mHVHCwbC1kjQsssQ1cVE5aTSLZFfxgb1zIvgrhZKKWmG0ygZDV78XMHZ8fCvBACcIlt9Suy9JdpJMZG8omvtYwF6wvGj7w1DiZwr67Pmt0i2959MgFA6nIYxo8mJbEaDaAnTYAheiNHWnTXxVXCQDECN5xNkd81tNfj0wMQa8CqgH0KbH/lJCFSyruKA4Up/7h7xhU/SG9nx8DyzV1g0phxkggHcXBeFGeupwHS6d5O80N9ThxVMLznrHgpyqdz4ZZORtGjvPoH+3ehLg22CbZasPhEPzkfT4wg0NBRNqf7YUqEz+WS41CQVHL/PqON39YzEkqy0MZhAdhSWKjSopdQBfZxji5DujbofX17qSMD3IWsTpF3IV3pRCZAqAmH77R93FxJWBtrJiUztGf4Po8fiYScusNiEORhFru0x1+avH8aHxbNvDfsPGN5NOAsoQ8AD6SAvPDwTacl8YydccAv71To+sNQYP+qgmKk4ir6Nn5sB/tAXDLzvfx9QHU+j6jgY48abU4q2ZKA2GKG6po/lu+jawugA==
+      IPFS_CLUSTER_PASSWORD: AgCncDxtjXgBmdaMT+a7NN8PIZQaNH7PJ/C5RmGkcXwYUsS9JONjWqBny+7OEPM9Y82DaYGrsH3LwBrCG2wJ8ja5zelHfrkl4ue9HnplpbhnG9eigMRer0pFHbwdrqagGZNI5EOZXEQNZb3uy/BUFbvB+dWvEbYnNz4UYHU/3hQAAbuQ81VniTRmmSb6kbNN58mHVHCwbC1kjQsssQ1cVE5aTSLZFfxgb1zIvgrhZKKWmG0ygZDV78XMHZ8fCvBACcIlt9Suy9JdpJMZG8omvtYwF6wvGj7w1DiZwr67Pmt0i2959MgFA6nIYxo8mJbEaDaAnTYAheiNHWnTXxVXCQDECN5xNkd81tNfj0wMQa8CqgH0KbH/lJCFSyruKA4Up/7h7xhU/SG9nx8DyzV1g0phxkggHcXBeFGeupwHS6d5O80N9ThxVMLznrHgpyqdz4ZZORtGjvPoH+3ehLg22CbZasPhEPzkfT4wg0NBRNqf7YUqEz+WS41CQVHL/PqON39YzEkqy0MZhAdhSWKjSopdQBfZxji5DujbofX17qSMD3IWsTpF3IV3pRCZAqAmH77R93FxJWBtrJiUztGf4Po8fiYScusNiEORhFru0x1+avH8aHxbNvDfsPGN5NOAsoQ8AD6SAvPDwTacl8YydccAv71To+sNQYP+qgmKk4ir6Nn5sB/tAXDLzvfx9QHU+j6jgY48abU4q2ZKA2GKG6po/lu+jawugA==
diff --git a/devops/sandbox/values.yaml b/devops/sandbox/values.yaml
@@ -110,6 +110,7 @@ iam-cache-server-helm:
     NATS_CLIENTS_URL: iam-cache-server-nats.ics.svc.cluster.local:4222
     LOGS_DIRECTORY: /app/logs
     NESTJS_PORT: 3000
+    IPFS_CLIENT_PROTOCOL: https
     IPFS_CLIENT_PORT: 5001
     IPFS_CLIENT_PROJECT_SECRET: 9506198a91243f03417aaed60e5eb654
     IPFS_CLIENT_PROJECT_ID: 2DBFBP5zsjfImw1wBPn1Vx08Yd5

diff --git a/devops/staging/values.yaml b/devops/staging/values.yaml
@@ -138,6 +138,7 @@ iam-cache-server-helm:
     NESTJS_PORT: 3000
     DB_MAXIMUM_CONNECTION_POOL: 150 # 1024^3 * 4 / 9531392 = 450 . 450/3 = 150. https://github.com/energywebfoundation/iam-cache-server/blob/develop/README.md#connection-pooling
     SENTRY_ENV: staging
+    IPFS_CLIENT_PROTOCOL: https
     IPFS_CLIENT_PORT: 5001
     IPFS_CLIENT_PROJECT_ID: 2DFOoYU78LsICBTNhrwRGctWE7i
     IPFS_CLIENT_HOST: ipfs.infura.io

diff --git a/docs/api/README.md b/docs/api/README.md
@@ -19,15 +19,7 @@ $ cp .env.dev .env
 $ cp docker-compose.dev.yml docker-compose.yml
 ```
 
-Set the following values in your `.env`:
-
-```text
-ENS_URL=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_HOST=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PORT=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PROJECT_ID=<PUT_YOUR_VALUE_HERE>
-IPFS_CLIENT_PROJECT_SECRET=<PUT_YOUR_VALUE_HERE>
-```
+Fill in configuration values in your `.env`. For reference look at `.env.dev`
 
 ### Production
 ```bash

diff --git a/docs/api/classes/common_cid_pipe.CIDPipe.md b/docs/api/classes/common_cid_pipe.CIDPipe.md
@@ -0,0 +1,43 @@
+# Class: CIDPipe
+
+[common/cid.pipe](../modules/common_cid_pipe.md).CIDPipe
+
+## Implements
+
+- `PipeTransform`
+
+## Table of contents
+
+### Constructors
+
+- [constructor](common_cid_pipe.CIDPipe.md#constructor)
+
+### Methods
+
+- [transform](common_cid_pipe.CIDPipe.md#transform)
+
+## Constructors
+
+### constructor
+
+• **new CIDPipe**()
+
+## Methods
+
+### transform
+
+▸ **transform**(`cid`): `CID`
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `cid` | `string` |
+
+#### Returns
+
+`CID`
+
+#### Implementation of
+
+PipeTransform.transform
diff --git a/docs/api/classes/modules_did_did_processor.DIDProcessor.md b/docs/api/classes/modules_did_did_processor.DIDProcessor.md
@@ -10,6 +10,7 @@
 
 ### Methods
 
+- [OnQueueWaiting](modules_did_did_processor.DIDProcessor.md#onqueuewaiting)
 - [onActive](modules_did_did_processor.DIDProcessor.md#onactive)
 - [onError](modules_did_did_processor.DIDProcessor.md#onerror)
 - [onFailed](modules_did_did_processor.DIDProcessor.md#onfailed)
@@ -34,6 +35,22 @@
 
 ## Methods
 
+### OnQueueWaiting
+
+▸ **OnQueueWaiting**(`job`): `Promise`<`void`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `job` | `Job`<`any`\> |
+
+#### Returns
+
+`Promise`<`void`\>
+
+___
+
 ### onActive
 
 ▸ **onActive**(`job`): `void`

diff --git a/docs/api/classes/modules_did_did_service.DIDService.md b/docs/api/classes/modules_did_did_service.DIDService.md
@@ -28,7 +28,7 @@
 
 ### constructor
 
-• **new DIDService**(`config`, `schedulerRegistry`, `httpService`, `didQueue`, `logger`, `didRepository`, `provider`, `sentryTracingService`, `registrySettings`, `didStore`)
+• **new DIDService**(`config`, `schedulerRegistry`, `httpService`, `didQueue`, `logger`, `didRepository`, `provider`, `sentryTracingService`, `registrySettings`, `ipfsService`)
 
 #### Parameters
 
@@ -43,7 +43,7 @@
 | `provider` | [`Provider`](common_provider.Provider.md) |
 | `sentryTracingService` | [`SentryTracingService`](modules_sentry_sentry_tracing_service.SentryTracingService.md) |
 | `registrySettings` | `RegistrySettings` |
-| `didStore` | `DidStore` |
+| `ipfsService` | [`IPFSService`](modules_ipfs_ipfs_service.IPFSService.md) |
 
 ## Methods
 

diff --git a/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md b/docs/api/classes/modules_ipfs_ipfs_controller.IPFSController.md
@@ -0,0 +1,58 @@
+# Class: IPFSController
+
+[modules/ipfs/ipfs.controller](../modules/modules_ipfs_ipfs_controller.md).IPFSController
+
+## Table of contents
+
+### Constructors
+
+- [constructor](modules_ipfs_ipfs_controller.IPFSController.md#constructor)
+
+### Methods
+
+- [get](modules_ipfs_ipfs_controller.IPFSController.md#get)
+- [save](modules_ipfs_ipfs_controller.IPFSController.md#save)
+
+## Constructors
+
+### constructor
+
+• **new IPFSController**(`ipfsService`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `ipfsService` | [`IPFSService`](modules_ipfs_ipfs_service.IPFSService.md) |
+
+## Methods
+
+### get
+
+▸ **get**(`cid`): `Promise`<`string`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `cid` | `CID` |
+
+#### Returns
+
+`Promise`<`string`\>
+
+___
+
+### save
+
+▸ **save**(`credential`): `Promise`<`string`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `credential` | `string` \| `object` |
+
+#### Returns
+
+`Promise`<`string`\>
diff --git a/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md b/docs/api/classes/modules_ipfs_ipfs_service.IPFSService.md
@@ -10,16 +10,67 @@
 
 ### Methods
 
+- [get](modules_ipfs_ipfs_service.IPFSService.md#get)
+- [save](modules_ipfs_ipfs_service.IPFSService.md#save)
 - [isCID](modules_ipfs_ipfs_service.IPFSService.md#iscid)
 
 ## Constructors
 
 ### constructor
 
-• **new IPFSService**()
+• **new IPFSService**(`didStoreCluster`, `didStoreInfura`, `pinsQueue`, `logger`)
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `didStoreCluster` | `DidStore` |
+| `didStoreInfura` | `DidStore` |
+| `pinsQueue` | `Queue`<`string`\> |
+| `logger` | [`Logger`](modules_logger_logger_service.Logger.md) |
 
 ## Methods
 
+### get
+
+▸ **get**(`cid`): `Promise`<`string`\>
+
+Get claim from cluster. If claim isn't found tries to get from gateway
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `cid` | `string` | Content identifier. |
+
+#### Returns
+
+`Promise`<`string`\>
+
+Stringified credential
+
+___
+
+### save
+
+▸ **save**(`credential`): `Promise`<`string`\>
+
+Saves credential on cluster
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `credential` | `string` | Credential being persisted |
+
+#### Returns
+
+`Promise`<`string`\>
+
+CID of the persisted credential
+
+___
+
 ### isCID
 
 ▸ `Static` **isCID**(`hash`): `boolean`