feat: witness car store

config/default.json

@@ -20,6 +20,13 @@
     "s3BucketName": "myS3Bucket",
     "s3Endpoint": ""
   },
+  "witnessStorage": {
+    "awsRegion": "us-east-1",
+    "dynamoDbTableName": "",
+    "dynamoDbEndpoint": "",
+    "dynamoDbTtl": "",
+    "mode": "inmemory"
+  },
   "ipfsConfig": {
     "url": "http://localhost:5001",
     "pubsubTopic": "/ceramic/testnet-clay",

config/env/dev.json

@@ -20,6 +20,13 @@
     "s3BucketName": "@@S3_BUCKET_NAME",
     "s3Endpoint": "@@S3_ENDPOINT"
   },
+  "witnessStorage": {
+    "awsRegion": "@@AWS_REGION",
+    "dynamoDbEndpoint": "@@DYNAMODB_ENDPOINT",
+    "dynamoDbTableName": "@@DYNAMODB_WITNESS_TABLE",
+    "dynamoDbTtl": "@@DYNAMODB_WITNESS_TTL",
+    "mode": "@@WITNESS_CAR_STORAGE_MODE"
+  },
   "ipfsConfig": {
     "url": "@@IPFS_API_URL",
     "pubsubTopic": "@@IPFS_PUBSUB_TOPIC",

config/env/prod.json

@@ -20,6 +20,13 @@
     "s3BucketName": "@@S3_BUCKET_NAME",
     "s3Endpoint": "@@S3_ENDPOINT"
   },
+  "witnessStorage": {
+    "awsRegion": "@@AWS_REGION",
+    "dynamoDbEndpoint": "@@DYNAMODB_ENDPOINT",
+    "dynamoDbTableName": "@@DYNAMODB_WITNESS_TABLE",
+    "dynamoDbTtl": "@@DYNAMODB_WITNESS_TTL",
+    "mode": "@@WITNESS_CAR_STORAGE_MODE"
+  },
   "ipfsConfig": {
     "url": "@@IPFS_API_URL",
     "pubsubTopic": "@@IPFS_PUBSUB_TOPIC",

config/env/test.json

@@ -9,6 +9,9 @@
     "mode": "s3",
     "s3BucketName": "ceramic-tnet-cas"
   },
+  "witnessStorage": {
+    "mode": "inmemory"
+  },
   "blockchain": {
     "selectedConnector": "ethereum",
     "connectors": {

package.json

@@ -47,6 +47,7 @@
     }
   },
   "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.574.0",
     "@aws-sdk/client-sqs": "^3.348.0",
     "@ceramicnetwork/anchor-utils": "^1.11.0-rc.0",
     "@ceramicnetwork/codecs": "^1.3.0-rc.0",

src/app.ts

@@ -43,7 +43,7 @@ import {
   ValidationSqsQueueService,
 } from './services/queue/sqs-queue-service.js'
 import { makeMerkleCarService, type IMerkleCarService } from './services/merkle-car-service.js'
-import { WitnessService } from './services/witness-service.js'
+import { makeWitnessService, type IWitnessService } from './services/witness-service.js'
 
 type DependenciesContext = {
   config: Config
@@ -66,7 +66,7 @@ type ProvidedContext = {
   requestService: RequestService
   merkleCarService: IMerkleCarService
   continualAnchoringScheduler: TaskSchedulerService
-  witnessService: WitnessService
+  witnessService: IWitnessService
 } & DependenciesContext
 
 /**
@@ -107,10 +107,10 @@ export class CeramicAnchorApp {
       .provideClass('anchorBatchQueueService', AnchorBatchSqsQueueService)
       .provideClass('validationQueueService', ValidationSqsQueueService)
       .provideFactory('merkleCarService', makeMerkleCarService)
+      .provideFactory('witnessService', makeWitnessService)
       .provideClass('anchorService', AnchorService)
       .provideClass('markReadyScheduler', TaskSchedulerService)
       .provideClass('healthcheckService', HealthcheckService)
-      .provideClass('witnessService', WitnessService)
       .provideClass('requestPresentationService', RequestPresentationService)
       .provideClass('anchorRequestParamsParser', AnchorRequestParamsParser)
       .provideClass('requestService', RequestService)
@@ -161,6 +161,9 @@ export class CeramicAnchorApp {
       await ipfsService.init()
     }
 
+    const witnessService = this.container.resolve('witnessService')
+    await witnessService.init()
+
     switch (this.mode) {
       case AppMode.SERVER:
         await this._startServer()

src/services/__tests__/anchor-service.test.ts

@@ -37,6 +37,7 @@ import { FakeFactory } from './fake-factory.util.js'
 import { FakeEthereumBlockchainService } from './fake-ethereum-blockchain-service.util.js'
 import { MockEventProducerService } from './mock-event-producer-service.util.js'
 import { type IMerkleCarService, makeMerkleCarService } from '../merkle-car-service.js'
+import { type IWitnessService, makeWitnessService } from '../witness-service.js'
 
 process.env['NODE_ENV'] = 'test'
 
@@ -68,6 +69,7 @@ type Context = {
   metadataService: IMetadataService
   metadataRepository: MetadataRepository
   merkleCarService: IMerkleCarService
+  witnessService: IWitnessService
   anchorBatchQueueService: MockQueueService<any>
   blockchainService: FakeEthereumBlockchainService
 }
@@ -117,6 +119,7 @@ describe('anchor service', () => {
       .provideClass('metadataService', MetadataService)
       .provideClass('anchorBatchQueueService', MockQueueService<AnchorBatchQMessage>)
       .provideFactory('merkleCarService', makeMerkleCarService)
+      .provideFactory('witnessService', makeWitnessService)
       .provideClass('anchorService', AnchorService)
 
     ipfsService = injector.resolve('ipfsService')

src/services/__tests__/request-presentation-service.test.ts

@@ -7,14 +7,14 @@ import type {
 } from '../../repositories/anchor-repository.type.js'
 import { generateRequest } from '../../__tests__/test-utils.js'
 import { InMemoryMerkleCarService } from '../merkle-car-service.js'
-import { WitnessService } from '../witness-service.js'
+import { InMemoryWitnessService } from '../witness-service.js'
 import { CID } from 'multiformats/cid'
 
 const anchorRepository = {
   findByRequest: jest.fn(),
 } as unknown as IAnchorRepository
 const merkleCarService = new InMemoryMerkleCarService()
-const witnessService = new WitnessService()
+const witnessService = new InMemoryWitnessService()
 
 const service = new RequestPresentationService(anchorRepository, merkleCarService, witnessService)
 

src/services/__tests__/witness-service.test.ts

@@ -1,7 +1,7 @@
 import { beforeAll, describe, expect, test } from '@jest/globals'
 import { RequestStatus } from '../../models/request.js'
 import { Transaction } from '../../models/transaction.js'
-import { verifyWitnessCAR, witnessCIDs, WitnessService } from '../witness-service.js'
+import { IWitnessService, makeWitnessService } from '../witness-service.js'
 import { FakeFactory } from './fake-factory.util.js'
 import { RequestRepository } from '../../repositories/request-repository.js'
 import { AnchorService } from '../anchor-service.js'
@@ -27,7 +27,6 @@ const MERKLE_DEPTH_LIMIT = 3
 const READY_RETRY_INTERVAL_MS = 1000
 const STREAM_LIMIT = Math.pow(2, MERKLE_DEPTH_LIMIT)
 const MIN_STREAM_COUNT = Math.floor(STREAM_LIMIT / 2)
-const witnessService = new WitnessService()
 const carFactory = new CARFactory()
 
 type Context = {
@@ -36,6 +35,7 @@ type Context = {
   anchorService: AnchorService
   requestRepository: RequestRepository
   metadataService: IMetadataService
+  witnessService: IWitnessService
 }
 
 let connection: Knex
@@ -44,6 +44,7 @@ let requestRepository: RequestRepository
 let anchorService: AnchorService
 let anchorRepository: AnchorRepository
 let ipfsService: IIpfsService
+let witnessService: IWitnessService
 let injector: Injector<Context>
 
 beforeAll(async () => {
@@ -68,12 +69,14 @@ beforeAll(async () => {
     .provideClass('metadataService', MetadataService)
     .provideClass('anchorBatchQueueService', AnchorBatchSqsQueueService)
     .provideFactory('merkleCarService', makeMerkleCarService)
+    .provideFactory('witnessService', makeWitnessService)
     .provideClass('anchorService', AnchorService)
 
   requestRepository = injector.resolve('requestRepository')
   anchorService = injector.resolve('anchorService')
   anchorRepository = injector.resolve('anchorRepository')
   ipfsService = injector.resolve('ipfsService')
+  witnessService = injector.resolve('witnessService')
   const metadataService = injector.resolve('metadataService')
   fake = new FakeFactory(ipfsService, metadataService, requestRepository)
 })
@@ -105,9 +108,9 @@ describe('create witness CAR', () => {
 
     for (const freshAnchor of anchors) {
       const anchor = await anchorRepository.findByRequestId(freshAnchor.requestId)
-      const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
+      const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
       // CIDs that are part of witness
-      const cidsInvolved = await all(witnessCIDs(witnessCAR)).then((cids) =>
+      const cidsInvolved = await all(witnessService.cids(witnessCAR)).then((cids) =>
         // `.slice` to remove the last element: a link to a `anchorCommit.prev`, which the CAR file does not have
         cids.map(String).slice(0, -1).sort()
       )
@@ -117,7 +120,7 @@ describe('create witness CAR', () => {
       )
       // Should be the same
       expect(cidsContained).toEqual(cidsInvolved)
-      const anchorCommitCID = verifyWitnessCAR(witnessCAR)
+      const anchorCommitCID = witnessService.verify(witnessCAR)
       expect(anchorCommitCID).toBeTruthy()
       expect(anchorCommitCID.equals(anchor.cid)).toBeTruthy()
     }
@@ -128,8 +131,8 @@ describe('create witness CAR', () => {
     const { anchors, merkleTree } = await createAnchors([request])
     for (const freshAnchor of anchors) {
       const anchor = await anchorRepository.findByRequestId(freshAnchor.requestId)
-      const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
-      const anchorCommitCID = verifyWitnessCAR(witnessCAR)
+      const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
+      const anchorCommitCID = witnessService.verify(witnessCAR)
       expect(anchorCommitCID).toBeTruthy()
       expect(anchorCommitCID.equals(anchor.cid)).toBeTruthy()
     }
@@ -141,21 +144,21 @@ describe('verify witness', () => {
     const request = await fake.request()
     const { anchors, merkleTree } = await createAnchors([request])
     const anchor = await anchorRepository.findByRequestId(anchors[0].requestId)
-    const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
+    const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
     const invalidCAR = new CAR(
       witnessCAR.version,
       [],
       witnessCAR.blocks,
       witnessCAR.codecs,
       witnessCAR.hashers
     )
-    expect(() => verifyWitnessCAR(invalidCAR)).toThrow(/No root found/)
+    expect(() => witnessService.verify(invalidCAR)).toThrow(/No root found/)
   })
   test('no anchor commit', async () => {
     const request = await fake.request()
     const { anchors, merkleTree } = await createAnchors([request])
     const anchor = await anchorRepository.findByRequestId(anchors[0].requestId)
-    const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
+    const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
     const invalidCAR = carFactory.build()
     const anchorCommitCID = witnessCAR.roots[0]
     for (const block of Array.from(witnessCAR.blocks)) {
@@ -164,13 +167,13 @@ describe('verify witness', () => {
       }
     }
     witnessCAR.roots.forEach((root) => invalidCAR.roots.push(root))
-    expect(() => verifyWitnessCAR(invalidCAR)).toThrow(/No anchor commit found/)
+    expect(() => witnessService.verify(invalidCAR)).toThrow(/No anchor commit found/)
   })
   test('no proof', async () => {
     const request = await fake.request()
     const { anchors, merkleTree } = await createAnchors([request])
     const anchor = await anchorRepository.findByRequestId(anchors[0].requestId)
-    const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
+    const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
     const invalidCAR = carFactory.build()
     const anchorCommitCID = witnessCAR.roots[0]
     const proofCID = witnessCAR.get(anchorCommitCID).proof
@@ -180,13 +183,13 @@ describe('verify witness', () => {
       }
     }
     witnessCAR.roots.forEach((root) => invalidCAR.roots.push(root))
-    expect(() => verifyWitnessCAR(invalidCAR)).toThrow(/No proof found/)
+    expect(() => witnessService.verify(invalidCAR)).toThrow(/No proof found/)
   })
   test('no Merkle root', async () => {
     const request = await fake.request()
     const { anchors, merkleTree } = await createAnchors([request])
     const anchor = await anchorRepository.findByRequestId(anchors[0].requestId)
-    const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
+    const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
     const invalidCAR = carFactory.build()
     const anchorCommitCID = witnessCAR.roots[0]
     const proofCID = witnessCAR.get(anchorCommitCID).proof
@@ -197,13 +200,13 @@ describe('verify witness', () => {
       }
     }
     witnessCAR.roots.forEach((root) => invalidCAR.roots.push(root))
-    expect(() => verifyWitnessCAR(invalidCAR)).toThrow(/No Merkle root found/)
+    expect(() => witnessService.verify(invalidCAR)).toThrow(/No Merkle root found/)
   })
   test('missing witness node', async () => {
     const requests = await fake.multipleRequests(4)
     const { anchors, merkleTree } = await createAnchors(requests)
     const anchor = await anchorRepository.findByRequestId(anchors[0].requestId)
-    const witnessCAR = witnessService.buildWitnessCAR(anchor.cid, merkleTree.car)
+    const witnessCAR = witnessService.build(anchor.cid, merkleTree.car)
     const invalidCAR = carFactory.build()
     const anchorCommitCID = witnessCAR.roots[0]
     const proofCID = witnessCAR.get(anchorCommitCID).proof
@@ -215,6 +218,6 @@ describe('verify witness', () => {
       }
     }
     witnessCAR.roots.forEach((root) => invalidCAR.roots.push(root))
-    expect(() => verifyWitnessCAR(invalidCAR)).toThrow(/Missing witness node/)
+    expect(() => witnessService.verify(invalidCAR)).toThrow(/Missing witness node/)
   })
 })

src/services/anchor-service.ts

@@ -34,6 +34,7 @@ import { create as createMultihash } from 'multiformats/hashes/digest'
 import { CAR } from 'cartonne'
 import { AbortOptions } from '@ceramicnetwork/common'
 import type { IMerkleCarService } from './merkle-car-service.js'
+import type { IWitnessService } from './witness-service.js'
 
 const CONTRACT_TX_TYPE = 'f(bytes32)'
 
@@ -144,6 +145,7 @@ export class AnchorService {
     'metadataService',
     'anchorBatchQueueService',
     'merkleCarService',
+    'witnessService',
   ] as const
 
   constructor(
@@ -157,7 +159,8 @@ export class AnchorService {
     private readonly eventProducerService: EventProducerService,
     private readonly metadataService: IMetadataService,
     private readonly anchorBatchQueueService: IQueueConsumerService<AnchorBatchQMessage>,
-    private readonly merkleCarService: IMerkleCarService
+    private readonly merkleCarService: IMerkleCarService,
+    private readonly witnessService: IWitnessService
   ) {
     this.merkleDepthLimit = config.merkleDepthLimit
     this.useSmartContractAnchors = config.useSmartContractAnchors
@@ -342,6 +345,9 @@ export class AnchorService {
       throw e
     }
 
+    logger.debug('Storing witness CAR files')
+    await this._storeWitnessCARs(anchors, merkleTree.car)
+
     // Update the database to record the successful anchors
     logger.debug('Persisting results to local database')
     const persistedAnchorsCount = await this._persistAnchorResult(anchors, candidates)
@@ -536,6 +542,26 @@ export class AnchorService {
     }
   }
 
+  /**
+   * For each anchored CID, create and store the corresponding witness CAR file.
+   * @private
+   * @param anchors Array of Anchor objects corresponding to anchored requests.
+   * @param merkleCAR Merkle CAR file.
+   */
+  async _storeWitnessCARs(anchors: FreshAnchor[], merkleCAR: CAR): Promise<void> {
+    for (const anchor of anchors) {
+      logger.debug(`Created witness CAR for anchor commit ${anchor.cid}`)
+      const witnessCAR = this.witnessService.build(anchor.cid, merkleCAR)
+      try {
+        await this.witnessService.store(anchor.cid, witnessCAR)
+      } catch (err) {
+        // An error storing the witness CAR file should not prevent the anchor from being considered successful
+        Metrics.count(METRIC_NAMES.WITNESS_CAR_STORAGE_FAILURE, 1)
+        logger.err(`Error storing witness CAR for anchor commit ${anchor.cid}: ${err}`)
+      }
+    }
+  }
+
   /**
    * Updates the anchor and request repositories in the local database with the results
    * of the anchor