feat: custom s3 endpoint (#1166)

ceramicnetwork · Sep 21, 2023 · 057158e · 057158e
1 parent 4f861d4
commit 057158e
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 5 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -81,9 +81,26 @@ jobs:
         if: ${{ env.BRANCH == 'main' || env.BRANCH == 'release-candidate' || env.BRANCH == 'develop' }}
         run: echo "PUBLISH=true" >> $GITHUB_ENV
       -
-        name: Push Docker image
+        name: Push private Docker image
         if: ${{ env.PUBLISH == 'true' }}
         run: dagger do push -w "actions:push:\"${{ env.AWS_REGION }}\":\"${{ env.ENV_TAG }}\":\"${{ env.BRANCH }}\":\"${{ env.SHA }}\":\"${{ env.SHA_TAG }}\":_" -p ${{ env.DAGGER_PLAN }}
+      -
+        name: Login to public ECR
+        if: ${{ env.BRANCH == 'develop' }}
+        uses: docker/login-action@v2
+        with:
+          registry: public.ecr.aws
+          username: ${{ env.AWS_ACCESS_KEY_ID }}
+          password: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        env:
+          AWS_REGION: us-east-1
+      -
+        name: Push public Docker image
+        if: ${{ env.BRANCH == 'develop' }}
+        run: |
+          docker buildx build --load --target base -t 3box/cas .
+          docker tag 3box/cas:latest public.ecr.aws/r5b3e0r5/3box/cas:latest
+          docker push public.ecr.aws/r5b3e0r5/3box/cas:latest
       -
         name: Create deployment job
         if: ${{ env.PUBLISH == 'true' }}

diff --git a/config/default.json b/config/default.json
@@ -16,7 +16,8 @@
   "schedulerStopAfterNoOp": false,
   "carStorage": {
     "mode": "inmemory",
-    "s3BucketName": "myS3Bucket"
+    "s3BucketName": "myS3Bucket",
+    "s3Endpoint": ""
   },
   "ipfsConfig": {
     "url": "http://localhost:5001",

diff --git a/config/env/dev.json b/config/env/dev.json
@@ -16,7 +16,8 @@
   "schedulerStopAfterNoOp": "@@SCHEDULER_STOP_AFTER_NO_OP",
   "carStorage": {
     "mode": "@@MERKLE_CAR_STORAGE_MODE",
-    "s3BucketName": "@@S3_BUCKET_NAME"
+    "s3BucketName": "@@S3_BUCKET_NAME",
+    "s3Endpoint": "@@S3_ENDPOINT"
   },
   "ipfsConfig": {
     "url": "@@IPFS_API_URL",

diff --git a/config/env/prod.json b/config/env/prod.json
@@ -16,7 +16,8 @@
   "schedulerStopAfterNoOp": "@@SCHEDULER_STOP_AFTER_NO_OP",
   "carStorage": {
     "mode": "@@MERKLE_CAR_STORAGE_MODE",
-    "s3BucketName": "@@S3_BUCKET_NAME"
+    "s3BucketName": "@@S3_BUCKET_NAME",
+    "s3Endpoint": "@@S3_ENDPOINT"
   },
   "ipfsConfig": {
     "url": "@@IPFS_API_URL",

diff --git a/src/services/merkle-car-service.ts b/src/services/merkle-car-service.ts
@@ -54,11 +54,13 @@ const MAX_CACHE_SIZE = 100 // ~40MiB if 1 batch is 400KiB
 
 export class S3MerkleCarService implements IMerkleCarService {
   readonly s3StorePath: string
+  readonly s3Endpoint?: string
   private _s3store?: LevelUp.LevelUp
   private readonly cache: LRUCache<string, CAR>
 
   constructor(config: Config) {
     this.s3StorePath = config.carStorage.s3BucketName + S3_STORE_SUFFIX
+    this.s3Endpoint = config.carStorage.s3Endpoint ? config.carStorage.s3Endpoint : undefined
     this.cache = new LRUCache(MAX_CACHE_SIZE)
   }
 
@@ -68,7 +70,17 @@ export class S3MerkleCarService implements IMerkleCarService {
    */
   get s3store(): LevelUp.LevelUp {
     if (!this._s3store) {
-      this._s3store = new LevelUp(new S3LevelDOWN(this.s3StorePath, new AWSSDK.S3()))
+      const levelDown = this.s3Endpoint
+        ? new S3LevelDOWN(
+          this.s3StorePath,
+          new AWSSDK.S3({
+            endpoint: this.s3Endpoint,
+            s3ForcePathStyle: true,
+          })
+        )
+        : new S3LevelDOWN(this.s3StorePath)
+
+      this._s3store = new LevelUp(levelDown)
     }
     return this._s3store
   }