Try adding the lambda to dev

PermanentOrg · Aug 27, 2024 · 5cca98d · 5cca98d
1 parent 655993f
commit 5cca98d
Show file tree

Hide file tree

Showing 7 changed files with 301 additions and 42 deletions.
diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml
@@ -51,10 +51,31 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
         run: docker push $AM_CLEANUP_IMAGE_TAG
+  build_record_thumbnail_lambda:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG -f Dockerfile.record_thumbnail_attacher .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG
   deploy:
     needs:
       - build_api
       - build_am_cleanup
+      - build_record_thumbnail_lambda
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -73,6 +94,10 @@ jobs:
         run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Record Thumbnail Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
         id: init
         run: terraform init
@@ -83,11 +108,16 @@ jobs:
       # the -target option restricts terraform to just updating the dev deployment
       - name: Terraform Plan for API
         id: plan_api
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
       - name: Terraform Apply for API
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
       - name: Terraform Plan for Archivematica cleanup
         id: plan_am_cleanup
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
       - name: Terraform Apply for Archivematica cleanup
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+      - name: Terraform Plan for Record Thumbnail Lambda
+        id: plan_record_thumbnail_lambda
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=aws_lambda_function.record_thumbnail_lambda
+      - name: Terraform Apply for Record Thumbnail Lambda
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=aws_lambda_function.record_thumbnail_lambda
diff --git a/.github/workflows/full_test_deploy.yml b/.github/workflows/full_test_deploy.yml
@@ -48,10 +48,31 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
         run: docker push $AM_CLEANUP_IMAGE_TAG
+  build_record_thumbnail_lambda:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG -f Dockerfile.record_thumbnail_attacher .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG
   deploy:
     needs:
       - build_api
       - build_am_cleanup
+      - build_record_thumbnail_lambda
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -70,6 +91,10 @@ jobs:
         run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Record Thumbnail Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
         id: init
         run: terraform init
@@ -78,6 +103,6 @@ jobs:
         run: terraform validate -no-color
       - name: Terraform Plan
         id: plan
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG"
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG"
       - name: Terraform Apply
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG"
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG"
diff --git a/packages/record_thumbnail_attacher/src/index.test.ts b/packages/record_thumbnail_attacher/src/index.test.ts
@@ -48,10 +48,16 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_upload-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
-              },
+            Message: {
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_upload-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
+                    },
+                  },
+                },
+              ],
             },
           }),
           attributes: {
@@ -107,7 +113,19 @@ describe("handler", () => {
         {
           messageId: "1",
           receiptHandle: "1",
-          body: JSON.stringify({ s3: { object: { key: "1" } } }),
+          body: JSON.stringify({
+            Message: {
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_upload-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
+                    },
+                  },
+                },
+              ],
+            },
+          }),
           attributes: {
             ApproximateReceiveCount: "1",
             SentTimestamp: "1",
@@ -154,10 +172,16 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_load-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/thumbnails/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
-              },
+            Message: {
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/thumbnails/2_load-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
+                    },
+                  },
+                },
+              ],
             },
           }),
           attributes: {
@@ -197,10 +221,16 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: testKey,
-              },
+            Message: {
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: testKey,
+                    },
+                  },
+                },
+              ],
             },
           }),
           attributes: {
@@ -288,10 +318,16 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: testKey,
-              },
+            Message: {
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: testKey,
+                    },
+                  },
+                },
+              ],
             },
           }),
           attributes: {
@@ -350,10 +386,16 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: testKey,
-              },
+            Message: {
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: testKey,
+                    },
+                  },
+                },
+              ],
             },
           }),
           attributes: {

diff --git a/packages/record_thumbnail_attacher/src/index.ts b/packages/record_thumbnail_attacher/src/index.ts
@@ -6,9 +6,13 @@ import { db } from "./database";
 import { validateNewDisseminationPackageJpgEvent } from "./validators";
 
 const getKeyFromS3Message = (message: SQSRecord): string => {
+  logger.info(`Received message: ${JSON.stringify(message)}`);
   const { body } = message;
   const parsedBody: unknown = JSON.parse(body);
-  if (!validateNewDisseminationPackageJpgEvent(parsedBody)) {
+  if (
+    !validateNewDisseminationPackageJpgEvent(parsedBody) ||
+    !parsedBody.Message.Records[0]
+  ) {
     logger.error(
       `Invalid message body: ${JSON.stringify(
         validateNewDisseminationPackageJpgEvent.errors
@@ -17,7 +21,7 @@ const getKeyFromS3Message = (message: SQSRecord): string => {
     throw new Error("Invalid message body");
   }
 
-  const { key } = parsedBody.s3.object;
+  const { key } = parsedBody.Message.Records[0].s3.object;
   return key;
 };
 

diff --git a/packages/record_thumbnail_attacher/src/validators.ts b/packages/record_thumbnail_attacher/src/validators.ts
@@ -4,34 +4,55 @@ import type { JSONSchemaType } from "ajv";
 const ajv = new Ajv({ coerceTypes: true });
 
 interface NewDisseminationPackageJpgEvent {
-  s3: {
-    object: {
-      key: string;
-    };
+  Message: {
+    Records: {
+      s3: {
+        object: {
+          key: string;
+        };
+      };
+    }[];
   };
 }
 
 const newDisseminationPackageJpgEventSchema: JSONSchemaType<NewDisseminationPackageJpgEvent> =
   {
     type: "object",
     properties: {
-      s3: {
+      Message: {
         type: "object",
         properties: {
-          object: {
-            type: "object",
-            properties: {
-              key: { type: "string" },
+          Records: {
+            type: "array",
+            items: {
+              type: "object",
+              properties: {
+                s3: {
+                  type: "object",
+                  properties: {
+                    object: {
+                      type: "object",
+                      properties: {
+                        key: { type: "string" },
+                      },
+                      required: ["key"],
+                      additionalProperties: true,
+                    },
+                  },
+                  required: ["object"],
+                  additionalProperties: true,
+                },
+              },
+              required: ["s3"],
+              additionalProperties: true,
             },
-            required: ["key"],
-            additionalProperties: true,
           },
         },
-        required: ["object"],
+        required: ["Records"],
         additionalProperties: true,
       },
     },
-    required: ["s3"],
+    required: ["Message"],
     additionalProperties: true,
   };