Try adding the lambda to dev

PermanentOrg · Aug 28, 2024 · 8c449d7 · 8c449d7
1 parent 655993f
commit 8c449d7
Show file tree

Hide file tree

Showing 11 changed files with 336 additions and 54 deletions.
diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml
@@ -51,10 +51,31 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
         run: docker push $AM_CLEANUP_IMAGE_TAG
+  build_record_thumbnail_lambda:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG -f Dockerfile.record_thumbnail_attacher .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG
   deploy:
     needs:
       - build_api
       - build_am_cleanup
+      - build_record_thumbnail_lambda
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -73,6 +94,10 @@ jobs:
         run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Record Thumbnail Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
         id: init
         run: terraform init
@@ -83,11 +108,16 @@ jobs:
       # the -target option restricts terraform to just updating the dev deployment
       - name: Terraform Plan for API
         id: plan_api
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
       - name: Terraform Apply for API
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
       - name: Terraform Plan for Archivematica cleanup
         id: plan_am_cleanup
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
       - name: Terraform Apply for Archivematica cleanup
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+      - name: Terraform Plan for Record Thumbnail Lambda
+        id: plan_record_thumbnail_lambda
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=aws_lambda_function.record_thumbnail_lambda
+      - name: Terraform Apply for Record Thumbnail Lambda
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG" -target=aws_lambda_function.record_thumbnail_lambda
diff --git a/.github/workflows/full_test_deploy.yml b/.github/workflows/full_test_deploy.yml
@@ -48,10 +48,31 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
         run: docker push $AM_CLEANUP_IMAGE_TAG
+  build_record_thumbnail_lambda:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG --build-arg="AWS_RDS_CERT_BUNDLE=$AWS_RDS_CERT_BUNDLE" -f Dockerfile.record_thumbnail_attacher .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG
   deploy:
     needs:
       - build_api
       - build_am_cleanup
+      - build_record_thumbnail_lambda
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -70,6 +91,10 @@ jobs:
         run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Record Thumbnail Image Tag
+        run: echo "RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:record_thumbnail-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
         id: init
         run: terraform init
@@ -78,6 +103,6 @@ jobs:
         run: terraform validate -no-color
       - name: Terraform Plan
         id: plan
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG"
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG"
       - name: Terraform Apply
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG"
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -var="record_thumbnail_dev_lambda_image=$RECORD_THUMBNAIL_LAMBDA_IMAGE_TAG"
diff --git a/Dockerfile.record_thumbnail_attacher b/Dockerfile.record_thumbnail_attacher
@@ -14,14 +14,19 @@ RUN npm run build -ws
 
 
 FROM public.ecr.aws/lambda/nodejs:18 as final
+
+ARG AWS_RDS_CERT_BUNDLE
+
 WORKDIR ${LAMBDA_TASK_ROOT}
 
+RUN mkdir /etc/ca-certificates
+RUN echo -e $AWS_RDS_CERT_BUNDLE > /etc/ca-certificates/rds-us-west-2-ca-bundle.pem
+
 COPY --from=builder /usr/local/apps/stela/packages/record_thumbnail_attacher/dist ./packages/record_thumbnail_attacher/dist
 COPY --from=builder /usr/local/apps/stela/packages/record_thumbnail_attacher/package.json ./packages/record_thumbnail_attacher/package.json
 COPY --from=builder /usr/local/apps/stela/packages/logger/dist ./packages/logger/dist
 COPY --from=builder /usr/local/apps/stela/packages/logger/package.json ./packages/logger/package.json
 COPY --from=builder /usr/local/apps/stela/package.json ./package.json
-COPY --from=builder /usr/local/apps/stela/package-lock.json ./package-lock.json
 
 RUN npm install -g [email protected]
 RUN npm install --workspace @stela/record_thumbnail_attacher

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/api/package.json b/packages/api/package.json
@@ -53,7 +53,7 @@
     "mixpanel": "^0.18.0",
     "newrelic": "^11.22.0",
     "node-fetch": "^2.6.9",
-    "pg": "^8.5.1",
+    "pg": "8.5.1",
     "require-env-variable": "^3.1.2",
     "tinypg": "^7.0.0",
     "ts-md5": "^1.3.1",

diff --git a/packages/record_thumbnail_attacher/package.json b/packages/record_thumbnail_attacher/package.json
@@ -38,6 +38,7 @@
     "aws-cloudfront-sign": "^3.0.2",
     "dotenv": "^8.2.0",
     "require-env-variable": "^3.1.2",
+    "pg": "8.5.1",
     "tinypg": "^7.0.0"
   },
   "devDependencies": {

diff --git a/packages/record_thumbnail_attacher/src/index.test.ts b/packages/record_thumbnail_attacher/src/index.test.ts
@@ -48,11 +48,17 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_upload-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
-              },
-            },
+            Message: JSON.stringify({
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_upload-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
+                    },
+                  },
+                },
+              ],
+            }),
           }),
           attributes: {
             ApproximateReceiveCount: "1",
@@ -107,7 +113,19 @@ describe("handler", () => {
         {
           messageId: "1",
           receiptHandle: "1",
-          body: JSON.stringify({ s3: { object: { key: "1" } } }),
+          body: JSON.stringify({
+            Message: JSON.stringify({
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_upload-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
+                    },
+                  },
+                },
+              ],
+            }),
+          }),
           attributes: {
             ApproximateReceiveCount: "1",
             SentTimestamp: "1",
@@ -154,11 +172,17 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/2_load-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/thumbnails/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
-              },
-            },
+            Message: JSON.stringify({
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: "_Liam/access_copies/e38e/8582/b417/430c/953d/5c7e/8040/1ae2/thumbnails/2_load-cb45fa84-f0ea-4a9e-b1da-309e485a4f4a/object/710a1def-caf8-48f2-8eee-0848b4cfda10.jpg",
+                    },
+                  },
+                },
+              ],
+            }),
           }),
           attributes: {
             ApproximateReceiveCount: "1",
@@ -197,11 +221,17 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: testKey,
-              },
-            },
+            Message: JSON.stringify({
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: testKey,
+                    },
+                  },
+                },
+              ],
+            }),
           }),
           attributes: {
             ApproximateReceiveCount: "1",
@@ -288,11 +318,17 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: testKey,
-              },
-            },
+            Message: JSON.stringify({
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: testKey,
+                    },
+                  },
+                },
+              ],
+            }),
           }),
           attributes: {
             ApproximateReceiveCount: "1",
@@ -350,11 +386,17 @@ describe("handler", () => {
           messageId: "1",
           receiptHandle: "1",
           body: JSON.stringify({
-            s3: {
-              object: {
-                key: testKey,
-              },
-            },
+            Message: JSON.stringify({
+              Records: [
+                {
+                  s3: {
+                    object: {
+                      key: testKey,
+                    },
+                  },
+                },
+              ],
+            }),
           }),
           attributes: {
             ApproximateReceiveCount: "1",

diff --git a/packages/record_thumbnail_attacher/src/index.ts b/packages/record_thumbnail_attacher/src/index.ts
@@ -3,12 +3,27 @@ import { getSignedUrl } from "aws-cloudfront-sign";
 import * as Sentry from "@sentry/aws-serverless";
 import { logger } from "@stela/logger";
 import { db } from "./database";
-import { validateNewDisseminationPackageJpgEvent } from "./validators";
+import {
+  validateNewDisseminationPackageJpgEvent,
+  validateSqsMessage,
+} from "./validators";
 
 const getKeyFromS3Message = (message: SQSRecord): string => {
+  logger.info(`Received message: ${JSON.stringify(message)}`);
   const { body } = message;
   const parsedBody: unknown = JSON.parse(body);
-  if (!validateNewDisseminationPackageJpgEvent(parsedBody)) {
+  logger.info(`Parsed message: ${JSON.stringify(parsedBody)}`);
+  if (!validateSqsMessage(parsedBody)) {
+    logger.error(
+      `Invalid message body: ${JSON.stringify(validateSqsMessage.errors)}`
+    );
+    throw new Error("Invalid message body");
+  }
+  const parsedMessage: unknown = JSON.parse(parsedBody.Message);
+  if (
+    !validateNewDisseminationPackageJpgEvent(parsedMessage) ||
+    !parsedMessage.Records[0]
+  ) {
     logger.error(
       `Invalid message body: ${JSON.stringify(
         validateNewDisseminationPackageJpgEvent.errors
@@ -17,7 +32,7 @@ const getKeyFromS3Message = (message: SQSRecord): string => {
     throw new Error("Invalid message body");
   }
 
-  const { key } = parsedBody.s3.object;
+  const { key } = parsedMessage.Records[0].s3.object;
   return key;
 };