IaC for load custom csv lambda

Author: williamputraintan

lib/workload/stateless/stacks/metadata-manager/README.md

@@ -61,7 +61,7 @@ on the model of the record.
 
 ## How things work
 
-### How Syncing The Data Works
+### How Tracking Sheet Syncing Works
 
 In the near future, we might introduce different ways to load data into the application. For the time being, we are
 loading data
@@ -100,6 +100,18 @@ Some important notes of the sync:
 
 Please refer to the [tracking-sheet-service](proc/service/tracking_sheet_srv.py) implementation.
 
+### Loading from external csv
+
+The Metadata Manager has the capability to import metadata from an external CSV file. This CSV file should follow the 
+same mapping structure as specified in the tracking sync process. The loading operation utilizes a presigned URL, which 
+is subsequently used to load the data into the Metadata Manager. Not all header should be present in the CSV file, but 
+the required fields are:
+
+- `library_id`
+- `subject_id`
+
+To trigger this operation, trigger from the lambda specified in `./deploy/README.md`.
+
 ### Audit Data
 
 The application is configured with [django-simple-history](https://django-simple-history.readthedocs.io/en/latest/)

lib/workload/stateless/stacks/metadata-manager/deploy/README.md

@@ -58,3 +58,35 @@ aws lambda invoke \
   --cli-binary-format raw-in-base64-out \
   res.json
 ```
+
+### CustomCsvLambda
+
+- Load tracking sheet data from csv presigned url
+
+To manually trigger the sync, the lambda ARN is stored in the SSM Parameter Store named
+`/orcabus/metadata-manager/load-custom-csv-lambda-arn`.
+
+To query in a local terminal
+
+```sh
+load_custom_csv_lambda_arn=$(aws ssm get-parameter --name '/orcabus/metadata-manager/load-custom-csv-lambda-arn' --with-decryption | jq -r .Parameter.Value)
+```
+
+The lambda handler will accept a json which only accepts a single key `url` which is the presigned url of the csv file.
+
+```json
+{
+  "url": "https://example.com/csv"
+}
+```
+
+Invoking lambda cmd:
+
+```sh
+aws lambda invoke \
+  --function-name load_custom_csv_lambda_arn \
+  --invocation-type Event \
+  --payload '{ "url": "https://the.url.csv" }' \
+  --cli-binary-format raw-in-base64-out \
+  res.json
+```

lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-load-custom-csv/index.ts

@@ -0,0 +1,54 @@
+import path from 'path';
+import { Construct } from 'constructs';
+import { Duration } from 'aws-cdk-lib';
+import { PythonFunction } from '@aws-cdk/aws-lambda-python-alpha';
+import { ISecret } from 'aws-cdk-lib/aws-secretsmanager';
+import { StringParameter } from 'aws-cdk-lib/aws-ssm';
+import {
+  DockerImageFunction,
+  DockerImageFunctionProps,
+  DockerImageCode,
+} from 'aws-cdk-lib/aws-lambda';
+
+type LambdaProps = {
+  /**
+   * The basic common lambda properties that it should inherit from
+   */
+  basicLambdaConfig: Partial<DockerImageFunctionProps>;
+  /**
+   * The secret for the db connection where the lambda will need access to
+   */
+  dbConnectionSecret: ISecret;
+};
+
+export class LambdaLoadCustomCSVConstruct extends Construct {
+  private readonly lambda: PythonFunction;
+
+  constructor(scope: Construct, id: string, lambdaProps: LambdaProps) {
+    super(scope, id);
+
+    this.lambda = new DockerImageFunction(this, 'LoadCustomCSVLambda', {
+      environment: {
+        ...lambdaProps.basicLambdaConfig.environment,
+      },
+      securityGroups: lambdaProps.basicLambdaConfig.securityGroups,
+      vpc: lambdaProps.basicLambdaConfig.vpc,
+      vpcSubnets: lambdaProps.basicLambdaConfig.vpcSubnets,
+      architecture: lambdaProps.basicLambdaConfig.architecture,
+      code: DockerImageCode.fromImageAsset(path.join(__dirname, '../../../'), {
+        file: 'deploy/construct/lambda-load-custom-csv/lambda.Dockerfile',
+      }),
+      timeout: Duration.minutes(15),
+      memorySize: 4096,
+    });
+
+    lambdaProps.dbConnectionSecret.grantRead(this.lambda);
+
+    // We need to store this lambda ARN somewhere so that we could refer when need to load this manually
+    const ssmParameter = new StringParameter(this, 'LoadCustomCSVLambdaArnParameterStore', {
+      parameterName: '/orcabus/metadata-manager/load-custom-csv-lambda-arn',
+      description: 'The ARN of the lambda that load metadata from a presigned URL CSV file',
+      stringValue: this.lambda.functionArn,
+    });
+  }
+}

lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-load-custom-csv/lambda.Dockerfile

@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.12
+
+WORKDIR ${LAMBDA_TASK_ROOT}
+
+# COPY all files
+COPY . .
+
+# Install the specified packages
+RUN pip install -r deps/requirements-full.txt
+
+# Specify handler
+CMD [ "handler.load_custom_metadata_csv.handler" ]