Merge pull request #783 from umccr/fix/pg-dd-lambda

fix(pg-dd): python version and environment variables

lib/workload/stateless/stacks/pg-dd/.env.example

@@ -7,5 +7,5 @@ PG_DD_DATABASE_METADATA_MANAGER=metadata_manager
 PG_DD_DATABASE_SEQUENCE_RUN_MANAGER=sequence_run_manager
 PG_DD_DATABASE_WORKFLOW_MANAGER=workflow_manager
 PG_DD_DATABASE_FILEMANAGER=filemanager
-PG_DD_DATABASE_FILEMANAGER_SQL_DUMP='select * from s3_object order by sequencer limit 10000'
+PG_DD_DATABASE_FILEMANAGER_SQL_DUMP='select * from s3_object order by event_time desc limit 10000'
 PG_DD_DATABASE_FILEMANAGER_SQL_LOAD=s3_object

lib/workload/stateless/stacks/pg-dd/Dockerfile

@@ -2,7 +2,7 @@
 # When running this microservice from the Docker Compose root, this Dockerfile
 # will build the image, install dependencies, and start the server
 
-FROM public.ecr.aws/docker/library/python:3.13
+FROM public.ecr.aws/docker/library/python:3.12
 
 RUN pip3 install poetry
 

lib/workload/stateless/stacks/pg-dd/deploy/stack.ts

@@ -108,7 +108,9 @@ export class PgDDStack extends Stack {
         PG_DD_DATABASE_SEQUENCE_RUN_MANAGER: 'sequence_run_manager',
         PG_DD_DATABASE_WORKFLOW_MANAGER: 'workflow_manager',
         PG_DD_DATABASE_FILEMANAGER: 'filemanager',
-        PG_DD_DATABASE_FILEMANAGER_SQL: 'select * from s3_object order by sequencer limit 10000',
+        PG_DD_DATABASE_FILEMANAGER_SQL_DUMP:
+          'select * from s3_object order by event_time desc limit 10000',
+        PG_DD_DATABASE_FILEMANAGER_SQL_LOAD: 's3_object',
         ...(props.prefix && { PG_DD_PREFIX: props.prefix }),
       },
     });

lib/workload/stateless/stacks/pg-dd/pg_dd/handler.py

@@ -1,11 +1,13 @@
 import json
 import logging
 import os
+import tempfile
+import uuid
 from types import SimpleNamespace
 
 from libumccr.aws import libsm
 
-from pg_dd.pg_dd import PgDDS3
+from pg_dd.pg_dd import PgDDS3, PgDDLocal
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -22,4 +24,7 @@
 
 
 def handler(_event, _context):
-    PgDDS3(logger=logger).write_to_bucket()
+    out_dir = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
+
+    PgDDLocal(logger=logger, out_dir=out_dir).write_to_dir()
+    PgDDS3(logger=logger, out_dir=out_dir).write_to_bucket()

lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py

@@ -184,9 +184,13 @@ class PgDDLocal(PgDD):
     Commands related to dumping/loading CSV files to a local directory.
     """
 
-    def __init__(self, logger: logging.Logger = logging.getLogger(__name__)):
+    def __init__(
+        self,
+        out_dir=os.getenv("PG_DD_DIR"),
+        logger: logging.Logger = logging.getLogger(__name__),
+    ):
         super().__init__(logger=logger)
-        self.out = os.getenv("PG_DD_DIR")
+        self.out = out_dir
         self.bucket = os.getenv("PG_DD_BUCKET")
         self.prefix = os.getenv("PG_DD_PREFIX")
         self.s3: S3ServiceResource = boto3.resource("s3")
@@ -242,11 +246,15 @@ class PgDDS3(PgDD):
     Commands related to dumping/loading from S3.
     """
 
-    def __init__(self, logger: logging.Logger = logging.getLogger(__name__)):
+    def __init__(
+        self,
+        out_dir=os.getenv("PG_DD_DIR"),
+        logger: logging.Logger = logging.getLogger(__name__),
+    ):
         super().__init__(logger=logger)
         self.bucket = os.getenv("PG_DD_BUCKET")
         self.prefix = os.getenv("PG_DD_PREFIX")
-        self.dir = os.getenv("PG_DD_DIR")
+        self.dir = out_dir
         self.s3: S3ServiceResource = boto3.resource("s3")
 
     def write_to_bucket(self, db: str = None):

lib/workload/stateless/stacks/pg-dd/pyproject.toml

@@ -7,7 +7,7 @@ readme = "README.md"
 packages = [{ include = "pg_dd" }]
 
 [tool.poetry.dependencies]
-python = "^3.13"
+python = "^3.12"
 boto3 = "^1"
 psycopg = { version = "^3", extras = ["binary"] }
 python-dotenv = "^1"