Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(pg-dd): python version and environment variables #783

Merged
merged 2 commits into from
Dec 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/workload/stateless/stacks/pg-dd/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ PG_DD_DATABASE_METADATA_MANAGER=metadata_manager
PG_DD_DATABASE_SEQUENCE_RUN_MANAGER=sequence_run_manager
PG_DD_DATABASE_WORKFLOW_MANAGER=workflow_manager
PG_DD_DATABASE_FILEMANAGER=filemanager
PG_DD_DATABASE_FILEMANAGER_SQL_DUMP='select * from s3_object order by sequencer limit 10000'
PG_DD_DATABASE_FILEMANAGER_SQL_DUMP='select * from s3_object order by event_time desc limit 10000'
PG_DD_DATABASE_FILEMANAGER_SQL_LOAD=s3_object
2 changes: 1 addition & 1 deletion lib/workload/stateless/stacks/pg-dd/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# When running this microservice from the Docker Compose root, this Dockerfile
# will build the image, install dependencies, and start the server

FROM public.ecr.aws/docker/library/python:3.13
FROM public.ecr.aws/docker/library/python:3.12

RUN pip3 install poetry

Expand Down
4 changes: 3 additions & 1 deletion lib/workload/stateless/stacks/pg-dd/deploy/stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ export class PgDDStack extends Stack {
PG_DD_DATABASE_SEQUENCE_RUN_MANAGER: 'sequence_run_manager',
PG_DD_DATABASE_WORKFLOW_MANAGER: 'workflow_manager',
PG_DD_DATABASE_FILEMANAGER: 'filemanager',
PG_DD_DATABASE_FILEMANAGER_SQL: 'select * from s3_object order by sequencer limit 10000',
PG_DD_DATABASE_FILEMANAGER_SQL_DUMP:
'select * from s3_object order by event_time desc limit 10000',
PG_DD_DATABASE_FILEMANAGER_SQL_LOAD: 's3_object',
...(props.prefix && { PG_DD_PREFIX: props.prefix }),
},
});
Expand Down
9 changes: 7 additions & 2 deletions lib/workload/stateless/stacks/pg-dd/pg_dd/handler.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import json
import logging
import os
import tempfile
import uuid
from types import SimpleNamespace

from libumccr.aws import libsm

from pg_dd.pg_dd import PgDDS3
from pg_dd.pg_dd import PgDDS3, PgDDLocal

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
Expand All @@ -22,4 +24,7 @@


def handler(_event, _context):
PgDDS3(logger=logger).write_to_bucket()
out_dir = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))

PgDDLocal(logger=logger, out_dir=out_dir).write_to_dir()
PgDDS3(logger=logger, out_dir=out_dir).write_to_bucket()
16 changes: 12 additions & 4 deletions lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,9 +184,13 @@ class PgDDLocal(PgDD):
Commands related to dumping/loading CSV files to a local directory.
"""

def __init__(self, logger: logging.Logger = logging.getLogger(__name__)):
def __init__(
self,
out_dir=os.getenv("PG_DD_DIR"),
logger: logging.Logger = logging.getLogger(__name__),
):
super().__init__(logger=logger)
self.out = os.getenv("PG_DD_DIR")
self.out = out_dir
self.bucket = os.getenv("PG_DD_BUCKET")
self.prefix = os.getenv("PG_DD_PREFIX")
self.s3: S3ServiceResource = boto3.resource("s3")
Expand Down Expand Up @@ -242,11 +246,15 @@ class PgDDS3(PgDD):
Commands related to dumping/loading from S3.
"""

def __init__(self, logger: logging.Logger = logging.getLogger(__name__)):
def __init__(
self,
out_dir=os.getenv("PG_DD_DIR"),
logger: logging.Logger = logging.getLogger(__name__),
):
super().__init__(logger=logger)
self.bucket = os.getenv("PG_DD_BUCKET")
self.prefix = os.getenv("PG_DD_PREFIX")
self.dir = os.getenv("PG_DD_DIR")
self.dir = out_dir
self.s3: S3ServiceResource = boto3.resource("s3")

def write_to_bucket(self, db: str = None):
Expand Down
35 changes: 18 additions & 17 deletions lib/workload/stateless/stacks/pg-dd/poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lib/workload/stateless/stacks/pg-dd/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ readme = "README.md"
packages = [{ include = "pg_dd" }]

[tool.poetry.dependencies]
python = "^3.13"
python = "^3.12"
boto3 = "^1"
psycopg = { version = "^3", extras = ["binary"] }
python-dotenv = "^1"
Expand Down
Loading