From ae393eb2186bf77498333808d9ad832e7b0d4281 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 26 Nov 2024 16:59:27 +1100 Subject: [PATCH] fix(pg-dd): double compressed files --- .../stateless/stacks/pg-dd/.env.example | 2 +- .../stateless/stacks/pg-dd/pg_dd/pg_dd.py | 6 +++-- .../stateless/stacks/pg-dd/poetry.lock | 22 +++++++++---------- .../stateless/stacks/pg-dd/pyproject.toml | 2 +- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/workload/stateless/stacks/pg-dd/.env.example b/lib/workload/stateless/stacks/pg-dd/.env.example index 49a3a57fc..e674b8e70 100644 --- a/lib/workload/stateless/stacks/pg-dd/.env.example +++ b/lib/workload/stateless/stacks/pg-dd/.env.example @@ -1,4 +1,4 @@ -PG_DD_URL=postgresql://orcabus:orcabus@0.0.0.0:5432# pragma: allowlist secret +PG_DD_URL=postgresql://orcabus:orcabus@0.0.0.0:5432 # pragma: allowlist secret PG_DD_DIR=data PG_DD_BUCKET=orcabus-test-data-843407916570-ap-southeast-2 PG_DD_PREFIX=pg-dd diff --git a/lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py b/lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py index 51032fb59..d5c8fb621 100644 --- a/lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py +++ b/lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py @@ -287,5 +287,7 @@ def download_local(self, exists_ok: bool = True): self.logger.info(f"file already exists: {file}") continue - s3_object = self.s3.Object(self.bucket, obj.key) - s3_object.download_file(file) + s3_object = self.s3.Object(self.bucket, obj.key).get() + data = gzip.decompress(s3_object["Body"].read()) + with open(file, "wb") as f: + f.write(data) diff --git a/lib/workload/stateless/stacks/pg-dd/poetry.lock b/lib/workload/stateless/stacks/pg-dd/poetry.lock index d10f581f3..fa87ea2f4 100644 --- a/lib/workload/stateless/stacks/pg-dd/poetry.lock +++ b/lib/workload/stateless/stacks/pg-dd/poetry.lock @@ -2,17 +2,17 @@ [[package]] name = "boto3" -version = "1.35.66" +version = "1.35.69" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.66-py3-none-any.whl", hash = "sha256:09a610f8cf4d3c22d4ca69c1f89079e3a1c82805ce94fa0eb4ecdd4d2ba6c4bc"}, - {file = "boto3-1.35.66.tar.gz", hash = "sha256:c392b9168b65e9c23483eaccb5b68d1f960232d7f967a1e00a045ba065ce050d"}, + {file = "boto3-1.35.69-py3-none-any.whl", hash = "sha256:20945912130cca1505f45819cd9b7183a0e376e91a1221a0b1f50c80d35fd7e2"}, + {file = "boto3-1.35.69.tar.gz", hash = "sha256:40db86c7732a310b282f595251995ecafcbd62009a57e47a22683862e570cc7a"}, ] [package.dependencies] -botocore = ">=1.35.66,<1.36.0" +botocore = ">=1.35.69,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -21,13 +21,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.66" +version = "1.35.69" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.66-py3-none-any.whl", hash = "sha256:d0683e9c18bb6852f768da268086c3749d925332a664db0dd1459cfa7e96e475"}, - {file = "botocore-1.35.66.tar.gz", hash = "sha256:51f43220315f384959f02ea3266740db4d421592dd87576c18824e424b349fdb"}, + {file = "botocore-1.35.69-py3-none-any.whl", hash = "sha256:cad8d9305f873404eee4b197d84e60a40975d43cbe1ab63abe893420ddfe6e3c"}, + {file = "botocore-1.35.69.tar.gz", hash = "sha256:f9f23dd76fb247d9b0e8d411d2995e6f847fc451c026f1e58e300f815b0b36eb"}, ] [package.dependencies] @@ -146,13 +146,13 @@ reports = ["lxml"] [[package]] name = "mypy-boto3-s3" -version = "1.35.61" -description = "Type annotations for boto3.S3 1.35.61 service generated with mypy-boto3-builder 8.2.1" +version = "1.35.69" +description = "Type annotations for boto3 S3 1.35.69 service generated with mypy-boto3-builder 8.3.1" optional = false python-versions = ">=3.8" files = [ - {file = "mypy_boto3_s3-1.35.61-py3-none-any.whl", hash = "sha256:4cfc410a02a302935876f0d1ae3f0738bf540acd686168790fb0c5986a085f1e"}, - {file = "mypy_boto3_s3-1.35.61.tar.gz", hash = "sha256:6965fe6c5f3d8362ac7895663540844ed29c885c92a62233c29a1c031ca28c90"}, + {file = "mypy_boto3_s3-1.35.69-py3-none-any.whl", hash = "sha256:11a34259983e09d67e4d3a322fd47904a006bbfff19984e4e36a77e30f2014bb"}, + {file = "mypy_boto3_s3-1.35.69.tar.gz", hash = "sha256:97f7944a84a4a49282825bef1483a25680dcdce75da6017745d709d2cf2aa1c0"}, ] [[package]] diff --git a/lib/workload/stateless/stacks/pg-dd/pyproject.toml b/lib/workload/stateless/stacks/pg-dd/pyproject.toml index 057ece265..0f41f94af 100644 --- a/lib/workload/stateless/stacks/pg-dd/pyproject.toml +++ b/lib/workload/stateless/stacks/pg-dd/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "postgres-data-dump" +name = "pg-dd" version = "0.1.0" description = "A serivce to dump postgres databases to S3." authors = ["Marko malenic "]