fix(pg-dd): double compressed files

umccr · Nov 26, 2024 · ae393eb · ae393eb
1 parent c2a8bdb
commit ae393eb
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 15 deletions.
diff --git a/lib/workload/stateless/stacks/pg-dd/.env.example b/lib/workload/stateless/stacks/pg-dd/.env.example
@@ -1,4 +1,4 @@
-PG_DD_URL=postgresql://orcabus:[email protected]:5432# pragma: allowlist secret
+PG_DD_URL=postgresql://orcabus:[email protected]:5432 # pragma: allowlist secret
 PG_DD_DIR=data
 PG_DD_BUCKET=orcabus-test-data-843407916570-ap-southeast-2
 PG_DD_PREFIX=pg-dd

diff --git a/lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py b/lib/workload/stateless/stacks/pg-dd/pg_dd/pg_dd.py
@@ -287,5 +287,7 @@ def download_local(self, exists_ok: bool = True):
                 self.logger.info(f"file already exists: {file}")
                 continue
 
-            s3_object = self.s3.Object(self.bucket, obj.key)
-            s3_object.download_file(file)
+            s3_object = self.s3.Object(self.bucket, obj.key).get()
+            data = gzip.decompress(s3_object["Body"].read())
+            with open(file, "wb") as f:
+                f.write(data)
diff --git a/lib/workload/stateless/stacks/pg-dd/poetry.lock b/lib/workload/stateless/stacks/pg-dd/poetry.lock
diff --git a/lib/workload/stateless/stacks/pg-dd/pyproject.toml b/lib/workload/stateless/stacks/pg-dd/pyproject.toml
@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "postgres-data-dump"
+name = "pg-dd"
 version = "0.1.0"
 description = "A serivce to dump postgres databases to S3."
 authors = ["Marko malenic <[email protected]>"]