Skip to content

Commit

Permalink
Add formating to cleanup_time logging
Browse files Browse the repository at this point in the history
  • Loading branch information
krysal committed Apr 22, 2024
1 parent 4bd68ee commit 98d3334
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions ingestion_server/ingestion_server/cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,9 +337,10 @@ def _upload_to_s3(self, field: str):
part_number = self.buffer[field].part
log.info(f"Uploading file part {part_number} of `{field}` to S3...")
s3_file_name = f"{self.s3_path}/{self.date}_{field}_{part_number}.tsv"
self.s3_bucket.upload_file(f"{field}.tsv", s3_file_name)
tsv_file_name = f"{field}.tsv"
self.s3_bucket.upload_file(tsv_file_name, s3_file_name)
self.buffer[field].part += 1
os.remove(f"{field}.tsv")
os.remove(tsv_file_name)

def save(self, result: dict) -> dict[str, int]:
for field, cleaned_items in result.items():
Expand Down Expand Up @@ -455,6 +456,6 @@ def clean_image_data(table):
end_time = time.perf_counter()
cleanup_time = end_time - start_time
log.info(
f"Cleaned all records in {cleanup_time} seconds,"
f"Cleaned all records in {cleanup_time:.3f} seconds,"
f"counts: {cleaned_counts_by_field}"
)

0 comments on commit 98d3334

Please sign in to comment.