From f196d029f0605666a70234a06c55ae22d854895c Mon Sep 17 00:00:00 2001 From: mkslofstra Date: Thu, 9 Mar 2023 08:20:34 +0100 Subject: [PATCH 1/7] fix: skip rds files in minio migration because they will point in the direction of the old minio instead of the new resources, hosted on the local filesystem --- scripts/migrate-minio.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index 1759dec3a..5886a92ae 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -9,6 +9,7 @@ import getopt import sys +from pprint import pprint from getpass import getpass from pathlib import Path @@ -64,7 +65,11 @@ def download_all_buckets(target_dir: Path): def download_bucket(minio_client: Minio, bucket: Bucket, target_dir: Path): bucket_dir = target_dir.joinpath(bucket.name) - obj_count = download_objects(bucket.name, bucket_dir, minio_client) + obj_count, skipped = download_objects(bucket.name, bucket_dir, minio_client) + + if len(skipped) > 0: + print("The following rds files weren't copied and will have to be regenerated:") + pprint.pprint(skipped) if obj_count == 0: print("> No files found") @@ -94,17 +99,21 @@ def create_minio_client(): def download_objects(bucket_name: str, bucket_dir: Path, client: Minio) -> int: objects = client.list_objects(bucket_name, recursive=True) obj_count = 0 + skipped = [] for obj in objects: print("- " + obj.object_name) target_file = bucket_dir.joinpath(obj.object_name) - client.fget_object( - bucket_name, - obj.object_name, - file_path=str(target_file) - ) - obj_count += 1 - return obj_count + if target_file.endswith(".rds"): + skipped.append(target_file) + else: + client.fget_object( + bucket_name, + obj.object_name, + file_path=str(target_file) + ) + obj_count += 1 + return obj_count, skipped if __name__ == '__main__': - main(sys.argv[1:]) + main(sys.argv[1:]) \ No newline at end of file From 163d143467e0b97bd01e0ff8d33f86b4d90b3350 Mon Sep 17 00:00:00 2001 From: mkslofstra Date: Thu, 9 Mar 2023 08:47:47 +0100 Subject: [PATCH 2/7] fix: convert posixpath to string --- scripts/migrate-minio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index 5886a92ae..4a94b6a4c 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -103,8 +103,8 @@ def download_objects(bucket_name: str, bucket_dir: Path, client: Minio) -> int: for obj in objects: print("- " + obj.object_name) target_file = bucket_dir.joinpath(obj.object_name) - if target_file.endswith(".rds"): - skipped.append(target_file) + if str(target_file.endswith(".rds")): + skipped.append(str(target_file)) else: client.fget_object( bucket_name, From 943de1c06196c74cace77de99d54b90c00e7fc60 Mon Sep 17 00:00:00 2001 From: mkslofstra Date: Thu, 9 Mar 2023 08:58:52 +0100 Subject: [PATCH 3/7] fix: move str --- scripts/migrate-minio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index 4a94b6a4c..caac6476e 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -103,7 +103,7 @@ def download_objects(bucket_name: str, bucket_dir: Path, client: Minio) -> int: for obj in objects: print("- " + obj.object_name) target_file = bucket_dir.joinpath(obj.object_name) - if str(target_file.endswith(".rds")): + if str(target_file).endswith(".rds"): skipped.append(str(target_file)) else: client.fget_object( From c18d12e347aa321fcd2c24667fb881533fdb1d2f Mon Sep 17 00:00:00 2001 From: mkslofstra Date: Thu, 9 Mar 2023 10:22:05 +0100 Subject: [PATCH 4/7] fix: skip rds files case insensitively --- scripts/migrate-minio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index caac6476e..38af248f7 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -103,7 +103,7 @@ def download_objects(bucket_name: str, bucket_dir: Path, client: Minio) -> int: for obj in objects: print("- " + obj.object_name) target_file = bucket_dir.joinpath(obj.object_name) - if str(target_file).endswith(".rds"): + if str(target_file).lower().endswith(".rds"): skipped.append(str(target_file)) else: client.fget_object( From 6c0b94f98900cf2a8206051d939fc0fe766ba8e9 Mon Sep 17 00:00:00 2001 From: mkslofstra Date: Fri, 10 Mar 2023 12:37:42 +0100 Subject: [PATCH 5/7] fix: move print statement --- scripts/migrate-minio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index 38af248f7..d79c819d2 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -32,7 +32,7 @@ def main(argv): target = None for opt, arg in opts: if opt == "-h": - print(help_string) + print(help_string)x sys.exit() elif opt in ("-m", "--minio"): minio_url = arg @@ -101,11 +101,11 @@ def download_objects(bucket_name: str, bucket_dir: Path, client: Minio) -> int: obj_count = 0 skipped = [] for obj in objects: - print("- " + obj.object_name) target_file = bucket_dir.joinpath(obj.object_name) if str(target_file).lower().endswith(".rds"): skipped.append(str(target_file)) else: + print("- " + obj.object_name) client.fget_object( bucket_name, obj.object_name, From 5033d1cd0e62d5214049b9927de40ce0c8b992b9 Mon Sep 17 00:00:00 2001 From: marikaris Date: Thu, 16 Mar 2023 12:32:00 +0100 Subject: [PATCH 6/7] fix: import pprint correctly --- scripts/migrate-minio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index d79c819d2..71f5d8a97 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -9,7 +9,7 @@ import getopt import sys -from pprint import pprint +import pprint from getpass import getpass from pathlib import Path @@ -116,4 +116,4 @@ def download_objects(bucket_name: str, bucket_dir: Path, client: Minio) -> int: if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) From acad9cce4c09f28584bf730843def50c9b5e4fe2 Mon Sep 17 00:00:00 2001 From: marikaris Date: Fri, 17 Mar 2023 14:14:00 +0100 Subject: [PATCH 7/7] Update migrate-minio.py --- scripts/migrate-minio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate-minio.py b/scripts/migrate-minio.py index 71f5d8a97..52e1339e0 100644 --- a/scripts/migrate-minio.py +++ b/scripts/migrate-minio.py @@ -32,7 +32,7 @@ def main(argv): target = None for opt, arg in opts: if opt == "-h": - print(help_string)x + print(help_string) sys.exit() elif opt in ("-m", "--minio"): minio_url = arg