From 17cbfc1b9a83b2bb8a9e9131a789c98d5b624c37 Mon Sep 17 00:00:00 2001 From: JimmyYang20 Date: Thu, 28 Oct 2021 17:30:55 +0800 Subject: [PATCH] storage initializer: keep dir name in s3 download when downloading the specified folder, keep the name of the folder itself. Signed-off-by: JimmyYang20 --- scripts/storage-initializer/README.md | 8 ++++++-- scripts/storage-initializer/download.py | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/storage-initializer/README.md b/scripts/storage-initializer/README.md index 6f4278208..a2133d533 100644 --- a/scripts/storage-initializer/README.md +++ b/scripts/storage-initializer/README.md @@ -12,8 +12,12 @@ python3 download.py s3://models/classification/model.tar.gz /tmp/models/ export S3_ENDPOINT_URL=https://play.min.io export ACCESS_KEY_ID=Q3AM3UQ867SPQQA43P2F export SECRET_ACCESS_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -python3 download.py s3://datasets/mnist /tmp/mnist -# we then download the content of mnist directory into /tmp/mnist/ + +python3 download.py s3://datasets/mnist/1.jpg /tmp +# we then download the file 1.jpg into /tmp, and result is /tmp/1.jpg. + +python3 download.py s3://datasets/mnist /tmp +# we then download the folder mnist into /tmp, and result is /tmp/mnist. ``` 3. http server: diff --git a/scripts/storage-initializer/download.py b/scripts/storage-initializer/download.py index 8d86196e5..65a2d34d6 100644 --- a/scripts/storage-initializer/download.py +++ b/scripts/storage-initializer/download.py @@ -176,6 +176,20 @@ def download_s3_with_multi_files(download_files, def _download_s3(client, uri, out_dir): + """ + The function downloads specified file or folder to local directory address. + this function supports: + 1. when downloading the specified file, keep the name of the file itself. + 2. when downloading the specified folder, keep the name of the folder itself. + + Parameters: + client: s3 client + s3_url(string): url in s3, e.g. file url: s3://dev/data/data.txt, directory url: s3://dev/data + out_dir(string): local directory address, e.g. /tmp/data/ + + Returns: + int: files of number in s3_url + """ bucket_args = uri.replace(_S3_PREFIX, "", 1).split("/", 1) bucket_name = bucket_args[0] bucket_path = len(bucket_args) > 1 and bucket_args[1] or "" @@ -186,9 +200,10 @@ def _download_s3(client, uri, out_dir): use_api_v1=True) count = 0 + root_path = os.path.split(os.path.normpath(bucket_path))[0] for obj in objects: # Replace any prefix from the object key with out_dir - subdir_object_key = obj.object_name[len(bucket_path):].strip("/") + subdir_object_key = obj.object_name[len(root_path):].strip("/") # fget_object handles directory creation if does not exist if not obj.is_dir: local_file = os.path.join(