From 2fae25e3cf97b5eea8a1543613ffaf010dd23a03 Mon Sep 17 00:00:00 2001 From: Noah Yoshida Date: Fri, 15 Dec 2023 15:33:30 -0800 Subject: [PATCH] fix sync script to account for subfolder bucket paths (#135) --- sync.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sync.sh b/sync.sh index 56d816c9c..3b556a8b9 100755 --- a/sync.sh +++ b/sync.sh @@ -63,8 +63,13 @@ aws s3 ls "${S3_PATH}" --recursive | awk '{print $4}' aws s3 ls "${S3_PATH}" --recursive | awk '{print $4}' | xargs -I {} bash -c 'sudo mkdir -p "${HUGGINGFACE_HUB_CACHE}/$(dirname "{}")"' copy_file() { + # The files are a list of files without the bucket prefix. + # In the event that the env variable HF_CACHE_BUCKET is not just a bucket, but a bucket plus a subfolder, + # the subfolder will already be included into the file variable. + # In this case, strip all subpaths from the env variable HF_CACHE_BUCKET before attempting to download weights. file="$1" - sudo -E aws s3 cp "s3://${HF_CACHE_BUCKET}/${file}" "${HUGGINGFACE_HUB_CACHE}/${file}" + true_bucket=`echo $HF_CACHE_BUCKET | cut -d / -f 1` + sudo -E aws s3 cp "s3://${true_bucket}/${file}" "${HUGGINGFACE_HUB_CACHE}/${file}" } export -f copy_file