-
Notifications
You must be signed in to change notification settings - Fork 0
/
s3_multipart_upload.sh
executable file
·82 lines (62 loc) · 2.77 KB
/
s3_multipart_upload.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
#!/bin/sh
# set -x
if [ $# -ne 3 ]
then
echo "Invalid number of parameters passed."
echo "Usage : sh $0 <Filename to be uploaded> <Split size in MB> <Target Bucket Name"
exit 1
fi
FILE_NAME=$1
SPLIT_SIZE=$2
TARGET_BUCKET=$3
call_s3upload_api(){
echo "`date` :: ---------- Part Number: ${i} upload started ------------"
ETAG=`aws s3api upload-part --bucket ${TARGET_BUCKET} --key ${FILE_NAME} --part-number ${i} --body ${split_file} --upload-id ${UPLOAD_ID}|jq -r ".ETag"`
echo "`date` :: ---------- Part Number: ${i} uploaded ------------------"
echo "PartNumber:${i}:ETag:${ETAG}" >> list.txt
}
FILE_SIZE_BYTE=$((`wc -c ${FILE_NAME}|tr -s " "|cut -d" " -f2`))
FILE_SIZE_MB=$((FILE_SIZE_BYTE / 1000000))
regex="^[0-9]M$|^[1-9][0-9]M$|^(100)M$"
split -b ${SPLIT_SIZE} ${FILE_NAME}
echo "`date` :: ${FILE_NAME} splited into `ls x*|wc -l|tr -s " "` parts"
if [ ${FILE_SIZE_MB} -gt 5 ]
then
echo "`date` :: Size of file ${FILE_NAME} (${FILE_SIZE_BYTE} Bytes) is greater than 5MB and is eligible for S3 multipart upload."
if [[ ${SPLIT_SIZE} =~ ${regex} ]]
then
start=$(date +%s)
echo "`date` :: Split Size is in correct format. Initiating the upload..."
UPLOAD_ID=`aws s3api create-multipart-upload --bucket ${TARGET_BUCKET} --key ${FILE_NAME}|jq -r ".UploadId"`
echo "`date` :: **************** Multipart upload initiated ****************"
# echo "Upload Id : ${UPLOAD_ID}"
i=0
touch list.txt
for split_file in `ls x*`
do
((i=i+1))
call_s3upload_api ${i}, ${split_file}, ${UPLOAD_ID} &
done
wait
sort --key 2 --field-separator : --numeric-sort list.txt|jq -Rs '{Parts:split("\n")|map(split(":")|{(.[0]):.[1]|tonumber,(.[2]):.[3]}?)}' > list.json
# echo "==============================list.json====================================="
# cat list.json
echo "`date` :: ****************Multipart upload completion started **********"
aws s3api complete-multipart-upload --multipart-upload file://list.json --bucket ${TARGET_BUCKET} --key ${FILE_NAME} --upload-id ${UPLOAD_ID} > s3_upload_status.txt
echo "========================S3 Multipart Upload Status============================"
cat s3_upload_status.txt
end=$(date +%s)
runtime=$(($end-$start))
echo "Total execution time: ${runtime} seconds."
rm -f list.*
rm -f x*
else
echo "Invalid split size format - correct format is NNM, where N is an integer"
exit 1
fi
else
echo "Size of file ${FILE_NAME} (${FILE_SIZE_BYTE} Bytes) is less than 5MB and not eligible for S3 multipart upload."
exit 1
fi
echo "`date` :: ****************Multipart upload completed****************"
exit 0