diff --git a/tests/commands/list_object_versions.sh b/tests/commands/list_object_versions.sh index f704b1b6..709dedb9 100644 --- a/tests/commands/list_object_versions.sh +++ b/tests/commands/list_object_versions.sh @@ -34,30 +34,10 @@ list_object_versions_rest() { log 2 "'list_object_versions_rest' requires bucket name" return 1 fi - generate_hash_for_payload "" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - # shellcheck disable=SC2154 - canonical_request="GET -/$1 -versions= -host:${AWS_ENDPOINT_URL#*//} -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -$payload_hash" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + log 5 "list object versions REST" + if ! result=$(BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object_versions.txt" ./tests/rest_scripts/list_object_versions.sh); then + log 2 "error listing object versions: $result" return 1 fi - - get_signature - # shellcheck disable=SC2034,SC2154 - reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1) + return 0 } \ No newline at end of file diff --git a/tests/rest_scripts/list_object_versions.sh b/tests/rest_scripts/list_object_versions.sh new file mode 100755 index 00000000..70572c34 --- /dev/null +++ b/tests/rest_scripts/list_object_versions.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +# shellcheck disable=SC2034 +canonical_request="GET +/$bucket_name +versions= +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/list_objects.sh b/tests/rest_scripts/list_objects.sh index 39ef7bca..f543163d 100755 --- a/tests/rest_scripts/list_objects.sh +++ b/tests/rest_scripts/list_objects.sh @@ -23,6 +23,8 @@ bucket_name="$BUCKET_NAME" version_two="${VERSION_TWO:-FALSE}" max_keys="${MAX_KEYS:-0}" # shellcheck disable=SC2153 +marker="$MARKER" +# shellcheck disable=SC2153 if [ "$CONTINUATION_TOKEN" != "" ]; then continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri') fi @@ -34,6 +36,9 @@ canonical_request="GET /$bucket_name " +if [ "$MARKER" != "" ]; then + add_parameter "canonical_request" "marker=$marker" +fi if [ "$CONTINUATION_TOKEN" != "" ]; then add_parameter "canonical_request" "continuation-token=$continuation_token" fi @@ -56,6 +61,9 @@ create_canonical_hash_sts_and_signature curl_command+=(curl -ks -w "\"%{http_code}\"") url="'$AWS_ENDPOINT_URL/$bucket_name" +if [ "$MARKER" != "" ]; then + add_parameter "url" "marker=$marker" +fi if [ "$CONTINUATION_TOKEN" != "" ]; then add_parameter "url" "continuation-token=$continuation_token" fi diff --git a/tests/test_common_acl.sh b/tests/test_common_acl.sh index 8af4697b..7d05bc3b 100644 --- a/tests/test_common_acl.sh +++ b/tests/test_common_acl.sh @@ -66,19 +66,21 @@ test_common_put_bucket_acl() { grantee_type="Group" grantee_id="http://acs.amazonaws.com/groups/global/AllUsers" else - grantee_type="ID" + grantee_type="CanonicalUser" grantee_id="$USERNAME_ONE" fi run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "$grantee_type" "$grantee_id" "READ" "$AWS_ACCESS_KEY_ID" assert_success + log 5 "acl: $(cat "$TEST_FILE_FOLDER/$acl_file")" + run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" assert_success run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME" assert_success - run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID" + run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID" assert_success run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file" diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 5cba21bc..1d07e7eb 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -36,6 +36,7 @@ source ./tests/util/util_list_buckets.sh source ./tests/util/util_list_objects.sh source ./tests/util/util_list_parts.sh source ./tests/util/util_lock_config.sh +source ./tests/util/util_multipart_before_completion.sh source ./tests/util/util_ownership.sh source ./tests/util/util_policy.sh source ./tests/util/util_public_access_block.sh @@ -318,7 +319,7 @@ export RUN_USERS=true @test "REST - get object attributes" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/916" + skip "https://github.com/versity/versitygw/issues/1000" fi test_file="test_file" @@ -340,7 +341,7 @@ export RUN_USERS=true @test "REST - attributes - invalid param" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/917" + skip "https://github.com/versity/versitygw/issues/1001" fi test_file="test_file" @@ -359,7 +360,7 @@ export RUN_USERS=true @test "REST - attributes - checksum" { if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/928" + skip "https://github.com/versity/versitygw/issues/1006" fi test_file="test_file" @@ -461,7 +462,7 @@ export RUN_USERS=true run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three" assert_success - run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" + run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "TRUE" assert_success continuation_token=$output @@ -469,3 +470,25 @@ export RUN_USERS=true run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}" assert_success } + +@test "REST - list objects v1 - no NextMarker without delimiter" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/999" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + test_file_two="test_file_2" + run create_test_files "$test_file" "$test_file_two" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two" + assert_success + + run list_objects_v1_check_nextmarker_empty "$BUCKET_ONE_NAME" + assert_success +} \ No newline at end of file diff --git a/tests/test_s3api_multipart.sh b/tests/test_s3api_multipart.sh index 9ae6c7d9..d6025ad9 100755 --- a/tests/test_s3api_multipart.sh +++ b/tests/test_s3api_multipart.sh @@ -19,6 +19,7 @@ source ./tests/test_s3api_root_inner.sh source ./tests/util/util_file.sh source ./tests/util/util_multipart.sh source ./tests/util/util_multipart_abort.sh +source ./tests/util/util_multipart_before_completion.sh source ./tests/util/util_tags.sh source ./tests/commands/get_object.sh source ./tests/commands/put_object.sh diff --git a/tests/test_s3api_policy.sh b/tests/test_s3api_policy.sh index 7fa5bb03..c5380d5d 100755 --- a/tests/test_s3api_policy.sh +++ b/tests/test_s3api_policy.sh @@ -21,6 +21,7 @@ source ./tests/test_s3api_policy_multipart.sh source ./tests/test_s3api_policy_object.sh source ./tests/util/util_multipart.sh source ./tests/util/util_multipart_abort.sh +source ./tests/util/util_multipart_before_completion.sh source ./tests/util/util_file.sh source ./tests/util/util_policy.sh source ./tests/util/util_tags.sh diff --git a/tests/util/util_attributes.sh b/tests/util/util_attributes.sh index e19a27da..e9456698 100644 --- a/tests/util/util_attributes.sh +++ b/tests/util/util_attributes.sh @@ -25,6 +25,7 @@ check_attributes_after_upload() { log 2 "'check_attributes_after_upload' requires file size" return 1 fi + log 5 "attributes: $(cat "$TEST_FILE_FOLDER/attributes.txt")" if ! object_size=$(xmllint --xpath '//*[local-name()="ObjectSize"]/text()' "$TEST_FILE_FOLDER/attributes.txt" 2>&1); then log 2 "error getting checksum: $object_size" return 1 @@ -54,10 +55,11 @@ check_attributes_after_upload() { log 2 "unexpected parts count, expected 4, was $parts_count" return 1 fi + return 1 } check_attributes_invalid_param() { - if [ "$1" -ne 1 ]; then + if [ $# -ne 1 ]; then log 2 "'check_attributes_invalid_param' requires test file" return 1 fi diff --git a/tests/util/util_list_objects.sh b/tests/util/util_list_objects.sh index 837311a0..804eb3db 100644 --- a/tests/util/util_list_objects.sh +++ b/tests/util/util_list_objects.sh @@ -246,11 +246,11 @@ list_objects_with_user_rest_verify_success() { } list_objects_check_params_get_token() { - if [ $# -ne 3 ]; then - log 2 "'list_objects_check_params_get_token' requires bucket name, files" + if [ $# -ne 4 ]; then + log 2 "'list_objects_check_params_get_token' requires bucket name, files, version two" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then log 2 "error attempting to get bucket ACL response: $result" return 1 fi @@ -300,4 +300,31 @@ list_objects_check_continuation_error() { log 2 "invalid error code" return 1 fi -} \ No newline at end of file +} + +list_objects_v1_check_nextmarker_empty() { + if [ $# -ne 1 ]; then + log 2 "'get_next_objects_v1' requires bucket name" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then + log 2 "error attempting to get bucket ACL response: $result" + return 1 + fi + log 5 "output: $(cat "$TEST_FILE_FOLDER/objects.txt")" + if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then + if [[ "$next_marker" != *"XPath set is empty"* ]]; then + log 2 "unexpected error: $next_marker" + return 1 + fi + return 0 + fi + log 5 "next marker: $next_marker" + marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1) + log 5 "marker text: $marker_text" + if [[ "$marker_text" != *"Document is empty"* ]]; then + log 2 "NextMarker text should be empty, but is $marker_text" + return 1 + fi + return 0 +} diff --git a/tests/util/util_multipart.sh b/tests/util/util_multipart.sh index cb8dcf5c..4cd0eb64 100644 --- a/tests/util/util_multipart.sh +++ b/tests/util/util_multipart.sh @@ -14,208 +14,6 @@ # specific language governing permissions and limitations # under the License. -create_upload_and_test_parts_listing() { - if [ $# -ne 2 ]; then - log 2 "'create_upload_and_test_parts_listing' requires test file, policy_file" - return 1 - fi - if ! create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$1" "$USERNAME_ONE" "$PASSWORD_ONE"; then - log 2 "error creating multipart upload with user" - return 1 - fi - - # shellcheck disable=SC2154 - if list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then - log 2 "list parts with user succeeded despite lack of policy permissions" - return 1 - fi - - if ! setup_policy_with_single_statement "$TEST_FILE_FOLDER/$2" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:ListMultipartUploadParts" "arn:aws:s3:::$BUCKET_ONE_NAME/*"; then - log 2 "error setting up policy" - return 1 - fi - - if ! put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$2"; then - log 2 "error putting policy" - return 1 - fi - - if ! list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then - log 2 "error listing parts after policy add" - return 1 - fi - return 0 -} - -start_multipart_upload_list_check_parts() { - if [ $# -ne 3 ]; then - log 2 "'start_multipart_upload_and_list_parts' requires bucket, key, original source" - return 1 - fi - if ! start_multipart_upload_and_list_parts "$1" "$2" "$3" 4; then - log 2 "error starting upload" - return 1 - fi - - declare -a parts_map - # shellcheck disable=SC2154 - log 5 "parts: $parts" - for i in {0..3}; do - if ! parse_parts_and_etags "$i"; then - log 2 "error parsing part $i" - return 1 - fi - done - if [[ ${#parts_map[@]} -eq 0 ]]; then - log 2 "error loading multipart upload parts to check" - return 1 - fi - - for i in {0..3}; do - if ! compare_parts_to_listed_parts "$i"; then - log 2 "error comparing parts to listed parts" - return 1 - fi - done - return 0 -} - -parse_parts_and_etags() { - if [ $# -ne 1 ]; then - log 2 "'parse_parts_and_etags' requires part id" - return 1 - fi - local part_number - local etag - # shellcheck disable=SC2154 - if ! part=$(echo "$parts" | grep -v "InsecureRequestWarning" | jq -r ".[$i]" 2>&1); then - log 2 "error getting part: $part" - return 1 - fi - if ! part_number=$(echo "$part" | jq ".PartNumber" 2>&1); then - log 2 "error parsing part number: $part_number" - return 1 - fi - if [[ $part_number == "" ]]; then - log 2 "error: blank part number" - return 1 - fi - if ! etag=$(echo "$part" | jq ".ETag" 2>&1); then - log 2 "error parsing etag: $etag" - return 1 - fi - if [[ $etag == "" ]]; then - log 2 "error: blank etag" - return 1 - fi - # shellcheck disable=SC2004 - parts_map[$part_number]=$etag -} - -compare_parts_to_listed_parts() { - if [ $# -ne 1 ]; then - log 2 "'compare_parts_to_listed_parts' requires part number" - return 1 - fi - local part_number - local etag - # shellcheck disable=SC2154 - if ! listed_part=$(echo "$listed_parts" | grep -v "InsecureRequestWarning" | jq -r ".Parts[$i]" 2>&1); then - log 2 "error parsing listed part: $listed_part" - return 1 - fi - if ! part_number=$(echo "$listed_part" | jq ".PartNumber" 2>&1); then - log 2 "error parsing listed part number: $part_number" - return 1 - fi - if ! etag=$(echo "$listed_part" | jq ".ETag" 2>&1); then - log 2 "error getting listed etag: $etag" - return 1 - fi - if [[ ${parts_map[$part_number]} != "$etag" ]]; then - log 2 "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)" - return 1 - fi -} - -# list parts of an unfinished multipart upload -# params: bucket, key, local file location, and parts to split into before upload -# export parts on success, return 1 for error -start_multipart_upload_and_list_parts() { - if [ $# -ne 4 ]; then - log 2 "list multipart upload parts command requires bucket, key, file, and part count" - return 1 - fi - - if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then - log 2 "error performing pre-completion multipart upload" - return 1 - fi - - if ! list_parts "$1" "$2" "$upload_id"; then - log 2 "Error listing multipart upload parts: $listed_parts" - return 1 - fi - export listed_parts -} - -create_list_check_multipart_uploads() { - if [ $# -ne 3 ]; then - log 2 "list multipart uploads command requires bucket and two keys" - return 1 - fi - if ! create_and_list_multipart_uploads "$1" "$2" "$3"; then - log 2 "error creating and listing multipart uploads" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "Uploads: $uploads" - raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning") - if ! key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1); then - log 2 "error getting key one: $key_one" - return 1 - fi - if ! key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1); then - log 2 "error getting key two: $key_two" - return 1 - fi - if [[ "$2" != "$key_one" ]]; then - log 2 "Key mismatch ($2, $key_one)" - return 1 - fi - if [[ "$3" != "$key_two" ]]; then - log 2 "Key mismatch ($3, $key_two)" - return 1 - fi - return 0 -} - -# list unfinished multipart uploads -# params: bucket, key one, key two -# export current two uploads on success, return 1 for error -create_and_list_multipart_uploads() { - if [ $# -ne 3 ]; then - log 2 "list multipart uploads command requires bucket and two keys" - return 1 - fi - - if ! create_multipart_upload "$1" "$2"; then - log 2 "error creating multpart upload" - return 1 - fi - - if ! create_multipart_upload "$1" "$3"; then - log 2 "error creating multpart upload two" - return 1 - fi - - if ! list_multipart_uploads "$1"; then - log 2 "error listing uploads" - return 1 - fi - return 0 -} - multipart_upload_from_bucket() { if [ $# -ne 4 ]; then log 2 "multipart upload from bucket command missing bucket, copy source, key, and/or part count" @@ -242,10 +40,12 @@ multipart_upload_from_bucket() { parts="[" for ((i = 1; i <= $4; i++)); do + # shellcheck disable=SC2154 if ! upload_part_copy "$1" "$2-copy" "$upload_id" "$2" "$i"; then log 2 "error uploading part $i" return 1 fi + # shellcheck disable=SC2154 parts+="{\"ETag\": $etag, \"PartNumber\": $i}" if [[ $i -ne $4 ]]; then parts+="," @@ -301,114 +101,6 @@ multipart_upload_from_bucket_range() { return 0 } -# perform all parts of a multipart upload before completion command -# params: bucket, key, file to split and upload, number of file parts to upload -# return: 0 for success, 1 for failure -multipart_upload_before_completion() { - if [ $# -ne 4 ]; then - log 2 "multipart upload pre-completion command missing bucket, key, file, and/or part count" - return 1 - fi - - if ! split_file "$3" "$4"; then - log 2 "error splitting file" - return 1 - fi - - if ! create_multipart_upload "$1" "$2"; then - log 2 "error creating multpart upload" - return 1 - fi - - parts="[" - for ((i = 1; i <= $4; i++)); do - # shellcheck disable=SC2154 - if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then - log 2 "error uploading part $i" - return 1 - fi - parts+="{\"ETag\": $etag, \"PartNumber\": $i}" - if [[ $i -ne $4 ]]; then - parts+="," - fi - done - parts+="]" - - export parts -} - -multipart_upload_before_completion_with_params() { - if [ $# -ne 10 ]; then - log 2 "multipart upload command missing bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" - return 1 - fi - - split_file "$3" "$4" || split_result=$? - if [[ $split_result -ne 0 ]]; then - log 2 "error splitting file" - return 1 - fi - - create_multipart_upload_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}" || local create_result=$? - if [[ $create_result -ne 0 ]]; then - log 2 "error creating multpart upload" - return 1 - fi - - parts="[" - for ((i = 1; i <= $4; i++)); do - upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$? - if [[ $upload_result -ne 0 ]]; then - log 2 "error uploading part $i" - return 1 - fi - parts+="{\"ETag\": $etag, \"PartNumber\": $i}" - if [[ $i -ne $4 ]]; then - parts+="," - fi - done - parts+="]" - - export parts -} - -multipart_upload_before_completion_custom() { - if [ $# -lt 4 ]; then - log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional params" - return 1 - fi - - split_file "$3" "$4" || local split_result=$? - if [[ $split_result -ne 0 ]]; then - log 2 "error splitting file" - return 1 - fi - - # shellcheck disable=SC2086 disable=SC2048 - create_multipart_upload_custom "$1" "$2" ${*:5} || local create_result=$? - if [[ $create_result -ne 0 ]]; then - log 2 "error creating multipart upload" - return 1 - fi - log 5 "upload ID: $upload_id" - - parts="[" - for ((i = 1; i <= $4; i++)); do - upload_part "$1" "$2" "$upload_id" "$3" "$i" || local upload_result=$? - if [[ $upload_result -ne 0 ]]; then - log 2 "error uploading part $i" - return 1 - fi - parts+="{\"ETag\": $etag, \"PartNumber\": $i}" - if [[ $i -ne $4 ]]; then - parts+="," - fi - done - parts+="]" - - export parts -} - multipart_upload_custom() { if [ $# -lt 4 ]; then log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional additional params" @@ -416,15 +108,13 @@ multipart_upload_custom() { fi # shellcheck disable=SC2086 disable=SC2048 - multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5} || local result=$? - if [[ $result -ne 0 ]]; then + if ! multipart_upload_before_completion_custom "$1" "$2" "$3" "$4" ${*:5}; then log 2 "error performing pre-completion multipart upload" return 1 fi log 5 "upload ID: $upload_id, parts: $parts" - complete_multipart_upload "$1" "$2" "$upload_id" "$parts" || local completed=$? - if [[ $completed -ne 0 ]]; then + if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then log 2 "Error completing upload" return 1 fi @@ -437,14 +127,12 @@ multipart_upload() { return 1 fi - multipart_upload_before_completion "$1" "$2" "$3" "$4" || local result=$? - if [[ $result -ne 0 ]]; then + if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then log 2 "error performing pre-completion multipart upload" return 1 fi - complete_multipart_upload "$1" "$2" "$upload_id" "$parts" || local completed=$? - if [[ $completed -ne 0 ]]; then + if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then log 2 "Error completing upload" return 1 fi @@ -461,101 +149,20 @@ multipart_upload_with_params() { fi log 5 "1: $1, 2: $2, 3: $3, 4: $4, 5: $5, 6: $6, 7: $7, 8: $8, 9: $9, 10: ${10}" - multipart_upload_before_completion_with_params "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" || result=$? - if [[ $result -ne 0 ]]; then + if ! multipart_upload_before_completion_with_params "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}"; then log 2 "error performing pre-completion multipart upload" return 1 fi log 5 "Upload parts: $parts" - complete_multipart_upload "$1" "$2" "$upload_id" "$parts" || local completed=$? - if [[ $completed -ne 0 ]]; then + if ! complete_multipart_upload "$1" "$2" "$upload_id" "$parts"; then log 2 "Error completing upload" return 1 fi return 0 } -create_upload_and_get_id_rest() { - if [ $# -ne 2 ]; then - log 2 "'create_upload_and_get_id_rest' requires bucket, key" - return 1 - fi - if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY=$2 OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/create_multipart_upload.sh); then - log 2 "error creating multipart upload: $result" - return 1 - fi - if [ "$result" != "200" ]; then - log 2 "error: response code: $result, output: $(cat "$TEST_FILE_FOLDER/output.txt")" - return 1 - fi - log 5 "multipart upload create info: $(cat "$TEST_FILE_FOLDER/output.txt")" - if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/output.txt" 2>&1); then - log 2 "error getting upload ID: $upload_id" - return 1 - fi - log 5 "upload ID: $upload_id" - return 0 -} - -multipart_upload_range_too_large() { - if [ $# -ne 3 ]; then - log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location" - return 1 - fi - if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then - log 2 "multipart upload succeeded despite overly large range" - return 1 - fi - log 5 "error: $upload_part_copy_error" - if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then - log 2 "unexpected error: $upload_part_copy_error" - return 1 - fi - return 0 -} -list_and_check_upload() { - if [ $# -lt 2 ]; then - log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)" - return 1 - fi - if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then - log 2 "error listing multipart uploads before upload: $result" - return 1 - fi - if ! upload_count=$(xmllint --xpath 'count(//*[local-name()="Upload"])' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then - log 2 "error retrieving upload count: $upload_count" - return 1 - fi - if [[ (( $# == 2 ) && ( $upload_count != 0 )) ]]; then - log 2 "upload count mismatch (expected 0, actual $upload_count)" - return 1 - elif [[ (( $# == 3 ) && ( $upload_count != 1 )) ]]; then - log 2 "upload count mismatch (expected 1, actual $upload_count)" - return 1 - fi - if [ $# -eq 2 ]; then - return 0 - fi - if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then - log 2 "error retrieving key: $key" - return 1 - fi - if [ "$key" != "$2" ]; then - log 2 "key mismatch (expected '$2', actual '$key')" - return 1 - fi - if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then - log 2 "error retrieving upload ID: $upload_id" - return 1 - fi - if [ "$upload_id" != "$3" ]; then - log 2 "upload ID mismatch (expected '$3', actual '$upload_id')" - return 1 - fi - return 0 -} run_and_verify_multipart_upload_with_valid_range() { if [ $# -ne 3 ]; then @@ -582,25 +189,3 @@ run_and_verify_multipart_upload_with_valid_range() { fi return 0 } - -list_check_multipart_upload_key() { - if [ $# -ne 4 ]; then - log 2 "'list_check_multipart_upload_key' requires bucket, username, password, expected key" - return 1 - fi - if ! list_multipart_uploads_with_user "$1" "$2" "$3"; then - log 2 "error listing multipart uploads with user" - return 1 - fi - # shellcheck disable=SC2154 - log 5 "$uploads" - if ! upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1); then - log 2 "error parsing upload key from uploads message: $upload_key" - return 1 - fi - if [[ "$4" != "$upload_key" ]]; then - log 2 "upload key doesn't match file marked as being uploaded (expected: '$4', actual: '$upload_key')" - return 1 - fi - return 0 -} diff --git a/tests/util/util_multipart_before_completion.sh b/tests/util/util_multipart_before_completion.sh new file mode 100644 index 00000000..74c41633 --- /dev/null +++ b/tests/util/util_multipart_before_completion.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +create_upload_and_test_parts_listing() { + if [ $# -ne 2 ]; then + log 2 "'create_upload_and_test_parts_listing' requires test file, policy_file" + return 1 + fi + if ! create_multipart_upload_with_user "$BUCKET_ONE_NAME" "$1" "$USERNAME_ONE" "$PASSWORD_ONE"; then + log 2 "error creating multipart upload with user" + return 1 + fi + + # shellcheck disable=SC2154 + if list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then + log 2 "list parts with user succeeded despite lack of policy permissions" + return 1 + fi + + if ! setup_policy_with_single_statement "$TEST_FILE_FOLDER/$2" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:ListMultipartUploadParts" "arn:aws:s3:::$BUCKET_ONE_NAME/*"; then + log 2 "error setting up policy" + return 1 + fi + + if ! put_bucket_policy "s3api" "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/$2"; then + log 2 "error putting policy" + return 1 + fi + + if ! list_parts_with_user "$USERNAME_ONE" "$PASSWORD_ONE" "$BUCKET_ONE_NAME" "$1" "$upload_id"; then + log 2 "error listing parts after policy add" + return 1 + fi + return 0 +} + +start_multipart_upload_list_check_parts() { + if [ $# -ne 3 ]; then + log 2 "'start_multipart_upload_and_list_parts' requires bucket, key, original source" + return 1 + fi + if ! start_multipart_upload_and_list_parts "$1" "$2" "$3" 4; then + log 2 "error starting upload" + return 1 + fi + + declare -a parts_map + # shellcheck disable=SC2154 + log 5 "parts: $parts" + for i in {0..3}; do + if ! parse_parts_and_etags "$i"; then + log 2 "error parsing part $i" + return 1 + fi + done + if [[ ${#parts_map[@]} -eq 0 ]]; then + log 2 "error loading multipart upload parts to check" + return 1 + fi + + for i in {0..3}; do + if ! compare_parts_to_listed_parts "$i"; then + log 2 "error comparing parts to listed parts" + return 1 + fi + done + return 0 +} + +parse_parts_and_etags() { + if [ $# -ne 1 ]; then + log 2 "'parse_parts_and_etags' requires part id" + return 1 + fi + local part_number + local etag + # shellcheck disable=SC2154 + if ! part=$(echo "$parts" | grep -v "InsecureRequestWarning" | jq -r ".[$i]" 2>&1); then + log 2 "error getting part: $part" + return 1 + fi + if ! part_number=$(echo "$part" | jq ".PartNumber" 2>&1); then + log 2 "error parsing part number: $part_number" + return 1 + fi + if [[ $part_number == "" ]]; then + log 2 "error: blank part number" + return 1 + fi + if ! etag=$(echo "$part" | jq ".ETag" 2>&1); then + log 2 "error parsing etag: $etag" + return 1 + fi + if [[ $etag == "" ]]; then + log 2 "error: blank etag" + return 1 + fi + # shellcheck disable=SC2004 + parts_map[$part_number]=$etag +} + +compare_parts_to_listed_parts() { + if [ $# -ne 1 ]; then + log 2 "'compare_parts_to_listed_parts' requires part number" + return 1 + fi + local part_number + local etag + # shellcheck disable=SC2154 + if ! listed_part=$(echo "$listed_parts" | grep -v "InsecureRequestWarning" | jq -r ".Parts[$i]" 2>&1); then + log 2 "error parsing listed part: $listed_part" + return 1 + fi + if ! part_number=$(echo "$listed_part" | jq ".PartNumber" 2>&1); then + log 2 "error parsing listed part number: $part_number" + return 1 + fi + if ! etag=$(echo "$listed_part" | jq ".ETag" 2>&1); then + log 2 "error getting listed etag: $etag" + return 1 + fi + if [[ ${parts_map[$part_number]} != "$etag" ]]; then + log 2 "error: etags don't match (part number: $part_number, etags ${parts_map[$part_number]},$etag)" + return 1 + fi +} + +# list parts of an unfinished multipart upload +# params: bucket, key, local file location, and parts to split into before upload +# export parts on success, return 1 for error +start_multipart_upload_and_list_parts() { + if [ $# -ne 4 ]; then + log 2 "list multipart upload parts command requires bucket, key, file, and part count" + return 1 + fi + + if ! multipart_upload_before_completion "$1" "$2" "$3" "$4"; then + log 2 "error performing pre-completion multipart upload" + return 1 + fi + + if ! list_parts "$1" "$2" "$upload_id"; then + log 2 "Error listing multipart upload parts: $listed_parts" + return 1 + fi + export listed_parts +} + +create_list_check_multipart_uploads() { + if [ $# -ne 3 ]; then + log 2 "list multipart uploads command requires bucket and two keys" + return 1 + fi + if ! create_and_list_multipart_uploads "$1" "$2" "$3"; then + log 2 "error creating and listing multipart uploads" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "Uploads: $uploads" + raw_uploads=$(echo "$uploads" | grep -v "InsecureRequestWarning") + if ! key_one=$(echo "$raw_uploads" | jq -r '.Uploads[0].Key' 2>&1); then + log 2 "error getting key one: $key_one" + return 1 + fi + if ! key_two=$(echo "$raw_uploads" | jq -r '.Uploads[1].Key' 2>&1); then + log 2 "error getting key two: $key_two" + return 1 + fi + if [[ "$2" != "$key_one" ]]; then + log 2 "Key mismatch ($2, $key_one)" + return 1 + fi + if [[ "$3" != "$key_two" ]]; then + log 2 "Key mismatch ($3, $key_two)" + return 1 + fi + return 0 +} + +# list unfinished multipart uploads +# params: bucket, key one, key two +# export current two uploads on success, return 1 for error +create_and_list_multipart_uploads() { + if [ $# -ne 3 ]; then + log 2 "list multipart uploads command requires bucket and two keys" + return 1 + fi + + if ! create_multipart_upload "$1" "$2"; then + log 2 "error creating multpart upload" + return 1 + fi + + if ! create_multipart_upload "$1" "$3"; then + log 2 "error creating multpart upload two" + return 1 + fi + + if ! list_multipart_uploads "$1"; then + log 2 "error listing uploads" + return 1 + fi + return 0 +} + +# perform all parts of a multipart upload before completion command +# params: bucket, key, file to split and upload, number of file parts to upload +# return: 0 for success, 1 for failure +multipart_upload_before_completion() { + if [ $# -ne 4 ]; then + log 2 "multipart upload pre-completion command missing bucket, key, file, and/or part count" + return 1 + fi + + if ! split_file "$3" "$4"; then + log 2 "error splitting file" + return 1 + fi + + if ! create_multipart_upload "$1" "$2"; then + log 2 "error creating multpart upload" + return 1 + fi + + parts="[" + for ((i = 1; i <= $4; i++)); do + # shellcheck disable=SC2154 + if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then + log 2 "error uploading part $i" + return 1 + fi + parts+="{\"ETag\": $etag, \"PartNumber\": $i}" + if [[ $i -ne $4 ]]; then + parts+="," + fi + done + parts+="]" + + export parts +} + +multipart_upload_before_completion_with_params() { + if [ $# -ne 10 ]; then + log 2 "multipart upload command missing bucket, key, file, part count, content type, metadata, hold status, lock mode, retain until date, tagging" + return 1 + fi + + if ! result=$(split_file "$3" "$4" 2>&1); then + log 2 "error splitting file: $result" + return 1 + fi + + if ! create_multipart_upload_params "$1" "$2" "$5" "$6" "$7" "$8" "$9" "${10}"; then + log 2 "error creating multpart upload" + return 1 + fi + + parts="[" + for ((i = 1; i <= $4; i++)); do + if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then + log 2 "error uploading part $i" + return 1 + fi + parts+="{\"ETag\": $etag, \"PartNumber\": $i}" + if [[ $i -ne $4 ]]; then + parts+="," + fi + done + parts+="]" + + export parts +} + +multipart_upload_before_completion_custom() { + if [ $# -lt 4 ]; then + log 2 "multipart upload custom command missing bucket, key, file, part count, and/or optional params" + return 1 + fi + + if ! result=$(split_file "$3" "$4" 2>&1); then + log 2 "error splitting file" + return 1 + fi + + # shellcheck disable=SC2086 disable=SC2048 + if ! create_multipart_upload_custom "$1" "$2" ${*:5}; then + log 2 "error creating multipart upload" + return 1 + fi + log 5 "upload ID: $upload_id" + + parts="[" + for ((i = 1; i <= $4; i++)); do + if ! upload_part "$1" "$2" "$upload_id" "$3" "$i"; then + log 2 "error uploading part $i" + return 1 + fi + parts+="{\"ETag\": $etag, \"PartNumber\": $i}" + if [[ $i -ne $4 ]]; then + parts+="," + fi + done + parts+="]" + + export parts +} + +create_upload_and_get_id_rest() { + if [ $# -ne 2 ]; then + log 2 "'create_upload_and_get_id_rest' requires bucket, key" + return 1 + fi + if ! result=$(COMMAND_LOG=$COMMAND_LOG BUCKET_NAME=$1 OBJECT_KEY=$2 OUTPUT_FILE="$TEST_FILE_FOLDER/output.txt" ./tests/rest_scripts/create_multipart_upload.sh); then + log 2 "error creating multipart upload: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "error: response code: $result, output: $(cat "$TEST_FILE_FOLDER/output.txt")" + return 1 + fi + log 5 "multipart upload create info: $(cat "$TEST_FILE_FOLDER/output.txt")" + if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/output.txt" 2>&1); then + log 2 "error getting upload ID: $upload_id" + return 1 + fi + log 5 "upload ID: $upload_id" + return 0 +} + +multipart_upload_range_too_large() { + if [ $# -ne 3 ]; then + log 2 "'multipart_upload_range_too_large' requires bucket name, key, file location" + return 1 + fi + if multipart_upload_from_bucket_range "$1" "$2" "$3" 4 "bytes=0-1000000000"; then + log 2 "multipart upload succeeded despite overly large range" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "error: $upload_part_copy_error" + if [[ $upload_part_copy_error != *"Range specified is not valid"* ]] && [[ $upload_part_copy_error != *"InvalidRange"* ]]; then + log 2 "unexpected error: $upload_part_copy_error" + return 1 + fi + return 0 +} + +list_and_check_upload() { + if [ $# -lt 2 ]; then + log 2 "'list_and_check_upload' requires bucket, key, upload ID (optional)" + return 1 + fi + if ! uploads=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/uploads.txt" ./tests/rest_scripts/list_multipart_uploads.sh); then + log 2 "error listing multipart uploads before upload: $result" + return 1 + fi + if ! upload_count=$(xmllint --xpath 'count(//*[local-name()="Upload"])' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then + log 2 "error retrieving upload count: $upload_count" + return 1 + fi + if [[ (( $# == 2 ) && ( $upload_count != 0 )) ]]; then + log 2 "upload count mismatch (expected 0, actual $upload_count)" + return 1 + elif [[ (( $# == 3 ) && ( $upload_count != 1 )) ]]; then + log 2 "upload count mismatch (expected 1, actual $upload_count)" + return 1 + fi + if [ $# -eq 2 ]; then + return 0 + fi + if ! key=$(xmllint --xpath '//*[local-name()="Key"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then + log 2 "error retrieving key: $key" + return 1 + fi + if [ "$key" != "$2" ]; then + log 2 "key mismatch (expected '$2', actual '$key')" + return 1 + fi + if ! upload_id=$(xmllint --xpath '//*[local-name()="UploadId"]/text()' "$TEST_FILE_FOLDER/uploads.txt" 2>&1); then + log 2 "error retrieving upload ID: $upload_id" + return 1 + fi + if [ "$upload_id" != "$3" ]; then + log 2 "upload ID mismatch (expected '$3', actual '$upload_id')" + return 1 + fi + return 0 +} + +list_check_multipart_upload_key() { + if [ $# -ne 4 ]; then + log 2 "'list_check_multipart_upload_key' requires bucket, username, password, expected key" + return 1 + fi + if ! list_multipart_uploads_with_user "$1" "$2" "$3"; then + log 2 "error listing multipart uploads with user" + return 1 + fi + # shellcheck disable=SC2154 + log 5 "$uploads" + if ! upload_key=$(echo "$uploads" | grep -v "InsecureRequestWarning" | jq -r ".Uploads[0].Key" 2>&1); then + log 2 "error parsing upload key from uploads message: $upload_key" + return 1 + fi + if [[ "$4" != "$upload_key" ]]; then + log 2 "upload key doesn't match file marked as being uploaded (expected: '$4', actual: '$upload_key')" + return 1 + fi + return 0 +}