test: UploadPartCopy

tests/commands/command.sh

@@ -17,6 +17,7 @@
 source ./tests/logger.sh
 
 send_command() {
+  echo "command" >> "temp.txt"
   if [ $# -eq 0 ]; then
     return 1
   fi
@@ -27,6 +28,7 @@ send_command() {
     fi
     # shellcheck disable=SC2154
     echo "${masked_args[*]}" >> "$COMMAND_LOG"
+    echo "${masked_args[*]}" >> "temp.txt"
     "$@"
     return $?
   fi

tests/rest_scripts/rest.sh

@@ -37,11 +37,13 @@ fi
 
 add_command_recording_if_enabled() {
   if [ -n "$COMMAND_LOG" ]; then
+    echo "add" >> "temp.txt"
     curl_command+=(send_command)
   fi
 }
 
 create_canonical_hash_sts_and_signature() {
+  echo "start" >> "temp.txt"
   # shellcheck disable=SC2154
   canonical_request_hash="$(echo -n "$canonical_request" | openssl dgst -sha256 | awk '{print $2}')"
 
@@ -64,6 +66,7 @@ $canonical_request_hash"
 
   curl_command=()
   add_command_recording_if_enabled
+  echo "end" >> "temp.txt"
 }
 
 add_parameter() {

tests/rest_scripts/upload_part.sh

@@ -46,8 +46,6 @@ $payload_hash"
 
 create_canonical_hash_sts_and_signature
 
-sleep 5
-
 curl_command+=(curl -isk -w "\"%{http_code}\"" "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\""
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
 -H "\"x-amz-content-sha256: $payload_hash\""

tests/rest_scripts/upload_part_copy.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+# shellcheck disable=SC2153
+key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+part_number="$PART_NUMBER"
+# shellcheck disable=SC2153
+upload_id="$UPLOAD_ID"
+# shellcheck disable=SC2153
+part_location=$PART_LOCATION
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+# shellcheck disable=SC2034
+header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+# shellcheck disable=SC2154
+canonical_request="PUT
+/$bucket_name/$key
+partNumber=$part_number&uploadId=$upload_id
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-copy-source:$part_location
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-copy-source;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "\"$AWS_ENDPOINT_URL/$bucket_name/$key?partNumber=$part_number&uploadId=$upload_id\""
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-copy-source;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-copy-source: $part_location\""
+-H "\"x-amz-date: $current_date_time\""
+-o "\"$OUTPUT_FILE\"")
+echo "${curl_command[*]}" >> "temp.txt"
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/setup.sh

@@ -14,9 +14,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-load ./bats-support/load
-load ./bats-assert/load
-
 source ./tests/env.sh
 source ./tests/report.sh
 source ./tests/setup_mc.sh
@@ -68,11 +65,14 @@ delete_temp_log_if_exists() {
 # bats teardown function
 teardown() {
   # shellcheck disable=SC2154
-  if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
-    log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
-  fi
-  if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then
-    log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
+  if [ "$DELETE_BUCKETS_AFTER_TEST" != "false" ]; then
+    log 5 "deleting or clearing buckets"
+    if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+      log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
+    fi
+    if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then
+      log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
+    fi
   fi
   if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
     log 3 "error deleting user $USERNAME_ONE"

tests/teardown_static.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2024 Versity Software
 # This file is licensed under the Apache License, Version 2.0
@@ -21,12 +21,14 @@ if ! base_setup; then
   log 2 "error starting versity to set up static buckets"
   exit 1
 fi
-if ! delete_bucket_recursive "s3" "$BUCKET_ONE_NAME"; then
-  log 2 "error creating static bucket one"
-elif ! delete_bucket_recursive "s3" "$BUCKET_TWO_NAME"; then
-  log 2 "error creating static bucket two"
+#if ! delete_bucket_recursive "s3api" "$BUCKET_ONE_NAME"; then
+if ! clear_bucket_s3api "$BUCKET_ONE_NAME"; then
+  log 2 "error deleting static bucket one"
+elif ! delete_bucket_recursive "s3api" "$BUCKET_TWO_NAME"; then
+  log 2 "error deleting static bucket two"
+else
+  log 4 "buckets deleted successfully"
 fi
-log 4 "buckets deleted successfully"
 if ! stop_versity; then
   log 2 "error stopping versity"
 fi

tests/test_mc.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 source ./tests/setup.sh
 source ./tests/util/util_create_bucket.sh

tests/test_mc_file_count.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 
 export RUN_MC=true

tests/test_rest.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/commands/create_multipart_upload.sh
 source ./tests/commands/delete_object_tagging.sh
 source ./tests/commands/get_bucket_versioning.sh
@@ -383,9 +386,6 @@ export RUN_USERS=true
 }
 
 @test "REST - bucket tagging - tags" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/932"
-  fi
   test_key="testKey"
   test_value="testValue"
 
@@ -491,4 +491,34 @@ export RUN_USERS=true
 
   run list_objects_v1_check_nextmarker_empty "$BUCKET_ONE_NAME"
   assert_success
-}
+}
+
+@test "REST - complete upload - invalid part" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1008"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  run create_large_file "$test_file"
+  assert_success
+
+  run create_upload_finish_wrong_etag "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - upload part copy" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  run create_large_file "$test_file"
+  assert_success
+
+  run create_upload_part_copy_rest "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  assert_success
+
+  run download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
+  assert_success
+}

tests/test_rest_acl.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/commands/put_object.sh
 source ./tests/logger.sh
 source ./tests/setup.sh
@@ -23,9 +26,6 @@ source ./tests/util/util_acl.sh
 export RUN_USERS=true
 
 @test "REST - get ACL" {
-  if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/971"
-  fi
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
 

tests/test_s3.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 source ./tests/util/util_file.sh
 

tests/test_s3_file_count.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 source ./tests/util/util_file.sh
 

tests/test_s3api_bucket.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/util/util.sh
 source ./tests/util/util_create_bucket.sh

tests/test_s3api_multipart.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/test_s3api_root_inner.sh
 source ./tests/util/util_file.sh

tests/test_s3api_object.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/util/util.sh
 source ./tests/util/util_create_bucket.sh

tests/test_s3api_policy.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/logger.sh
 source ./tests/setup.sh
 source ./tests/test_s3api_policy_bucket.sh

tests/test_s3cmd.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/setup.sh
 source ./tests/test_common.sh
 source ./tests/test_common_acl.sh

tests/test_s3cmd_file_count.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_common.sh
 
 export RUN_S3CMD=true

tests/test_user_aws.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_user_common.sh
 source ./tests/util/util_users.sh
 source ./tests/commands/get_object.sh

tests/test_user_s3cmd.sh

@@ -14,6 +14,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+load ./bats-support/load
+load ./bats-assert/load
+
 source ./tests/test_user_common.sh
 
 export RUN_S3CMD=true

tests/util/util_bucket.sh

@@ -211,16 +211,12 @@ bucket_cleanup() {
 # return 0 for success, 1 for error
 bucket_cleanup_if_bucket_exists() {
   log 6 "bucket_cleanup_if_bucket_exists"
-  if [ $# -ne 2 ]; then
-    log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name"
+  if [ $# -lt 2 ]; then
+    log 2 "'bucket_cleanup_if_bucket_exists' requires client, bucket name, bucket known to exist (optional)"
     return 1
   fi
 
-  if bucket_exists "$1" "$2"; then
-    if [ "$DELETE_BUCKETS_AFTER_TEST" == "false" ]; then
-      log 2 "skipping bucket cleanup/deletion"
-      return 0
-    fi
+  if [ "$3" == "true" ] || bucket_exists "$1" "$2"; then
     if ! bucket_cleanup "$1" "$2"; then
       log 2 "error deleting bucket and/or contents"
       return 1
@@ -256,12 +252,16 @@ setup_bucket() {
     return 1
   fi
 
-  if ! bucket_exists "$1" "$2" && [[ $RECREATE_BUCKETS == "false" ]]; then
-    log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
-    return 1
+  bucket_exists="true"
+  if ! bucket_exists "$1" "$2"; then
+    if [[ $RECREATE_BUCKETS == "false" ]]; then
+      log 2 "When RECREATE_BUCKETS isn't set to \"true\", buckets should be pre-created by user"
+      return 1
+    fi
+    bucket_exists="false"
   fi
 
-  if ! bucket_cleanup_if_bucket_exists "$1" "$2"; then
+  if ! bucket_cleanup_if_bucket_exists "$1" "$2" "$bucket_exists"; then
     log 2 "error deleting bucket or contents if they exist"
     return 1
   fi