test: REST ListObjects v1 multi-page test

tests/commands/list_object_versions.sh

@@ -34,30 +34,10 @@ list_object_versions_rest() {
     log 2 "'list_object_versions_rest' requires bucket name"
     return 1
   fi
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-versions=
-host:${AWS_ENDPOINT_URL#*//}
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "list object versions REST"
+  if ! result=$(BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object_versions.txt" ./tests/rest_scripts/list_object_versions.sh); then
+    log 2 "error listing object versions: $result"
     return 1
   fi
-
-  get_signature
-  # shellcheck disable=SC2034,SC2154
-  reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
-         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-         -H "x-amz-content-sha256: $payload_hash" \
-         -H "x-amz-date: $current_date_time" \
-         -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
+  return 0
 }

tests/rest_scripts/list_object_versions.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+# shellcheck disable=SC2034
+canonical_request="GET
+/$bucket_name
+versions=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions"
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/list_objects.sh

@@ -23,6 +23,8 @@ bucket_name="$BUCKET_NAME"
 version_two="${VERSION_TWO:-FALSE}"
 max_keys="${MAX_KEYS:-0}"
 # shellcheck disable=SC2153
+marker="$MARKER"
+# shellcheck disable=SC2153
 if [ "$CONTINUATION_TOKEN" != "" ]; then
   continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri')
 fi
@@ -34,6 +36,9 @@ canonical_request="GET
 /$bucket_name
 "
 
+if [ "$MARKER" != "" ]; then
+  add_parameter "canonical_request" "marker=$marker"
+fi
 if [ "$CONTINUATION_TOKEN" != "" ]; then
   add_parameter "canonical_request" "continuation-token=$continuation_token"
 fi
@@ -56,6 +61,9 @@ create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"")
 url="'$AWS_ENDPOINT_URL/$bucket_name"
+if [ "$MARKER" != "" ]; then
+  add_parameter "url" "marker=$marker"
+fi
 if [ "$CONTINUATION_TOKEN" != "" ]; then
   add_parameter "url" "continuation-token=$continuation_token"
 fi

tests/test_common_acl.sh

@@ -66,19 +66,21 @@ test_common_put_bucket_acl() {
     grantee_type="Group"
     grantee_id="http://acs.amazonaws.com/groups/global/AllUsers"
   else
-    grantee_type="ID"
+    grantee_type="CanonicalUser"
     grantee_id="$USERNAME_ONE"
   fi
   run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "$grantee_type" "$grantee_id" "READ" "$AWS_ACCESS_KEY_ID"
   assert_success
 
+  log 5 "acl: $(cat "$TEST_FILE_FOLDER/$acl_file")"
+
   run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file"
   assert_success
 
   run get_check_acl_after_first_put "$1" "$BUCKET_ONE_NAME"
   assert_success
 
-  run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
+  run setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$USERNAME_ONE" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"
   assert_success
 
   run put_bucket_acl_s3api "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER"/"$acl_file"

tests/test_rest.sh

@@ -36,6 +36,7 @@ source ./tests/util/util_list_buckets.sh
 source ./tests/util/util_list_objects.sh
 source ./tests/util/util_list_parts.sh
 source ./tests/util/util_lock_config.sh
+source ./tests/util/util_multipart_before_completion.sh
 source ./tests/util/util_ownership.sh
 source ./tests/util/util_policy.sh
 source ./tests/util/util_public_access_block.sh
@@ -318,7 +319,7 @@ export RUN_USERS=true
 
 @test "REST - get object attributes" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/916"
+    skip "https://github.com/versity/versitygw/issues/1000"
   fi
   test_file="test_file"
 
@@ -340,7 +341,7 @@ export RUN_USERS=true
 
 @test "REST - attributes - invalid param" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/917"
+    skip "https://github.com/versity/versitygw/issues/1001"
   fi
   test_file="test_file"
 
@@ -359,7 +360,7 @@ export RUN_USERS=true
 
 @test "REST - attributes - checksum" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/928"
+    skip "https://github.com/versity/versitygw/issues/1006"
   fi
   test_file="test_file"
 
@@ -461,11 +462,33 @@ export RUN_USERS=true
   run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three"
   assert_success
 
-  run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
+  run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "TRUE"
   assert_success
   continuation_token=$output
 
   # interestingly, AWS appears to accept continuation tokens that are a few characters off, so have to remove three chars
   run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}"
   assert_success
 }
+
+@test "REST - list objects v1 - no NextMarker without delimiter" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/999"
+  fi
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  test_file_two="test_file_2"
+  run create_test_files "$test_file" "$test_file_two"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two"
+  assert_success
+
+  run list_objects_v1_check_nextmarker_empty "$BUCKET_ONE_NAME"
+  assert_success
+}

tests/test_s3api_multipart.sh

@@ -19,6 +19,7 @@ source ./tests/test_s3api_root_inner.sh
 source ./tests/util/util_file.sh
 source ./tests/util/util_multipart.sh
 source ./tests/util/util_multipart_abort.sh
+source ./tests/util/util_multipart_before_completion.sh
 source ./tests/util/util_tags.sh
 source ./tests/commands/get_object.sh
 source ./tests/commands/put_object.sh

tests/test_s3api_policy.sh

@@ -21,6 +21,7 @@ source ./tests/test_s3api_policy_multipart.sh
 source ./tests/test_s3api_policy_object.sh
 source ./tests/util/util_multipart.sh
 source ./tests/util/util_multipart_abort.sh
+source ./tests/util/util_multipart_before_completion.sh
 source ./tests/util/util_file.sh
 source ./tests/util/util_policy.sh
 source ./tests/util/util_tags.sh

tests/util/util_attributes.sh

@@ -25,6 +25,7 @@ check_attributes_after_upload() {
     log 2 "'check_attributes_after_upload' requires file size"
     return 1
   fi
+  log 5 "attributes: $(cat "$TEST_FILE_FOLDER/attributes.txt")"
   if ! object_size=$(xmllint --xpath '//*[local-name()="ObjectSize"]/text()' "$TEST_FILE_FOLDER/attributes.txt" 2>&1); then
     log 2 "error getting checksum: $object_size"
     return 1
@@ -54,10 +55,11 @@ check_attributes_after_upload() {
     log 2 "unexpected parts count, expected 4, was $parts_count"
     return 1
   fi
+  return 1
 }
 
 check_attributes_invalid_param() {
-  if [ "$1" -ne 1 ]; then
+  if [ $# -ne 1 ]; then
     log 2 "'check_attributes_invalid_param' requires test file"
     return 1
   fi

tests/util/util_list_objects.sh

@@ -246,11 +246,11 @@ list_objects_with_user_rest_verify_success() {
 }
 
 list_objects_check_params_get_token() {
-  if [ $# -ne 3 ]; then
-    log 2 "'list_objects_check_params_get_token' requires bucket name, files"
+  if [ $# -ne 4 ]; then
+    log 2 "'list_objects_check_params_get_token' requires bucket name, files, version two"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
     log 2 "error attempting to get bucket ACL response: $result"
     return 1
   fi
@@ -300,4 +300,31 @@ list_objects_check_continuation_error() {
     log 2 "invalid error code"
     return 1
   fi
-}
+}
+
+list_objects_v1_check_nextmarker_empty() {
+  if [ $# -ne 1 ]; then
+    log 2 "'get_next_objects_v1' requires bucket name"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error attempting to get bucket ACL response: $result"
+    return 1
+  fi
+  log 5 "output: $(cat "$TEST_FILE_FOLDER/objects.txt")"
+  if ! next_marker=$(xmllint --xpath '//*[local-name()="NextMarker"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
+    if [[ "$next_marker" != *"XPath set is empty"* ]]; then
+      log 2 "unexpected error: $next_marker"
+      return 1
+    fi
+    return 0
+  fi
+  log 5 "next marker: $next_marker"
+  marker_text=$(xmllint --xpath 'string(/NextMarker)' <(echo "$next_marker") 2>&1)
+  log 5 "marker text: $marker_text"
+  if [[ "$marker_text" != *"Document is empty"* ]]; then
+    log 2 "NextMarker text should be empty, but is $marker_text"
+    return 1
+  fi
+  return 0
+}