test: more updates

tests/commands/list_object_versions.sh

@@ -34,30 +34,10 @@ list_object_versions_rest() {
     log 2 "'list_object_versions_rest' requires bucket name"
     return 1
   fi
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-versions=
-host:${AWS_ENDPOINT_URL#*//}
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "list object versions REST"
+  if ! result=$(BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object_versions.txt" ./tests/rest_scripts/list_object_versions.sh); then
+    log 2 "error listing object versions: $result"
     return 1
   fi
-
-  get_signature
-  # shellcheck disable=SC2034,SC2154
-  reply=$(send_command curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
-         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-         -H "x-amz-content-sha256: $payload_hash" \
-         -H "x-amz-date: $current_date_time" \
-         -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
+  return 0
 }

tests/rest_scripts/list_object_versions.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+# shellcheck disable=SC2034
+canonical_request="GET
+/$bucket_name
+versions=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "https://$host/$bucket_name?versions"
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/list_objects.sh

@@ -22,6 +22,7 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 version_two="${VERSION_TWO:-FALSE}"
 max_keys="${MAX_KEYS:-0}"
+marker="$MARKER"
 # shellcheck disable=SC2153
 if [ "$CONTINUATION_TOKEN" != "" ]; then
   continuation_token=$(jq -rn --arg token "$CONTINUATION_TOKEN" '$token | @uri')
@@ -34,6 +35,9 @@ canonical_request="GET
 /$bucket_name
 "
 
+if [ "$MARKER" != "" ]; then
+  add_parameter "canonical_request" "marker=$marker"
+fi
 if [ "$CONTINUATION_TOKEN" != "" ]; then
   add_parameter "canonical_request" "continuation-token=$continuation_token"
 fi
@@ -56,6 +60,9 @@ create_canonical_hash_sts_and_signature
 
 curl_command+=(curl -ks -w "\"%{http_code}\"")
 url="'$AWS_ENDPOINT_URL/$bucket_name"
+if [ "$MARKER" != "" ]; then
+  add_parameter "url" "marker=$marker"
+fi
 if [ "$CONTINUATION_TOKEN" != "" ]; then
   add_parameter "url" "continuation-token=$continuation_token"
 fi

tests/test_rest.sh

@@ -441,7 +441,7 @@ export RUN_USERS=true
 
 @test "REST - list objects v2 - invalid continuation token" {
   if [ "$DIRECT" != "true" ]; then
-    skip "https://github.com/versity/versitygw/issues/993"
+    skip "https://github.com/versity/versitygw/issues/995"
   fi
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
@@ -461,11 +461,30 @@ export RUN_USERS=true
   run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_three" "$BUCKET_ONE_NAME" "$test_file_three"
   assert_success
 
-  run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two"
+  run list_objects_check_params_get_token "$BUCKET_ONE_NAME" "$test_file" "$test_file_two" "TRUE"
   assert_success
   continuation_token=$output
 
   # interestingly, AWS appears to accept continuation tokens that are a few characters off, so have to remove three chars
   run list_objects_check_continuation_error "$BUCKET_ONE_NAME" "${continuation_token:0:${#continuation_token}-3}"
   assert_success
 }
+
+@test "REST - list objects v1 - initial marker empty" {
+  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  test_file="test_file"
+  test_file_two="test_file_2"
+  run create_test_files "$test_file" "$test_file_two"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file_two" "$BUCKET_ONE_NAME" "$test_file_two"
+  assert_success
+
+  run list_objects_v1_check_initial_marker_empty "$BUCKET_ONE_NAME"
+  assert_success
+}

tests/util/util_list_objects.sh

@@ -246,11 +246,11 @@ list_objects_with_user_rest_verify_success() {
 }
 
 list_objects_check_params_get_token() {
-  if [ $# -ne 3 ]; then
-    log 2 "'list_objects_check_params_get_token' requires bucket name, files"
+  if [ $# -ne 4 ]; then
+    log 2 "'list_objects_check_params_get_token' requires bucket name, files, version two"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="TRUE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="$4" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
     log 2 "error attempting to get bucket ACL response: $result"
     return 1
   fi
@@ -300,4 +300,26 @@ list_objects_check_continuation_error() {
     log 2 "invalid error code"
     return 1
   fi
-}
+}
+
+list_objects_v1_check_initial_marker_empty() {
+  if [ $# -ne 1 ]; then
+    log 2 "'get_next_objects_v1' requires bucket name"
+    return 1
+  fi
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" VERSION_TWO="FALSE" MAX_KEYS=1 OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error attempting to get bucket ACL response: $result"
+    return 1
+  fi
+  log 5 "output: $(cat "$TEST_FILE_FOLDER/objects.txt")"
+  if ! marker=$(xmllint --xpath '//*[local-name()="Marker"]' "$TEST_FILE_FOLDER/objects.txt" 2>&1); then
+    log 2 "error obtaining marker field: $marker"
+    return 1
+  fi
+  marker_text=$(xmllint --xpath 'text()' <(cat "$marker") 2>&1)
+  if [[ "$marker_text" != *"Document is empty"* ]]; then
+    log 2 "marker text should be empty, but is $marker_text"
+    return 1
+  fi
+  return 0
+}