test: PutBucketAcl with canned public-read, logging fix

tests/README.md

@@ -1,5 +1,19 @@
 # Command-Line Tests
 
+## Table of Contents
+
+[Instructions - Running Locally](#instructions---running-locally)<br>
+[* Posix Backend](#posix-backend)<br>
+[* Static Bucket Mode](#static-bucket-mode)<br>
+[* S3 Backend](#s3-backend)<br>
+[* Direct Mode](#direct-mode)<br>
+[Instructions - Running With Docker](#instructions---running-with-docker)<br>
+[Instructions - Running With Docker-Compose](#instructions---running-with-docker-compose)<br>
+[Environment Parameters](#environment-parameters)<br>
+[* Secret](#secret)<br>
+[* Non-Secret](#non-secret)<br>
+[REST Scripts](#rest-scripts)<br>
+
 ## Instructions - Running Locally
 
 ### Posix Backend
@@ -85,8 +99,18 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 
 ## Environment Parameters
 
+### Secret
+
 **AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**:  identical to the same parameters in **s3**.
 
+**AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the main user (owner)
+
+**ACL_AWS_CANONICAL_ID**:  for direct mode, the canonical ID for the user to test ACL changes and access by non-owners
+
+**ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**:  for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
+
+### Non-Secret
+
 **VERSITY_EXE**:  location of the versity executable relative to test folder.
 
 **RUN_VERSITYGW**:  whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
@@ -134,3 +158,21 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
 **VERSIONING_DIR**:  where to put gateway file versioning info.
 
 **COMMAND_LOG**:  where to store list of client commands, which if using will be reported during test failures.
+
+**TIME_LOG**:  optional log to show duration of individual tests
+
+**DIRECT_S3_ROOT_ACCOUNT_NAME**:  for direct mode, S3 username
+
+**DELETE_BUCKETS_AFTER_TEST**:  whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
+
+## REST Scripts
+
+REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder.  To call a script, the following parameters are needed:
+* **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**, etc.
+* **AWS_ENDPOINT_URL** (default:  `https://localhost:7070`)
+* **OUTPUT_FILE**:  file where the command's response data is written
+* Any other parameters specified at the top of the script file, such as payloads and variables.  Sometimes, defaults are included.
+
+Upon success, the script will return a response code, and write the data to the **OUTPUT_FILE** location.
+
+Example:  `AWS_ACCESS_KEY_ID={id} AWS_SECRET_ACCESS_KEY={key} AWS_ENDPOINT_URL=https://s3.amazonaws.com OUTPUT_FILE=./output_file.xml ./tests/rest_scripts/list_buckets.sh`

tests/commands/put_bucket_acl.sh

@@ -59,11 +59,11 @@ reset_bucket_acl() {
   fi
   # shellcheck disable=SC2154
   if [ "$DIRECT" != "true" ]; then
-    if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
+    if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
       log 2 "error resetting versitygw ACL"
       return 1
     fi
-  elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
+  elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
     log 2 "error resetting direct ACL"
     return 1
   fi

tests/logger.sh

@@ -55,31 +55,46 @@ log_mask() {
     return 1
   fi
 
-  log_message "$log_level" "${masked_args[*]}"
+  log_message "$log_level" "$masked_data"
 }
 
 mask_args() {
   if [ $# -ne 1 ]; then
     echo "'mask_args' requires string"
     return 1
   fi
-  IFS=' ' read -r -a array <<< "$1"
+  unmasked_array=()
+  masked_data=""
+  while IFS= read -r line; do
+    unmasked_array+=("$line")
+  done <<< "$1"
 
-  if ! mask_arg_array "${array[@]}"; then
-    echo "error masking arg array"
-    return 1
-  fi
+  # shellcheck disable=SC2068
+  first_line=true
+  for line in "${unmasked_array[@]}"; do
+    if ! mask_arg_array "$line"; then
+      echo "error masking arg array"
+      return 1
+    fi
+    if [ "$first_line" == "true" ]; then
+      masked_data="${masked_args[*]}"
+      first_line="false"
+    else
+      masked_data+=$(printf "\n%s" "${masked_args[*]}")
+    fi
+  done
 }
 
 mask_arg_array() {
-  masked_args=()  # Initialize an array to hold the masked arguments
   if [ $# -eq 0 ]; then
     echo "'mask_arg_array' requires parameters"
     return 1
   fi
   mask_next=false
   is_access=false
-  for arg in "$@"; do
+  masked_args=()  # Initialize an array to hold the masked arguments
+  # shellcheck disable=SC2068
+  for arg in $@; do
     if ! check_arg_for_mask "$arg"; then
       echo "error checking arg for mask"
       return 1
@@ -135,4 +150,5 @@ log_message() {
   if [[ -n "$TEST_LOG_FILE" ]]; then
     echo "$now $1 $2" >> "$TEST_LOG_FILE.tmp"
   fi
+  sync
 }

tests/rest_scripts/get_public_access_block.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="GET
+/$bucket_name
+publicAccessBlock=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/put_bucket_acl.sh

@@ -22,8 +22,14 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2153
 acl_file="$ACL_FILE"
+# shellcheck disable=SC2153
+canned_acl="$CANNED_ACL"
 
-payload="$(cat "$acl_file")"
+if [ -n "$ACL_FILE" ]; then
+  payload="$(cat "$acl_file")"
+else
+  payload=""
+fi
 
 payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
@@ -32,20 +38,39 @@ canonical_request="PUT
 /$bucket_name
 acl=
 host:$host
-x-amz-content-sha256:$payload_hash
+"
+if [ -n "$CANNED_ACL" ]; then
+  canonical_request+="x-amz-acl:$canned_acl
+"
+fi
+canonical_request+="x-amz-content-sha256:$payload_hash
 x-amz-date:$current_date_time
 
-host;x-amz-content-sha256;x-amz-date
+"
+canonical_request+="host;"
+if [ -n "$CANNED_ACL" ]; then
+  canonical_request+="x-amz-acl;"
+fi
+canonical_request+="x-amz-content-sha256;x-amz-date
 $payload_hash"
 
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl="
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"x-amz-content-sha256: $payload_hash\""
--H "\"x-amz-date: $current_date_time\""
--d "\"${payload//\"/\\\"}\""
--o "$OUTPUT_FILE")
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=")
+if [ -n "$CANNED_ACL" ]; then
+  curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-acl;x-amz-content-sha256;x-amz-date,Signature=$signature\"")
+else
+  curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"")
+fi
+if [ -n "$CANNED_ACL" ]; then
+  curl_command+=(-H "\"x-amz-acl: $canned_acl\"")
+fi
+curl_command+=(-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\"")
+if [ -n "$ACL_FILE" ]; then
+  curl_command+=(-d "\"${payload//\"/\\\"}\"")
+fi
+curl_command+=(-o "$OUTPUT_FILE")
 
 # shellcheck disable=SC2154
 eval "${curl_command[*]}" 2>&1

tests/rest_scripts/put_public_access_block.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+block_public_acls="${BLOCK_PUBLIC_ACLS:-TRUE}"
+ignore_public_acls="${IGNORE_PUBLIC_ACLS:-TRUE}"
+block_public_policy="${BLOCK_PUBLIC_POLICY:-TRUE}"
+restrict_public_buckets="${RESTRICT_PUBLIC_BUCKETS:-TRUE}"
+
+payload="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<PublicAccessBlockConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+      <BlockPublicAcls>$block_public_acls</BlockPublicAcls>
+      <IgnorePublicAcls>$ignore_public_acls</IgnorePublicAcls>
+      <BlockPublicPolicy>$block_public_policy</BlockPublicPolicy>
+      <RestrictPublicBuckets>$restrict_public_buckets</RestrictPublicBuckets>
+</PublicAccessBlockConfiguration>"
+
+payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="PUT
+/$bucket_name
+publicAccessBlock=
+host:$host
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\""
+-d "\"${payload//\"/\\\"}\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1

tests/setup.sh

@@ -86,6 +86,18 @@ teardown() {
       log 3 "unable to remove test file folder: $error"
     fi
   fi
+  stop_versity
+  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
+    end_time=$(date +%s)
+    total_time=$((end_time - start_time))
+    log 4 "Total test time: $total_time"
+    if [[ -n "$TIME_LOG" ]]; then
+      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
+    fi
+  fi
+  if [[ -n "$COVERAGE_DB" ]]; then
+    record_result
+  fi
   if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then
     if [[ -e "$COMMAND_LOG" ]]; then
       cat "$COMMAND_LOG"
@@ -108,16 +120,4 @@ teardown() {
       log 2 "error deleting temp log"
     fi
   fi
-  stop_versity
-  if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
-    end_time=$(date +%s)
-    total_time=$((end_time - start_time))
-    log 4 "Total test time: $total_time"
-    if [[ -n "$TIME_LOG" ]]; then
-      echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
-    fi
-  fi
-  if [[ -n "$COVERAGE_DB" ]]; then
-    record_result
-  fi
 }