diff --git a/tests/README.md b/tests/README.md
index 30ab7431..8c784d9c 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,5 +1,19 @@
# Command-Line Tests
+## Table of Contents
+
+[Instructions - Running Locally](#instructions---running-locally)
+[* Posix Backend](#posix-backend)
+[* Static Bucket Mode](#static-bucket-mode)
+[* S3 Backend](#s3-backend)
+[* Direct Mode](#direct-mode)
+[Instructions - Running With Docker](#instructions---running-with-docker)
+[Instructions - Running With Docker-Compose](#instructions---running-with-docker-compose)
+[Environment Parameters](#environment-parameters)
+[* Secret](#secret)
+[* Non-Secret](#non-secret)
+[REST Scripts](#rest-scripts)
+
## Instructions - Running Locally
### Posix Backend
@@ -85,8 +99,18 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
## Environment Parameters
+### Secret
+
**AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**: identical to the same parameters in **s3**.
+**AWS_CANONICAL_ID**: for direct mode, the canonical ID for the main user (owner)
+
+**ACL_AWS_CANONICAL_ID**: for direct mode, the canonical ID for the user to test ACL changes and access by non-owners
+
+**ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**: for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account.
+
+### Non-Secret
+
**VERSITY_EXE**: location of the versity executable relative to test folder.
**RUN_VERSITYGW**: whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**.
@@ -134,3 +158,21 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up
**VERSIONING_DIR**: where to put gateway file versioning info.
**COMMAND_LOG**: where to store list of client commands, which if using will be reported during test failures.
+
+**TIME_LOG**: optional log to show duration of individual tests
+
+**DIRECT_S3_ROOT_ACCOUNT_NAME**: for direct mode, S3 username
+
+**DELETE_BUCKETS_AFTER_TEST**: whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked
+
+## REST Scripts
+
+REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder. To call a script, the following parameters are needed:
+* **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**, etc.
+* **AWS_ENDPOINT_URL** (default: `https://localhost:7070`)
+* **OUTPUT_FILE**: file where the command's response data is written
+* Any other parameters specified at the top of the script file, such as payloads and variables. Sometimes, defaults are included.
+
+Upon success, the script will return a response code, and write the data to the **OUTPUT_FILE** location.
+
+Example: `AWS_ACCESS_KEY_ID={id} AWS_SECRET_ACCESS_KEY={key} AWS_ENDPOINT_URL=https://s3.amazonaws.com OUTPUT_FILE=./output_file.xml ./tests/rest_scripts/list_buckets.sh`
diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh
index fa3ff2aa..bdc1c86d 100644
--- a/tests/commands/put_bucket_acl.sh
+++ b/tests/commands/put_bucket_acl.sh
@@ -59,11 +59,11 @@ reset_bucket_acl() {
fi
# shellcheck disable=SC2154
if [ "$DIRECT" != "true" ]; then
- if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
+ if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then
log 2 "error resetting versitygw ACL"
return 1
fi
- elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
+ elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then
log 2 "error resetting direct ACL"
return 1
fi
diff --git a/tests/logger.sh b/tests/logger.sh
index e20df9be..ac221382 100644
--- a/tests/logger.sh
+++ b/tests/logger.sh
@@ -55,7 +55,7 @@ log_mask() {
return 1
fi
- log_message "$log_level" "${masked_args[*]}"
+ log_message "$log_level" "$masked_data"
}
mask_args() {
@@ -63,23 +63,38 @@ mask_args() {
echo "'mask_args' requires string"
return 1
fi
- IFS=' ' read -r -a array <<< "$1"
+ unmasked_array=()
+ masked_data=""
+ while IFS= read -r line; do
+ unmasked_array+=("$line")
+ done <<< "$1"
- if ! mask_arg_array "${array[@]}"; then
- echo "error masking arg array"
- return 1
- fi
+ # shellcheck disable=SC2068
+ first_line=true
+ for line in "${unmasked_array[@]}"; do
+ if ! mask_arg_array "$line"; then
+ echo "error masking arg array"
+ return 1
+ fi
+ if [ "$first_line" == "true" ]; then
+ masked_data="${masked_args[*]}"
+ first_line="false"
+ else
+ masked_data+=$(printf "\n%s" "${masked_args[*]}")
+ fi
+ done
}
mask_arg_array() {
- masked_args=() # Initialize an array to hold the masked arguments
if [ $# -eq 0 ]; then
echo "'mask_arg_array' requires parameters"
return 1
fi
mask_next=false
is_access=false
- for arg in "$@"; do
+ masked_args=() # Initialize an array to hold the masked arguments
+ # shellcheck disable=SC2068
+ for arg in $@; do
if ! check_arg_for_mask "$arg"; then
echo "error checking arg for mask"
return 1
@@ -135,4 +150,5 @@ log_message() {
if [[ -n "$TEST_LOG_FILE" ]]; then
echo "$now $1 $2" >> "$TEST_LOG_FILE.tmp"
fi
+ sync
}
diff --git a/tests/rest_scripts/get_public_access_block.sh b/tests/rest_scripts/get_public_access_block.sh
new file mode 100755
index 00000000..258fb2f9
--- /dev/null
+++ b/tests/rest_scripts/get_public_access_block.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="GET
+/$bucket_name
+publicAccessBlock=
+host:$host
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\""
+-H "\"x-amz-date: $current_date_time\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1
diff --git a/tests/rest_scripts/put_bucket_acl.sh b/tests/rest_scripts/put_bucket_acl.sh
index 112ba9bb..4b53e587 100755
--- a/tests/rest_scripts/put_bucket_acl.sh
+++ b/tests/rest_scripts/put_bucket_acl.sh
@@ -22,8 +22,14 @@ source ./tests/rest_scripts/rest.sh
bucket_name="$BUCKET_NAME"
# shellcheck disable=SC2153
acl_file="$ACL_FILE"
+# shellcheck disable=SC2153
+canned_acl="$CANNED_ACL"
-payload="$(cat "$acl_file")"
+if [ -n "$ACL_FILE" ]; then
+ payload="$(cat "$acl_file")"
+else
+ payload=""
+fi
payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
@@ -32,20 +38,40 @@ canonical_request="PUT
/$bucket_name
acl=
host:$host
-x-amz-content-sha256:$payload_hash
+"
+if [ -n "$CANNED_ACL" ]; then
+ canonical_request+="x-amz-acl:$canned_acl
+"
+fi
+canonical_request+="x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time
-host;x-amz-content-sha256;x-amz-date
+"
+canonical_request+="host;"
+if [ -n "$CANNED_ACL" ]; then
+ canonical_request+="x-amz-acl;"
+fi
+canonical_request+="x-amz-content-sha256;x-amz-date
$payload_hash"
create_canonical_hash_sts_and_signature
-curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl="
--H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
--H "\"x-amz-content-sha256: $payload_hash\""
--H "\"x-amz-date: $current_date_time\""
--d "\"${payload//\"/\\\"}\""
--o "$OUTPUT_FILE")
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=")
+if [ -n "$CANNED_ACL" ]; then
+ acl_header="x-amz-acl;"
+else
+ acl_header=""
+fi
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;${acl_header}x-amz-content-sha256;x-amz-date,Signature=$signature\"")
+if [ -n "$CANNED_ACL" ]; then
+ curl_command+=(-H "\"x-amz-acl: $canned_acl\"")
+fi
+curl_command+=(-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\"")
+if [ -n "$ACL_FILE" ]; then
+ curl_command+=(-d "\"${payload//\"/\\\"}\"")
+fi
+curl_command+=(-o "$OUTPUT_FILE")
# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1
diff --git a/tests/rest_scripts/put_public_access_block.sh b/tests/rest_scripts/put_public_access_block.sh
new file mode 100755
index 00000000..eb0a1233
--- /dev/null
+++ b/tests/rest_scripts/put_public_access_block.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+# Fields
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+block_public_acls="${BLOCK_PUBLIC_ACLS:-TRUE}"
+ignore_public_acls="${IGNORE_PUBLIC_ACLS:-TRUE}"
+block_public_policy="${BLOCK_PUBLIC_POLICY:-TRUE}"
+restrict_public_buckets="${RESTRICT_PUBLIC_BUCKETS:-TRUE}"
+
+payload="
+
+ $block_public_acls
+ $ignore_public_acls
+ $block_public_policy
+ $restrict_public_buckets
+"
+
+payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+
+canonical_request="PUT
+/$bucket_name
+publicAccessBlock=
+host:$host
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+create_canonical_hash_sts_and_signature
+
+curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock="
+-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
+-H "\"x-amz-content-sha256: $payload_hash\""
+-H "\"x-amz-date: $current_date_time\""
+-d "\"${payload//\"/\\\"}\""
+-o "$OUTPUT_FILE")
+# shellcheck disable=SC2154
+eval "${curl_command[*]}" 2>&1
diff --git a/tests/setup.sh b/tests/setup.sh
index a256d76b..6ec5f826 100644
--- a/tests/setup.sh
+++ b/tests/setup.sh
@@ -86,6 +86,18 @@ teardown() {
log 3 "unable to remove test file folder: $error"
fi
fi
+ stop_versity
+ if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
+ end_time=$(date +%s)
+ total_time=$((end_time - start_time))
+ log 4 "Total test time: $total_time"
+ if [[ -n "$TIME_LOG" ]]; then
+ echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
+ fi
+ fi
+ if [[ -n "$COVERAGE_DB" ]]; then
+ record_result
+ fi
if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then
if [[ -e "$COMMAND_LOG" ]]; then
cat "$COMMAND_LOG"
@@ -108,16 +120,4 @@ teardown() {
log 2 "error deleting temp log"
fi
fi
- stop_versity
- if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
- end_time=$(date +%s)
- total_time=$((end_time - start_time))
- log 4 "Total test time: $total_time"
- if [[ -n "$TIME_LOG" ]]; then
- echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG"
- fi
- fi
- if [[ -n "$COVERAGE_DB" ]]; then
- record_result
- fi
}
diff --git a/tests/test_rest.sh b/tests/test_rest.sh
index 76403393..d90a2161 100755
--- a/tests/test_rest.sh
+++ b/tests/test_rest.sh
@@ -38,6 +38,7 @@ source ./tests/util/util_list_parts.sh
source ./tests/util/util_lock_config.sh
source ./tests/util/util_ownership.sh
source ./tests/util/util_policy.sh
+source ./tests/util/util_public_access_block.sh
source ./tests/util/util_rest.sh
source ./tests/util/util_tags.sh
source ./tests/util/util_time.sh
@@ -244,9 +245,6 @@ export RUN_USERS=true
}
@test "versioning - retrieve after delete" {
- if [ "$DIRECT" != "true" ]; then
- skip "https://github.com/versity/versitygw/issues/888"
- fi
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -269,9 +267,6 @@ export RUN_USERS=true
}
@test "REST - legal hold, get without config" {
- if [ "$DIRECT" != "true" ]; then
- skip "https://github.com/versity/versitygw/issues/883"
- fi
test_file="test_file"
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
@@ -490,4 +485,42 @@ export RUN_USERS=true
run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file"
assert_success
-}
\ No newline at end of file
+}
+
+@test "REST - put public-read canned acl" {
+ run setup_bucket "s3api" "$BUCKET_ONE_NAME"
+ assert_success
+
+ test_file="test_file"
+ run create_test_files "$test_file"
+ assert_success
+
+ run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred"
+ assert_success
+
+ run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+ assert_success
+
+ run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE"
+ assert_success
+ canonical_id=${lines[0]}
+ user_canonical_id=${lines[1]}
+ username=${lines[2]}
+ password=${lines[3]}
+
+ run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password"
+ assert_success
+
+ run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id"
+ assert_success
+
+ if [ "$DIRECT" == "true" ]; then
+ run allow_public_access "$BUCKET_ONE_NAME"
+ assert_success
+ fi
+ run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt"
+ assert_success
+
+ run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file"
+ assert_success
+}
diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh
index 81a24add..a08fd003 100755
--- a/tests/test_user_aws.sh
+++ b/tests/test_user_aws.sh
@@ -125,9 +125,6 @@ export RUN_USERS=true
}
@test "test_admin_put_get_object" {
- if [ "$RECREATE_BUCKETS" == "false" ]; then
- skip "https://github.com/versity/versitygw/issues/888"
- fi
test_file="test_file"
run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "admin" "$BUCKET_ONE_NAME"
diff --git a/tests/util/util_public_access_block.sh b/tests/util/util_public_access_block.sh
new file mode 100644
index 00000000..45e1efb8
--- /dev/null
+++ b/tests/util/util_public_access_block.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+allow_public_access() {
+ if [ $# -ne 1 ]; then
+ log 2 "'allow_public_access' requires bucket name"
+ return 1
+ fi
+ if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/get_public_access_block.sh); then
+ log 2 "error getting public access block: $result"
+ return 1
+ fi
+ if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" BLOCK_PUBLIC_ACLS="FALSE" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_public_access_block.sh); then
+ log 2 "error getting public access block: $result"
+ return 1
+ fi
+
+ return 0
+}
\ No newline at end of file