diff --git a/tests/README.md b/tests/README.md index 30ab7431..8c784d9c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,19 @@ # Command-Line Tests +## Table of Contents + +[Instructions - Running Locally](#instructions---running-locally)
+[* Posix Backend](#posix-backend)
+[* Static Bucket Mode](#static-bucket-mode)
+[* S3 Backend](#s3-backend)
+[* Direct Mode](#direct-mode)
+[Instructions - Running With Docker](#instructions---running-with-docker)
+[Instructions - Running With Docker-Compose](#instructions---running-with-docker-compose)
+[Environment Parameters](#environment-parameters)
+[* Secret](#secret)
+[* Non-Secret](#non-secret)
+[REST Scripts](#rest-scripts)
+ ## Instructions - Running Locally ### Posix Backend @@ -85,8 +99,18 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up ## Environment Parameters +### Secret + **AWS_PROFILE**, **AWS_ENDPOINT_URL**, **AWS_REGION**, **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**: identical to the same parameters in **s3**. +**AWS_CANONICAL_ID**: for direct mode, the canonical ID for the main user (owner) + +**ACL_AWS_CANONICAL_ID**: for direct mode, the canonical ID for the user to test ACL changes and access by non-owners + +**ACL_AWS_ACCESS_KEY_ID**, **ACL_AWS_ACCESS_SECRET_KEY**: for direct mode, the ID and key for the S3 user in the **ACL_AWS_CANONICAL_ID** account. + +### Non-Secret + **VERSITY_EXE**: location of the versity executable relative to test folder. **RUN_VERSITYGW**: whether to run the versitygw executable, should be set to **false** when running tests directly against **s3**. @@ -134,3 +158,21 @@ A single instance can be run with `docker-compose -f docker-compose-bats.yml up **VERSIONING_DIR**: where to put gateway file versioning info. **COMMAND_LOG**: where to store list of client commands, which if using will be reported during test failures. + +**TIME_LOG**: optional log to show duration of individual tests + +**DIRECT_S3_ROOT_ACCOUNT_NAME**: for direct mode, S3 username + +**DELETE_BUCKETS_AFTER_TEST**: whether or not to delete buckets after individual tests, useful for debugging if the post-test bucket state needs to be checked + +## REST Scripts + +REST scripts are included for calls to S3's REST API in the `./tests/rest_scripts/` folder. To call a script, the following parameters are needed: +* **AWS_ACCESS_KEY_ID**, **AWS_SECRET_ACCESS_KEY**, etc. +* **AWS_ENDPOINT_URL** (default: `https://localhost:7070`) +* **OUTPUT_FILE**: file where the command's response data is written +* Any other parameters specified at the top of the script file, such as payloads and variables. Sometimes, defaults are included. + +Upon success, the script will return a response code, and write the data to the **OUTPUT_FILE** location. + +Example: `AWS_ACCESS_KEY_ID={id} AWS_SECRET_ACCESS_KEY={key} AWS_ENDPOINT_URL=https://s3.amazonaws.com OUTPUT_FILE=./output_file.xml ./tests/rest_scripts/list_buckets.sh` diff --git a/tests/commands/put_bucket_acl.sh b/tests/commands/put_bucket_acl.sh index fa3ff2aa..bdc1c86d 100644 --- a/tests/commands/put_bucket_acl.sh +++ b/tests/commands/put_bucket_acl.sh @@ -59,11 +59,11 @@ reset_bucket_acl() { fi # shellcheck disable=SC2154 if [ "$DIRECT" != "true" ]; then - if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then + if ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_ACCESS_KEY_ID" "FULL_CONTROL" "$AWS_ACCESS_KEY_ID"; then log 2 "error resetting versitygw ACL" return 1 fi - elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "ID" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then + elif ! setup_acl_json "$TEST_FILE_FOLDER/$acl_file" "CanonicalUser" "$AWS_CANONICAL_ID" "FULL_CONTROL" "$AWS_CANONICAL_ID"; then log 2 "error resetting direct ACL" return 1 fi diff --git a/tests/logger.sh b/tests/logger.sh index e20df9be..ac221382 100644 --- a/tests/logger.sh +++ b/tests/logger.sh @@ -55,7 +55,7 @@ log_mask() { return 1 fi - log_message "$log_level" "${masked_args[*]}" + log_message "$log_level" "$masked_data" } mask_args() { @@ -63,23 +63,38 @@ mask_args() { echo "'mask_args' requires string" return 1 fi - IFS=' ' read -r -a array <<< "$1" + unmasked_array=() + masked_data="" + while IFS= read -r line; do + unmasked_array+=("$line") + done <<< "$1" - if ! mask_arg_array "${array[@]}"; then - echo "error masking arg array" - return 1 - fi + # shellcheck disable=SC2068 + first_line=true + for line in "${unmasked_array[@]}"; do + if ! mask_arg_array "$line"; then + echo "error masking arg array" + return 1 + fi + if [ "$first_line" == "true" ]; then + masked_data="${masked_args[*]}" + first_line="false" + else + masked_data+=$(printf "\n%s" "${masked_args[*]}") + fi + done } mask_arg_array() { - masked_args=() # Initialize an array to hold the masked arguments if [ $# -eq 0 ]; then echo "'mask_arg_array' requires parameters" return 1 fi mask_next=false is_access=false - for arg in "$@"; do + masked_args=() # Initialize an array to hold the masked arguments + # shellcheck disable=SC2068 + for arg in $@; do if ! check_arg_for_mask "$arg"; then echo "error checking arg for mask" return 1 @@ -135,4 +150,5 @@ log_message() { if [[ -n "$TEST_LOG_FILE" ]]; then echo "$now $1 $2" >> "$TEST_LOG_FILE.tmp" fi + sync } diff --git a/tests/rest_scripts/get_public_access_block.sh b/tests/rest_scripts/get_public_access_block.sh new file mode 100755 index 00000000..258fb2f9 --- /dev/null +++ b/tests/rest_scripts/get_public_access_block.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="GET +/$bucket_name +publicAccessBlock= +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/put_bucket_acl.sh b/tests/rest_scripts/put_bucket_acl.sh index 112ba9bb..4b53e587 100755 --- a/tests/rest_scripts/put_bucket_acl.sh +++ b/tests/rest_scripts/put_bucket_acl.sh @@ -22,8 +22,14 @@ source ./tests/rest_scripts/rest.sh bucket_name="$BUCKET_NAME" # shellcheck disable=SC2153 acl_file="$ACL_FILE" +# shellcheck disable=SC2153 +canned_acl="$CANNED_ACL" -payload="$(cat "$acl_file")" +if [ -n "$ACL_FILE" ]; then + payload="$(cat "$acl_file")" +else + payload="" +fi payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") @@ -32,20 +38,40 @@ canonical_request="PUT /$bucket_name acl= host:$host -x-amz-content-sha256:$payload_hash +" +if [ -n "$CANNED_ACL" ]; then + canonical_request+="x-amz-acl:$canned_acl +" +fi +canonical_request+="x-amz-content-sha256:$payload_hash x-amz-date:$current_date_time -host;x-amz-content-sha256;x-amz-date +" +canonical_request+="host;" +if [ -n "$CANNED_ACL" ]; then + canonical_request+="x-amz-acl;" +fi +canonical_request+="x-amz-content-sha256;x-amz-date $payload_hash" create_canonical_hash_sts_and_signature -curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=" --H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" --H "\"x-amz-content-sha256: $payload_hash\"" --H "\"x-amz-date: $current_date_time\"" --d "\"${payload//\"/\\\"}\"" --o "$OUTPUT_FILE") +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?acl=") +if [ -n "$CANNED_ACL" ]; then + acl_header="x-amz-acl;" +else + acl_header="" +fi +curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;${acl_header}x-amz-content-sha256;x-amz-date,Signature=$signature\"") +if [ -n "$CANNED_ACL" ]; then + curl_command+=(-H "\"x-amz-acl: $canned_acl\"") +fi +curl_command+=(-H "\"x-amz-content-sha256: $payload_hash\"" +-H "\"x-amz-date: $current_date_time\"") +if [ -n "$ACL_FILE" ]; then + curl_command+=(-d "\"${payload//\"/\\\"}\"") +fi +curl_command+=(-o "$OUTPUT_FILE") # shellcheck disable=SC2154 eval "${curl_command[*]}" 2>&1 diff --git a/tests/rest_scripts/put_public_access_block.sh b/tests/rest_scripts/put_public_access_block.sh new file mode 100755 index 00000000..eb0a1233 --- /dev/null +++ b/tests/rest_scripts/put_public_access_block.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +block_public_acls="${BLOCK_PUBLIC_ACLS:-TRUE}" +ignore_public_acls="${IGNORE_PUBLIC_ACLS:-TRUE}" +block_public_policy="${BLOCK_PUBLIC_POLICY:-TRUE}" +restrict_public_buckets="${RESTRICT_PUBLIC_BUCKETS:-TRUE}" + +payload=" + + $block_public_acls + $ignore_public_acls + $block_public_policy + $restrict_public_buckets +" + +payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')" +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="PUT +/$bucket_name +publicAccessBlock= +host:$host +x-amz-content-sha256:$payload_hash +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +$payload_hash" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?publicAccessBlock=" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: $payload_hash\"" +-H "\"x-amz-date: $current_date_time\"" +-d "\"${payload//\"/\\\"}\"" +-o "$OUTPUT_FILE") +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/setup.sh b/tests/setup.sh index a256d76b..6ec5f826 100644 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -86,6 +86,18 @@ teardown() { log 3 "unable to remove test file folder: $error" fi fi + stop_versity + if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then + end_time=$(date +%s) + total_time=$((end_time - start_time)) + log 4 "Total test time: $total_time" + if [[ -n "$TIME_LOG" ]]; then + echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG" + fi + fi + if [[ -n "$COVERAGE_DB" ]]; then + record_result + fi if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then if [[ -e "$COMMAND_LOG" ]]; then cat "$COMMAND_LOG" @@ -108,16 +120,4 @@ teardown() { log 2 "error deleting temp log" fi fi - stop_versity - if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then - end_time=$(date +%s) - total_time=$((end_time - start_time)) - log 4 "Total test time: $total_time" - if [[ -n "$TIME_LOG" ]]; then - echo "$BATS_TEST_NAME: ${total_time}s" >> "$TIME_LOG" - fi - fi - if [[ -n "$COVERAGE_DB" ]]; then - record_result - fi } diff --git a/tests/test_rest.sh b/tests/test_rest.sh index 76403393..d90a2161 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -38,6 +38,7 @@ source ./tests/util/util_list_parts.sh source ./tests/util/util_lock_config.sh source ./tests/util/util_ownership.sh source ./tests/util/util_policy.sh +source ./tests/util/util_public_access_block.sh source ./tests/util/util_rest.sh source ./tests/util/util_tags.sh source ./tests/util/util_time.sh @@ -244,9 +245,6 @@ export RUN_USERS=true } @test "versioning - retrieve after delete" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/888" - fi test_file="test_file" run setup_bucket "s3api" "$BUCKET_ONE_NAME" @@ -269,9 +267,6 @@ export RUN_USERS=true } @test "REST - legal hold, get without config" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/883" - fi test_file="test_file" run setup_bucket "s3api" "$BUCKET_ONE_NAME" @@ -490,4 +485,42 @@ export RUN_USERS=true run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file" assert_success -} \ No newline at end of file +} + +@test "REST - put public-read canned acl" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_test_files "$test_file" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success + canonical_id=${lines[0]} + user_canonical_id=${lines[1]} + username=${lines[2]} + password=${lines[3]} + + run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password" + assert_success + + run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id" + assert_success + + if [ "$DIRECT" == "true" ]; then + run allow_public_access "$BUCKET_ONE_NAME" + assert_success + fi + run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt" + assert_success + + run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file" + assert_success +} diff --git a/tests/test_user_aws.sh b/tests/test_user_aws.sh index 81a24add..a08fd003 100755 --- a/tests/test_user_aws.sh +++ b/tests/test_user_aws.sh @@ -125,9 +125,6 @@ export RUN_USERS=true } @test "test_admin_put_get_object" { - if [ "$RECREATE_BUCKETS" == "false" ]; then - skip "https://github.com/versity/versitygw/issues/888" - fi test_file="test_file" run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "admin" "$BUCKET_ONE_NAME" diff --git a/tests/util/util_public_access_block.sh b/tests/util/util_public_access_block.sh new file mode 100644 index 00000000..45e1efb8 --- /dev/null +++ b/tests/util/util_public_access_block.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +allow_public_access() { + if [ $# -ne 1 ]; then + log 2 "'allow_public_access' requires bucket name" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/get_public_access_block.sh); then + log 2 "error getting public access block: $result" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" BLOCK_PUBLIC_ACLS="FALSE" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_public_access_block.sh); then + log 2 "error getting public access block: $result" + return 1 + fi + + return 0 +} \ No newline at end of file