From 8d4a8fc5e097e51219ce1c4b63f988da61d3f10d Mon Sep 17 00:00:00 2001 From: Luke McCrone Date: Fri, 13 Dec 2024 21:09:55 -0300 Subject: [PATCH] test: REST acl public-read-write test, other acl test, cleanup --- tests/commands/delete_object_tagging.sh | 34 +--- tests/env.sh | 2 +- tests/rest_scripts/delete_object_tagging.sh | 47 ++++++ tests/run.sh | 6 +- tests/test_rest.sh | 87 ---------- tests/test_rest_acl.sh | 167 ++++++++++++++++++++ tests/util/util.sh | 6 - tests/util/util_acl.sh | 40 +++++ tests/util/util_file.sh | 43 ----- tests/util/util_public_access_block.sh | 7 +- tests/util/util_users.sh | 8 - tests/versity.sh | 4 - 12 files changed, 265 insertions(+), 186 deletions(-) create mode 100755 tests/rest_scripts/delete_object_tagging.sh create mode 100755 tests/test_rest_acl.sh diff --git a/tests/commands/delete_object_tagging.sh b/tests/commands/delete_object_tagging.sh index 5473e309..78a21e93 100644 --- a/tests/commands/delete_object_tagging.sh +++ b/tests/commands/delete_object_tagging.sh @@ -43,38 +43,12 @@ delete_object_tagging_rest() { log 2 "'delete_object_tagging' requires bucket, key" return 1 fi - - generate_hash_for_payload "" - - current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") - aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//} - header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}') - # shellcheck disable=SC2154 - canonical_request="DELETE -/$1/$2 -tagging= -host:$aws_endpoint_url_address -x-amz-content-sha256:$payload_hash -x-amz-date:$current_date_time - -host;x-amz-content-sha256;x-amz-date -$payload_hash" - - if ! generate_sts_string "$current_date_time" "$canonical_request"; then - log 2 "error generating sts string" + if ! result=$(BUCKET_NAME="$1" OBJECT_KEY="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/delete_object_tagging.sh); then + log 2 "error sending delete object tagging REST command: $result" return 1 fi - get_signature - # shellcheck disable=SC2154 - reply=$(send_command curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2?tagging" \ - -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \ - -H "x-amz-content-sha256: $payload_hash" \ - -H "x-amz-date: $current_date_time" \ - -d "$tagging" -o "$TEST_FILE_FOLDER"/delete_tagging_error.txt 2>&1) - log 5 "reply status code: $reply" - if [[ "$reply" != "204" ]]; then - log 2 "reply error: $reply" - log 2 "put object tagging command returned error: $(cat "$TEST_FILE_FOLDER"/delete_tagging_error.txt)" + if [ "$result" != "204" ]; then + log 2 "delete-object-tagging returned code $result (response: $(cat "$TEST_FILE_FOLDER/response.txt"))" return 1 fi return 0 diff --git a/tests/env.sh b/tests/env.sh index 03f9fa2f..17cad190 100644 --- a/tests/env.sh +++ b/tests/env.sh @@ -136,7 +136,7 @@ check_universal_vars() { exit 1 fi if [ ! -d "$TEST_FILE_FOLDER" ]; then - if ! error=$(mkdir -p "$TEST_FILE_FOLDER"); then + if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then log 2 "error creating test folder: $error" exit 1 fi diff --git a/tests/rest_scripts/delete_object_tagging.sh b/tests/rest_scripts/delete_object_tagging.sh new file mode 100755 index 00000000..2cc9f11e --- /dev/null +++ b/tests/rest_scripts/delete_object_tagging.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/rest_scripts/rest.sh + +# Fields + +# shellcheck disable=SC2153 +bucket_name="$BUCKET_NAME" +# shellcheck disable=SC2153 +key="$OBJECT_KEY" + +current_date_time=$(date -u +"%Y%m%dT%H%M%SZ") + +canonical_request="DELETE +/$bucket_name/$key +tagging= +host:$host +x-amz-content-sha256:UNSIGNED-PAYLOAD +x-amz-date:$current_date_time + +host;x-amz-content-sha256;x-amz-date +UNSIGNED-PAYLOAD" + +create_canonical_hash_sts_and_signature + +curl_command+=(curl -ks -w "\"%{http_code}\"" -X DELETE "$AWS_ENDPOINT_URL/$bucket_name/$key?tagging" +-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\"" +-H "\"x-amz-content-sha256: UNSIGNED-PAYLOAD\"" +-H "\"x-amz-date: $current_date_time\"" +-o "$OUTPUT_FILE") + +# shellcheck disable=SC2154 +eval "${curl_command[*]}" 2>&1 diff --git a/tests/run.sh b/tests/run.sh index af89a0e2..9cb73a6a 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -141,7 +141,11 @@ run_suite() { ;; rest) echo "Running rest tests ..." - "$HOME"/bin/bats ./tests/test_rest.sh || exit_code=$? + if "$HOME"/bin/bats ./tests/test_rest.sh; then + "$HOME"/bin/bats ./tests/test_rest_acl.sh || exit_code=$? + else + exit_code=1 + fi ;; s3api-user) echo "Running s3api user tests ..." diff --git a/tests/test_rest.sh b/tests/test_rest.sh index d90a2161..a7a2b644 100755 --- a/tests/test_rest.sh +++ b/tests/test_rest.sh @@ -437,90 +437,3 @@ export RUN_USERS=true run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME" assert_success } - -@test "REST - get ACL" { - if [ "$DIRECT" != "true" ]; then - skip "https://github.com/versity/versitygw/issues/971" - fi - run setup_bucket "s3api" "$BUCKET_ONE_NAME" - assert_success - - run get_and_check_acl_rest "$BUCKET_ONE_NAME" - assert_success -} - -@test "REST - put ACL" { - run setup_bucket "s3api" "$BUCKET_ONE_NAME" - assert_success - - test_file="test_file" - run create_test_files "$test_file" - assert_success - - run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" - assert_success - - run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" - assert_success - - run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" - assert_success - canonical_id=${lines[0]} - user_canonical_id=${lines[1]} - username=${lines[2]} - password=${lines[3]} - - run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id" - assert_success - - run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password" - assert_success - - run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt" - assert_success - - if [ "$DIRECT" == "true" ]; then - sleep 5 - fi - - run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file" - assert_success -} - -@test "REST - put public-read canned acl" { - run setup_bucket "s3api" "$BUCKET_ONE_NAME" - assert_success - - test_file="test_file" - run create_test_files "$test_file" - assert_success - - run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" - assert_success - - run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" - assert_success - - run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" - assert_success - canonical_id=${lines[0]} - user_canonical_id=${lines[1]} - username=${lines[2]} - password=${lines[3]} - - run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password" - assert_success - - run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id" - assert_success - - if [ "$DIRECT" == "true" ]; then - run allow_public_access "$BUCKET_ONE_NAME" - assert_success - fi - run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt" - assert_success - - run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file" - assert_success -} diff --git a/tests/test_rest_acl.sh b/tests/test_rest_acl.sh new file mode 100755 index 00000000..2319d74e --- /dev/null +++ b/tests/test_rest_acl.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bats + +# Copyright 2024 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./tests/commands/put_object.sh +source ./tests/logger.sh +source ./tests/setup.sh +source ./tests/util/util.sh +source ./tests/util/util_acl.sh + +export RUN_USERS=true + +@test "REST - get ACL" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/971" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run get_and_check_acl_rest "$BUCKET_ONE_NAME" + assert_success +} + +@test "REST - put ACL" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_test_files "$test_file" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success + canonical_id=${lines[0]} + user_canonical_id=${lines[1]} + username=${lines[2]} + password=${lines[3]} + + run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READ" "$canonical_id" + assert_success + + run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password" + assert_success + + run put_acl_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt" + assert_success + + if [ "$DIRECT" == "true" ]; then + sleep 5 + fi + + run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file" + assert_success +} + +@test "REST - put public-read canned acl" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_test_files "$test_file" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" + assert_success + + run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success + canonical_id=${lines[0]} + user_canonical_id=${lines[1]} + username=${lines[2]} + password=${lines[3]} + + run list_objects_with_user_rest_verify_access_denied "$BUCKET_ONE_NAME" "$username" "$password" + assert_success + + if [ "$DIRECT" == "true" ]; then + run allow_public_access "$BUCKET_ONE_NAME" + assert_success + fi + run put_canned_acl_rest "$BUCKET_ONE_NAME" "public-read" + assert_success + + run list_objects_with_user_rest_verify_success "$BUCKET_ONE_NAME" "$username" "$password" "$test_file" + assert_success +} + +@test "REST - put invalid ACL" { + if [ "$DIRECT" != "true" ]; then + skip "https://github.com/versity/versitygw/issues/986" + fi + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success + canonical_id=${lines[0]} + user_canonical_id=${lines[1]} + username=${lines[2]} + password=${lines[3]} + + run setup_acl "$TEST_FILE_FOLDER/acl-file.txt" "$user_canonical_id" "READD" "$canonical_id" + assert_success + + if [ "$DIRECT" == "true" ]; then + run allow_public_access "$BUCKET_ONE_NAME" + assert_success + fi + run put_invalid_acl_rest_verify_failure "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/acl-file.txt" + assert_success +} + +@test "REST - put public-read-write canned acl" { + run setup_bucket "s3api" "$BUCKET_ONE_NAME" + assert_success + + test_file="test_file" + run create_test_files "$test_file" + assert_success + + run put_bucket_ownership_controls "$BUCKET_ONE_NAME" "BucketOwnerPreferred" + assert_success + + run create_versitygw_acl_user_or_get_direct_user "$USERNAME_ONE" "$PASSWORD_ONE" + assert_success + canonical_id=${lines[0]} + user_canonical_id=${lines[1]} + username=${lines[2]} + password=${lines[3]} + + run put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" + assert_failure + + if [ "$DIRECT" == "true" ]; then + run allow_public_access "$BUCKET_ONE_NAME" + assert_success + fi + run put_canned_acl_rest "$BUCKET_ONE_NAME" "public-read-write" + assert_success + + run put_object_with_user "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "$username" "$password" + assert_success +} diff --git a/tests/util/util.sh b/tests/util/util.sh index a984c3fa..4a8404dd 100644 --- a/tests/util/util.sh +++ b/tests/util/util.sh @@ -54,12 +54,6 @@ add_governance_bypass_policy() { log 2 "'add governance bypass policy' command requires bucket name" return 1 fi - if [[ -z "$GITHUB_ACTIONS" ]]; then - if ! create_test_file_folder; then - log 2 "error creating test file folder" - return 1 - fi - fi cat < "$TEST_FILE_FOLDER/policy-bypass-governance.txt" { "Version": "2012-10-17", diff --git a/tests/util/util_acl.sh b/tests/util/util_acl.sh index bcc72c47..f3460594 100644 --- a/tests/util/util_acl.sh +++ b/tests/util/util_acl.sh @@ -348,3 +348,43 @@ put_acl_rest() { fi return 0 } + +put_invalid_acl_rest_verify_failure() { + if [ $# -ne 2 ]; then + log 2 "'put_invalid_acl_rest_verify_failure' requires bucket name, ACL file" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" ACL_FILE="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then + log 2 "error attempting to put bucket acl: $result" + return 1 + fi + if [ "$result" != "400" ]; then + log 2 "response returned code: $result (error: $(cat "$TEST_FILE_FOLDER/response.txt"))" + return 1 + fi + if ! error_code=$(xmllint --xpath '//*[local-name()="Code"]/text()' "$TEST_FILE_FOLDER/response.txt" 2>&1); then + log 2 "error getting display name: $error_code" + return 1 + fi + if [ "$error_code" != "MalformedACLError" ]; then + log 2 "invalid error code, expected 'MalformedACLError', was '$error_code'" + return 1 + fi + return 0 +} + +put_canned_acl_rest() { + if [ $# -ne 2 ]; then + log 2 "'put_canned_acl_rest' requires bucket name, canned acl" + return 1 + fi + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" CANNED_ACL="$2" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_bucket_acl.sh); then + log 2 "error attempting to put bucket acl: $result" + return 1 + fi + if [ "$result" != "200" ]; then + log 2 "response code '$result' (message: $(cat "$TEST_FILE_FOLDER/response.txt"))" + return 1 + fi + return 0 +} diff --git a/tests/util/util_file.sh b/tests/util/util_file.sh index a36606cf..ad40326c 100644 --- a/tests/util/util_file.sh +++ b/tests/util/util_file.sh @@ -25,12 +25,6 @@ create_test_files() { log 2 "'create_test_files' requires file names" return 1 fi - if [[ -z "$GITHUB_ACTIONS" ]]; then - if ! create_test_file_folder; then - log 2 "error creating test file folder" - return 1 - fi - fi for name in "$@"; do if ! create_test_file "$name"; then log 2 "error creating test file" @@ -46,12 +40,6 @@ create_test_file() { log 2 "'create_test_file' requires filename, size (optional)" return 1 fi - if [[ -z "$GITHUB_ACTIONS" ]]; then - if ! create_test_file_folder; then - log 2 "error creating test file folder" - return 1 - fi - fi if [[ -e "$TEST_FILE_FOLDER/$1" ]]; then if ! error=$(rm "$TEST_FILE_FOLDER/$1" 2>&1); then log 2 "error removing existing file: $error" @@ -84,12 +72,6 @@ create_test_folder() { log 2 "'create_test_folder' requires folder names" return 1 fi - if [[ -z "$GITHUB_ACTIONS" ]]; then - if ! create_test_file_folder; then - log 2 "error creating test file folder" - return 1 - fi - fi for name in "$@"; do if ! error=$(mkdir -p "$TEST_FILE_FOLDER"/"$name" 2>&1); then log 2 "error creating folder $name: $error" @@ -167,19 +149,6 @@ compare_files() { return 1 } -# return 0 on success, 1 on error -create_test_file_folder() { - log 6 "create_test_file_folder" - if ! error=$(mkdir -p "$TEST_FILE_FOLDER" 2>&1); then - # shellcheck disable=SC2035 - if [[ "$error" != *"File exists"* ]]; then - log 2 "error making test file folder: $error" - return 1 - fi - fi - return 0 -} - # generate 160MB file # input: filename # fail on error @@ -189,12 +158,6 @@ create_large_file() { log 2 "'create_large_file' requires file name" return 1 fi - if [[ -z "$GITHUB_ACTIONS" ]]; then - if ! create_test_file_folder; then - log 2 "error creating test file folder" - return 1 - fi - fi filesize=$((160*1024*1024)) if ! error=$(dd if=/dev/urandom of="$TEST_FILE_FOLDER"/"$1" bs=1024 count=$((filesize/1024)) 2>&1); then @@ -211,12 +174,6 @@ create_test_file_count() { log 2 "'create_test_file_count' requires number of files" return 1 fi - if [[ -z "$GITHUB_ACTIONS" ]]; then - if ! create_test_file_folder; then - log 2 "error creating test file folder" - return 1 - fi - fi for ((i=1;i<=$1;i++)) { if ! error=$(touch "$TEST_FILE_FOLDER/file_$i" 2>&1); then log 2 "error creating file_$i: $error" diff --git a/tests/util/util_public_access_block.sh b/tests/util/util_public_access_block.sh index 45e1efb8..f1c3cb2e 100644 --- a/tests/util/util_public_access_block.sh +++ b/tests/util/util_public_access_block.sh @@ -5,14 +5,9 @@ allow_public_access() { log 2 "'allow_public_access' requires bucket name" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/get_public_access_block.sh); then + if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" BLOCK_PUBLIC_ACLS="FALSE" IGNORE_PUBLIC_ACLS="FALSE" RESTRICT_PUBLIC_BUCKETS="FALSE" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_public_access_block.sh); then log 2 "error getting public access block: $result" return 1 fi - if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" BLOCK_PUBLIC_ACLS="FALSE" OUTPUT_FILE="$TEST_FILE_FOLDER/response.txt" ./tests/rest_scripts/put_public_access_block.sh); then - log 2 "error getting public access block: $result" - return 1 - fi - return 0 } \ No newline at end of file diff --git a/tests/util/util_users.sh b/tests/util/util_users.sh index 5f3734ac..e9e976ae 100644 --- a/tests/util/util_users.sh +++ b/tests/util/util_users.sh @@ -110,10 +110,6 @@ put_user_policy_userplus() { log 2 "'put user policy userplus' function requires username" return 1 fi - if [[ -z "$TEST_FILE_FOLDER" ]] && [[ -z "$GITHUB_ACTIONS" ]] && ! create_test_file_folder; then - log 2 "unable to create test file folder" - return 1 - fi cat < "$TEST_FILE_FOLDER"/user_policy_file { @@ -154,10 +150,6 @@ put_user_policy() { log 2 "attaching user policy requires user ID, role, bucket name" return 1 fi - if [[ -z "$TEST_FILE_FOLDER" ]] && [[ -z "$GITHUB_ACTIONS" ]] && ! create_test_file_folder; then - log 2 "unable to create test file folder" - return 1 - fi case $2 in "user") diff --git a/tests/versity.sh b/tests/versity.sh index 526293e7..f1621bfc 100644 --- a/tests/versity.sh +++ b/tests/versity.sh @@ -21,10 +21,6 @@ start_versity_process() { log 1 "start versity process function requires number" exit 1 fi - if ! create_test_file_folder; then - log 1 "error creating test log folder" - exit 1 - fi build_run_and_log_command # shellcheck disable=SC2181 if [[ $? -ne 0 ]]; then