Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test/rest put bucket policy #967

Merged
merged 1 commit into from
Dec 4, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
test: REST PutBucketPolicy, logging, user changes
lrm25 committed Dec 4, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit a43eec0ae74b462a6fa6ec1497d8bc6e497f4870
1 change: 1 addition & 0 deletions tests/commands/create_multipart_upload.sh
Original file line number Diff line number Diff line change
@@ -54,6 +54,7 @@ create_multipart_upload_with_user() {
return 1
fi
upload_id="${upload_id//\"/}"
echo "$upload_id"
return 0
}

56 changes: 36 additions & 20 deletions tests/commands/get_bucket_policy.sh
Original file line number Diff line number Diff line change
@@ -97,32 +97,48 @@ get_bucket_policy_s3cmd() {
policy_brackets=false
# NOTE: versitygw sends policies back in multiple lines here, direct in single line
while IFS= read -r line; do
if [[ $policy_brackets == false ]]; then
policy_line=$(echo "$line" | grep 'Policy: ')
if [[ $policy_line != "" ]]; then
if [[ $policy_line != *'{'* ]]; then
break
fi
if [[ $policy_line == *'}'* ]]; then
log 5 "policy on single line"
bucket_policy=${policy_line//Policy:/}
break
else
policy_brackets=true
bucket_policy+="{"
fi
fi
else
bucket_policy+=$line
if [[ $line == "" ]]; then
break
fi
if check_and_load_policy_info; then
break
fi
done <<< "$info"
log 5 "bucket policy: $bucket_policy"
return 0
}

# return 0 for no policy, single-line policy, or loading complete, 1 for still searching or loading
check_and_load_policy_info() {
if [[ $policy_brackets == false ]]; then
if search_for_first_policy_line_or_full_policy; then
return 0
fi
else
bucket_policy+=$line
if [[ $line == "}" ]]; then
return 0
fi
fi
return 1
}

# return 0 for empty or single-line policy, 1 for other cases
search_for_first_policy_line_or_full_policy() {
policy_line=$(echo "$line" | grep 'Policy: ')
if [[ $policy_line != "" ]]; then
if [[ $policy_line != *'{'* ]]; then
return 0
fi
if [[ $policy_line == *'}'* ]]; then
log 5 "policy on single line"
bucket_policy=${policy_line//Policy:/}
return 0
else
policy_brackets=true
bucket_policy+="{"
fi
fi
return 1
}

get_bucket_policy_mc() {
record_command "get-bucket-policy" "client:mc"
if [[ $# -ne 1 ]]; then
5 changes: 3 additions & 2 deletions tests/commands/head_bucket.sh
Original file line number Diff line number Diff line change
@@ -36,14 +36,15 @@ head_bucket() {
elif [[ $1 == 'mc' ]]; then
bucket_info=$(send_command mc --insecure stat "$MC_ALIAS"/"$2" 2>&1) || exit_code=$?
else
fail "invalid command type $1"
log 2 "invalid command type $1"
fi
if [ $exit_code -ne 0 ]; then
log 2 "error getting bucket info: $bucket_info"
if [[ "$bucket_info" == *"404"* ]] || [[ "$bucket_info" == *"does not exist"* ]]; then
return 1
fi
log 2 "error getting bucket info: $bucket_info"
return 2
fi
echo "$bucket_info"
return 0
}
28 changes: 21 additions & 7 deletions tests/env.sh
Original file line number Diff line number Diff line change
@@ -63,13 +63,7 @@ check_universal_vars() {
source_config_file
fi
if [ -n "$COMMAND_LOG" ]; then
if [ -e "$COMMAND_LOG" ]; then
if ! error=$(rm "$COMMAND_LOG"); then
log 3 "error removing command log: $error"
return 1
fi
fi
echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"
init_command_log
fi

if [ "$GITHUB_ACTIONS" != "true" ] && [ -r "$SECRETS_FILE" ]; then
@@ -80,6 +74,10 @@ check_universal_vars() {
fi

if [[ -n "$LOG_LEVEL" ]]; then
if [[ $LOG_LEVEL -lt 2 ]]; then
log 1 "log level must be 2 or greater"
exit 1
fi
export LOG_LEVEL_INT=$LOG_LEVEL
fi

@@ -139,6 +137,22 @@ check_universal_vars() {
export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
}

delete_command_log() {
if [ -e "$COMMAND_LOG" ]; then
if ! error=$(rm "$COMMAND_LOG"); then
log 2 "error removing command log: $error"
return 1
fi
fi
}

init_command_log() {
if ! delete_command_log; then
exit 1
fi
echo "******** $(date +"%Y-%m-%d %H:%M:%S") $BATS_TEST_NAME COMMANDS ********" >> "$COMMAND_LOG"
}

check_versity_vars() {
if [ -z "$LOCAL_FOLDER" ]; then
log 1 "LOCAL_FOLDER missing"
6 changes: 4 additions & 2 deletions tests/logger.sh
Original file line number Diff line number Diff line change
@@ -129,8 +129,10 @@ log_message() {
return 1
fi
now="$(date "+%Y-%m-%d %H:%M:%S")"
echo "$now $1 $2"
if [[ ( "$1" == "CRIT" ) || ( "$1" == "ERROR" ) ]]; then
echo "$now $1 $2" >&2
fi
if [[ -n "$TEST_LOG_FILE" ]]; then
echo "$now $1 $2" >> "$TEST_LOG_FILE"
echo "$now $1 $2" >> "$TEST_LOG_FILE.tmp"
fi
}
51 changes: 51 additions & 0 deletions tests/rest_scripts/put_bucket_policy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

# Copyright 2024 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

source ./tests/rest_scripts/rest.sh

# Fields

# shellcheck disable=SC2153
bucket_name="$BUCKET_NAME"
# shellcheck disable=SC2153
policy_file="$POLICY_FILE"

payload="$(cat "$policy_file")"

payload_hash="$(echo -n "$payload" | sha256sum | awk '{print $1}')"
current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")

canonical_request="PUT
/$bucket_name
policy=
host:$host
x-amz-content-sha256:$payload_hash
x-amz-date:$current_date_time

host;x-amz-content-sha256;x-amz-date
$payload_hash"

create_canonical_hash_sts_and_signature

curl_command+=(curl -ks -w "\"%{http_code}\"" -X PUT "$AWS_ENDPOINT_URL/$bucket_name?policy="
-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature\""
-H "\"x-amz-content-sha256: $payload_hash\""
-H "\"x-amz-date: $current_date_time\""
-d "\"${payload//\"/\\\"}\""
-o "$OUTPUT_FILE")

# shellcheck disable=SC2154
eval "${curl_command[*]}" 2>&1
54 changes: 44 additions & 10 deletions tests/setup.sh
Original file line number Diff line number Diff line change
@@ -27,6 +27,13 @@ source ./tests/versity.sh
setup() {
base_setup

if [ -n "$TEST_LOG_FILE" ]; then
if ! error=$(touch "$TEST_LOG_FILE.tmp" 2>&1); then
log 2 "error creating log file: $error"
exit 1
fi
fi

log 4 "Running test $BATS_TEST_NAME"
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
start_time=$(date +%s)
@@ -48,32 +55,59 @@ setup() {
export AWS_PROFILE
}

# fail a test
# param: error message
#fail() {
# log 1 "$1"
# exit 1
#}
delete_temp_log_if_exists() {
if [ -e "$TEST_LOG_FILE.tmp" ]; then
if ! error=$(rm "$TEST_LOG_FILE.tmp" 2>&1); then
log 2 "error deleting temp log: $error"
return 1
fi
fi
return 0
}

# bats teardown function
teardown() {
if [[ ( "$BATS_TEST_COMPLETED" -ne 1 ) && ( -e "$COMMAND_LOG" ) ]]; then
cat "$COMMAND_LOG"
echo "**********************************************************************************"
fi
# shellcheck disable=SC2154
if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
log 3 "error deleting bucket $BUCKET_ONE_NAME or contents"
fi
if ! bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_TWO_NAME"; then
log 3 "error deleting bucket $BUCKET_TWO_NAME or contents"
fi
if user_exists "$USERNAME_ONE" && ! delete_user "$USERNAME_ONE"; then
log 3 "error deleting user $USERNAME_ONE"
fi
if user_exists "$USERNAME_TWO" && ! delete_user "$USERNAME_TWO"; then
log 3 "error deleting user $USERNAME_TWO"
fi
if [ "$REMOVE_TEST_FILE_FOLDER" == "true" ]; then
log 6 "removing test file folder"
if ! error=$(rm -rf "${TEST_FILE_FOLDER:?}" 2>&1); then
log 3 "unable to remove test file folder: $error"
fi
fi
if [[ "$BATS_TEST_COMPLETED" -ne 1 ]]; then
if [[ -e "$COMMAND_LOG" ]]; then
cat "$COMMAND_LOG"
echo "**********************************************************************************"
fi
if [[ -e "$TEST_LOG_FILE.tmp" ]]; then
echo "********************************** LOG *******************************************"
cat "$TEST_LOG_FILE.tmp"
echo "**********************************************************************************"
fi
fi
if ! delete_command_log; then
log 3 "error deleting command log"
fi
if [ -e "$TEST_LOG_FILE.tmp" ]; then
if ! error=$(cat "$TEST_LOG_FILE.tmp" >> "$TEST_LOG_FILE" 2>&1); then
log 2 "error appending temp log to main log: $error"
fi
if ! delete_temp_log_if_exists; then
log 2 "error deleting temp log"
fi
fi
stop_versity
if [[ $LOG_LEVEL -ge 5 ]] || [[ -n "$TIME_LOG" ]]; then
end_time=$(date +%s)
7 changes: 1 addition & 6 deletions tests/test_common.sh
Original file line number Diff line number Diff line change
@@ -382,12 +382,7 @@ test_common_get_put_delete_bucket_policy() {
assert_success

effect="Allow"
#principal="*"
if [[ $DIRECT == "true" ]]; then
principal="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/s3user\"}"
else
principal="\"*\""
fi
principal="*"
action="s3:GetObject"
resource="arn:aws:s3:::$BUCKET_ONE_NAME/*"

3 changes: 0 additions & 3 deletions tests/test_common_acl.sh
Original file line number Diff line number Diff line change
@@ -44,9 +44,6 @@ test_put_bucket_acl_s3cmd() {
}

test_common_put_bucket_acl() {
if [[ $RECREATE_BUCKETS == "false" ]]; then
skip "https://github.com/versity/versitygw/issues/716"
fi
assert [ $# -eq 1 ]

run setup_bucket "$1" "$BUCKET_ONE_NAME"
20 changes: 20 additions & 0 deletions tests/test_rest.sh
Original file line number Diff line number Diff line change
@@ -42,6 +42,8 @@ source ./tests/util/util_tags.sh
source ./tests/util/util_time.sh
source ./tests/util/util_versioning.sh

export RUN_USERS=true

@test "test_rest_list_objects" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success
@@ -422,4 +424,22 @@ source ./tests/util/util_versioning.sh

run get_and_check_no_policy_error "$BUCKET_ONE_NAME"
assert_success
}

@test "REST - put policy" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
assert_success

run setup_user_versitygw_or_direct "$USERNAME_ONE" "$PASSWORD_ONE" "user" "$BUCKET_ONE_NAME"
assert_success
log 5 "username: ${lines[0]}"
log 5 "password: ${lines[1]}"

sleep 5

run setup_policy_with_single_statement "$TEST_FILE_FOLDER/policy_file.txt" "2012-10-17" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME"
assert_success

run put_and_check_policy_rest "$BUCKET_ONE_NAME" "$TEST_FILE_FOLDER/policy_file.txt" "Allow" "$USERNAME_ONE" "s3:PutBucketTagging" "arn:aws:s3:::$BUCKET_ONE_NAME"
assert_success
}
11 changes: 5 additions & 6 deletions tests/test_s3api_bucket.sh
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ source ./tests/setup.sh
source ./tests/util/util.sh
source ./tests/util/util_create_bucket.sh
source ./tests/util/util_file.sh
source ./tests/util/util_head_bucket.sh
source ./tests/util/util_lock_config.sh
source ./tests/util/util_tags.sh
source ./tests/util/util_users.sh
@@ -101,13 +102,11 @@ export RUN_USERS=true
}

@test "test_head_bucket_doesnt_exist" {
run setup_bucket "s3api" "$BUCKET_ONE_NAME"
if [ "$RECREATE_BUCKETS" == "false" ]; then
skip "skip test for static buckets"
fi
run bucket_info_without_bucket
assert_success

head_bucket "s3api" "$BUCKET_ONE_NAME"a || local info_result=$?
[[ $info_result -eq 1 ]] || fail "bucket info for non-existent bucket returned"
[[ $bucket_info == *"404"* ]] || fail "404 not returned for non-existent bucket info"
bucket_cleanup "s3api" "$BUCKET_ONE_NAME"
}

@test "test_head_bucket_invalid_name" {
2 changes: 1 addition & 1 deletion tests/test_s3api_multipart.sh
Original file line number Diff line number Diff line change
@@ -119,7 +119,7 @@ source ./tests/commands/list_multipart_uploads.sh
run get_and_check_legal_hold "s3api" "$BUCKET_ONE_NAME" "$bucket_file" "OFF"
assert_success

run download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy" || fail "error getting object"
run download_and_compare_file "s3api" "$TEST_FILE_FOLDER/$bucket_file" "$BUCKET_ONE_NAME" "$bucket_file" "$TEST_FILE_FOLDER/$bucket_file-copy"
assert_success
}

Loading