Test cmdline rest delete objects

.github/workflows/docker-bats.yaml

@@ -25,4 +25,4 @@ jobs:
         run: sudo apt-get install -y docker-compose
 
       - name: Run Docker Container
-        run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from posix_backend posix_backend
+        run: docker-compose -f tests/docker-compose-bats.yml up --exit-code-from s3api_only s3api_only

.github/workflows/system.yml

@@ -179,6 +179,7 @@ jobs:
           USERNAME_TWO: HIJKLMN
           PASSWORD_TWO: 8901234
           TEST_FILE_FOLDER: ${{ github.workspace }}/versity-gwtest-files
+          REMOVE_TEST_FILE_FOLDER: true
         run: |
           make testbin
           export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPQRST

tests/Dockerfile_test_bats

@@ -86,4 +86,5 @@ RUN openssl genpkey -algorithm RSA -out versitygw-docker.pem -pkeyopt rsa_keygen
 ENV WORKSPACE=.
 ENV VERSITYGW_TEST_ENV=$CONFIG_FILE
 
-CMD ["tests/run_all.sh"]
+ENTRYPOINT ["tests/run.sh"]
+CMD ["s3api,s3,s3cmd,mc,rest"]

tests/commands/create_presigned_url.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+create_presigned_url() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "create presigned url function requires command type, bucket, and filename"
+    return 1
+  fi
+
+  local presign_result=0
+  if [[ $1 == 'aws' ]]; then
+    presigned_url=$(aws s3 presign "s3://$2/$3" --expires-in 900) || presign_result=$?
+  elif [[ $1 == 's3cmd' ]]; then
+    presigned_url=$(s3cmd --no-check-certificate "${S3CMD_OPTS[@]}" signurl "s3://$2/$3" "$(echo "$(date +%s)" + 900 | bc)") || presign_result=$?
+  elif [[ $1 == 'mc' ]]; then
+    presigned_url_data=$(mc --insecure share download --recursive "$MC_ALIAS/$2/$3") || presign_result=$?
+    presigned_url="${presigned_url_data#*Share: }"
+  else
+    log 2 "unrecognized command type $1"
+    return 1
+  fi
+  if [[ $presign_result -ne 0 ]]; then
+    log 2 "error generating presigned url: $presigned_url"
+    return 1
+  fi
+  export presigned_url
+  return 0
+}

tests/commands/delete_object.sh

@@ -31,6 +31,8 @@ delete_object() {
     delete_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate rm "s3://$2/$3" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
     delete_object_error=$(mc --insecure rm "$MC_ALIAS/$2/$3" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    delete_object_rest "$2" "$3" || exit_code=$?
   else
     log 2 "invalid command type $1"
     return 1
@@ -79,4 +81,44 @@ delete_object_with_user() {
     return 1
   fi
   return 0
+}
+
+delete_object_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'delete_object_rest' requires bucket name, object name"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="DELETE
+/$1/$2
+
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(curl -ks -w "%{http_code}" -X DELETE "$header://$aws_endpoint_url_address/$1/$2" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER"/delete_object_error.txt 2>&1)
+  if [[ "$reply" != "204" ]]; then
+    log 2 "delete object command returned error: $(cat "$TEST_FILE_FOLDER"/delete_object_error.txt)"
+    return 1
+  fi
+  return 0
 }

tests/commands/get_object.sh

@@ -30,6 +30,8 @@ get_object() {
     get_object_error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate get "s3://$2/$3" "$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
     get_object_error=$(mc --insecure get "$MC_ALIAS/$2/$3" "$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    get_object_rest "$2" "$3" "$4" || exit_code=$?
   else
     log 2 "'get object' command not implemented for '$1'"
     return 1
@@ -83,3 +85,45 @@ get_object_with_user() {
   fi
   return 0
 }
+
+get_object_rest() {
+  log 6 "get_object_rest"
+  if [ $# -ne 3 ]; then
+    log 2 "'get_object_rest' requires bucket name, object name, output file"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1/$2
+
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1/$2" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$3" 2>&1)
+  log 5 "reply: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "get object command returned error: $(cat "$3")"
+    return 1
+  fi
+  return 0
+}

tests/commands/get_object_tagging.sh

@@ -21,7 +21,7 @@ get_object_tagging() {
     return 1
   fi
   local result
-  if [[ $1 == 'aws' ]]; then
+  if [[ $1 == 'aws' ]] || [[ $1 == 's3api' ]]; then
     tags=$(aws --no-verify-ssl s3api get-object-tagging --bucket "$2" --key "$3" 2>&1) || result=$?
   elif [[ $1 == 'mc' ]]; then
     tags=$(mc --insecure tag list "$MC_ALIAS"/"$2"/"$3" 2>&1) || result=$?

tests/commands/put_object.sh

@@ -33,6 +33,8 @@ put_object() {
     error=$(s3cmd "${S3CMD_OPTS[@]}" --no-check-certificate put "$2" s3://"$3/$4" 2>&1) || exit_code=$?
   elif [[ $1 == 'mc' ]]; then
     error=$(mc --insecure put "$2" "$MC_ALIAS/$3/$4" 2>&1) || exit_code=$?
+  elif [[ $1 == 'rest' ]]; then
+    put_object_rest "$2" "$3" "$4" || exit_code=$?
   else
     log 2 "'put object' command not implemented for '$1'"
     return 1
@@ -66,3 +68,43 @@ put_object_with_user() {
   fi
   return 0
 }
+
+put_object_rest() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest' requires local file, bucket name, key"
+    return 1
+  fi
+
+  generate_hash_for_payload_file "$1"
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="PUT
+/$2/$3
+
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$2/$3" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -T "$1" -o "$TEST_FILE_FOLDER"/put_object_error.txt 2>&1)
+  if [[ "$reply" != "200" ]]; then
+    log 2 "put object command returned error: $(cat "$TEST_FILE_FOLDER"/put_object_error.txt)"
+    return 1
+  fi
+  return 0
+}

tests/docker-compose-bats.yml

@@ -1,34 +1,42 @@
 services:
   no_certs:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      context: ../
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.nocerts
   static_buckets:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      context: ../
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.static
   posix_backend:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      context: ../
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.default
     image: bats_test
   s3_backend:
     build:
-      context: .
-      dockerfile: Dockerfile_test_bats
+      context: ../
+      dockerfile: tests/Dockerfile_test_bats
       args:
         - CONFIG_FILE=tests/.env.s3
         - SECRETS_FILE=tests/.secrets.s3
+  s3api_only:
+    build:
+      context: ../
+      dockerfile: tests/Dockerfile_test_bats
+      args:
+        - CONFIG_FILE=tests/.env.default
+    image: bats_test
+    command: ["s3api"]
   direct:
     build:
-      context: .
-      dockerfile: Dockerfile_direct
+      context: ../
+      dockerfile: tests/Dockerfile_direct
     volumes:
       - ./.env.direct:/home/tester/tests/.env.direct
       - ./.secrets.direct:/home/tester/tests/.secrets.direct