Add PROPFIND/list test for public & authed path-style buckets

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required( VERSION 3.13 )
 
 project( xrootd-http/s3 )
 
-option( XROOTD_PLUGINS_BUILD_UNITTESTS "Build the scitokens-cpp unit tests" OFF )
+option( XROOTD_PLUGINS_BUILD_UNITTESTS "Build the XRootD plugins unit tests" OFF )
 option( XROOTD_PLUGINS_EXTERNAL_GTEST "Use an external/pre-installed copy of GTest" OFF )
 
 set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake )

test/CMakeLists.txt

@@ -118,6 +118,9 @@ add_test(NAME S3::s3_basic::test
 add_test(NAME S3::s3_basic::stress_test
   COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/s3-stress-test.sh" s3_basic)
 
+add_test(NAME S3::s3_basic::list_test
+  COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/s3-list-test.sh" s3_basic)
+
 list(APPEND S3_BASIC_TEST_LOGS ${CMAKE_CURRENT_BINARY_DIR}/tests/s3_basic/server.log)
 list(APPEND S3_BASIC_TEST_LOGS ${CMAKE_CURRENT_BINARY_DIR}/tests/s3_basic/client.log)
 
@@ -134,3 +137,10 @@ set_tests_properties(S3::s3_basic::stress_test
     ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR}"
     ATTACHED_FILES_ON_FAIL "${S3_BASIC_TEST_LOGS}"
 )
+
+set_tests_properties(S3::s3_basic::list_test
+  PROPERTIES
+    FIXTURES_REQUIRED S3::s3_basic
+    ENVIRONMENT "BINARY_DIR=${CMAKE_BINARY_DIR}"
+    ATTACHED_FILES_ON_FAIL "${S3_BASIC_TEST_LOGS}"
+)

test/s3-list-test.sh

@@ -0,0 +1,74 @@
+#!/bin/sh
+
+############################
+#        Test Setup        #
+############################
+TEST_NAME=$1
+
+if [ -z "$BINARY_DIR" ]; then
+  echo "\$BINARY_DIR environment variable is not set; cannot run test"
+  exit 1
+fi
+if [ ! -d "$BINARY_DIR" ]; then
+  echo "$BINARY_DIR is not a directory; cannot run test"
+  exit 1
+fi
+
+echo "Running $TEST_NAME - PROPFIND test"
+
+if [ ! -f "$BINARY_DIR/tests/$TEST_NAME/setup.sh" ]; then
+  echo "Test environment file $BINARY_DIR/tests/$TEST_NAME/setup.sh does not exist - cannot run test"
+  exit 1
+fi
+. "$BINARY_DIR/tests/$TEST_NAME/setup.sh"
+
+if [ -z "$XROOTD_URL" ]; then
+  echo "XRootD URL is not set; cannot test"
+  exit 1
+fi
+
+############################
+#     Start the tests      #
+############################
+
+# PROPFIND against public bucket with `hello` prefix
+RESPONSE_PUBLIC=$(curl --cacert "$X509_CA_FILE" -v --fail -X PROPFIND "$XROOTD_URL/test-public/hello" \
+  2>"$BINARY_DIR/tests/$TEST_NAME/propfind-client-public.log")
+CURL_EXIT_PUBLIC=$?
+
+if [ $CURL_EXIT_PUBLIC -ne 0 ]; then
+  echo "PROPFIND request against public bucket failed: CURL exit code $CURL_EXIT_PUBLIC"
+  exit 1
+fi
+
+# TODO: Adjust this XML when when lists are fixed -- this will fail until then
+# Validate the public bucket response contains the expected XML
+EXPECTED_XML_PUBLIC='<?xml version="1.0" encoding="UTF-8"?><D:multistatus xmlns:D="DAV:"><D:response><D:href>/test/hello_world.txt</D:href></D:response></D:multistatus>'
+if [ "$RESPONSE_PUBLIC" != "$EXPECTED_XML_PUBLIC" ]; then
+  echo "PROPFIND response for public bucket does not match expected output"
+  echo "Actual response: $RESPONSE_PUBLIC"
+  exit 1
+fi
+
+echo "Public PROPFIND test passed"
+
+# PROPFIND against authed bucket with `hello` prefix
+RESPONSE_AUTHED=$(curl --cacert "$X509_CA_FILE" -v --fail -X PROPFIND "$XROOTD_URL/test-authed/hello" \
+  2>"$BINARY_DIR/tests/$TEST_NAME/propfind-client-authed.log")
+CURL_EXIT_AUTHED=$?
+
+if [ $CURL_EXIT_AUTHED -ne 0 ]; then
+  echo "PROPFIND request against authed bucket failed"
+  exit 1
+fi
+
+# TODO: Adjust this XML when when lists are fixed -- this will fail until then
+# Validate the authed bucket response contains the expected XML
+EXPECTED_XML_AUTHED='<?xml version="1.0" encoding="UTF-8"?><D:multistatus xmlns:D="DAV:"><D:response><D:href>/test/hello_world.txt</D:href></D:response></D:multistatus>'
+if [ "$RESPONSE_AUTHED" != "$EXPECTED_XML_AUTHED" ]; then
+  echo "PROPFIND response for authed bucket does not match expected output"
+  echo "Actual response: $RESPONSE_AUTHED"
+  exit 1
+fi
+
+echo "Authed PROPFIND test passed"

test/s3-setup.sh

@@ -188,14 +188,18 @@ echo "Starting configuration of minio"
 "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" admin user add adminminio "$MINIO_USER" "$MINIO_PASSWORD"
 "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" alias set userminio "$MINIO_URL" "$MINIO_USER" "$MINIO_PASSWORD"
 "$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" admin policy attach adminminio readwrite --user "$MINIO_USER"
-"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" mb userminio/test-bucket
+"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" mb userminio/test-bucket-authed
+"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" mb userminio/test-bucket-public
+"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" anonymous set public userminio/test-bucket-public
+
 if [ $? -ne 0 ]; then
   echo "Failed to create test bucket in minio server"
   exit 1
 fi
 
 echo "Hello, World" > "$RUNDIR/hello_world.txt"
-"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket/hello_world.txt
+"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket-authed/hello_world.txt
+"$MC_BIN" --insecure --config-dir "$MINIO_CLIENTDIR" cp "$RUNDIR/hello_world.txt" userminio/test-bucket-public/hello_world.txt
 
 ####
 #    Starting XRootD config with S3 backend
@@ -234,9 +238,10 @@ ofs.osslib $BINARY_DIR/libXrdS3.so
 
 s3.trace debug
 
+# Setup the auth'ed bucket
 s3.begin
-s3.path_name /test
-s3.bucket_name test-bucket
+s3.path_name /test-authed
+s3.bucket_name test-bucket-authed
 s3.service_url $MINIO_URL
 s3.service_name $(hostname)
 s3.url_style path
@@ -245,6 +250,16 @@ s3.access_key_file $XROOTD_CONFIGDIR/access_key
 s3.secret_key_file $XROOTD_CONFIGDIR/secret_key
 s3.end
 
+# And the public/anonymous bucket
+s3.begin
+s3.path_name /test-public
+s3.bucket_name test-bucket-public
+s3.service_url $MINIO_URL
+s3.service_name $(hostname)
+s3.url_style path
+s3.region us-east-1
+s3.end
+
 EOF
 
 cat > $XROOTD_CONFIGDIR/authdb <<EOF

test/s3-stress-test.sh

@@ -28,7 +28,7 @@ IDX=1
 while [ $IDX -le 100 ]; do
   IDX=$(($IDX+1))
 
-  curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.log" > "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.out" &
+  curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test-authed/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.log" > "$BINARY_DIR/tests/$TEST_NAME/client-$IDX.out" &
   export CURL_${IDX}_PID=$!
 
 done

test/s3-test.sh

@@ -11,7 +11,7 @@ if [ ! -d "$BINARY_DIR" ]; then
   exit 1
 fi
 
-echo "Running $TEST_NAME - simple download"
+echo "Running $TEST_NAME - simple download, authed bucket"
 
 if [ ! -f "$BINARY_DIR/tests/$TEST_NAME/setup.sh" ]; then
   echo "Test environment file $BINARY_DIR/tests/$TEST_NAME/setup.sh does not exist - cannot run test"
@@ -24,22 +24,39 @@ if [ -z "$XROOTD_URL" ]; then
   exit 1
 fi
 
-CONTENTS=$(curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client.log")
+# Hit the authed bucket
+CONTENTS=$(curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test-authed/hello_world.txt" 2> "$BINARY_DIR/tests/$TEST_NAME/client.log")
 CURL_EXIT=$?
 
 if [ $CURL_EXIT -ne 0 ]; then
-  echo "Download of hello-world text failed"
+  echo "Download of hello-world text from authed bucket failed"
   exit 1
 fi
 
 if [ "$CONTENTS" != "Hello, World" ]; then
-  echo "Downloaded hello-world text is incorrect: $CONTENTS"
+  echo "Downloaded hello-world text from authed bucket is incorrect: $CONTENTS"
+  exit 1
+fi
+
+# Hit the public/anonymous bucket
+echo "Running $TEST_NAME - simple download, public bucket"
+# We still pass the CA file to curl to avoid having to pass -k for insecure https connections
+CONTENTS=$(curl --cacert $X509_CA_FILE -v --fail "$XROOTD_URL/test-public/hello_world.txt" 2>> "$BINARY_DIR/tests/$TEST_NAME/client.log")
+CURL_EXIT=$?
+
+if [ $CURL_EXIT -ne 0 ]; then
+  echo "Download of hello-world text from public bucket failed"
+  exit 1
+fi
+
+if [ "$CONTENTS" != "Hello, World" ]; then
+  echo "Downloaded hello-world text from public bucket is incorrect: $CONTENTS"
   exit 1
 fi
 
 echo "Running $TEST_NAME - missing object"
 
-HTTP_CODE=$(curl --cacert $X509_CA_FILE --output /dev/null -v --write-out '%{http_code}' "$XROOTD_URL/test/missing.txt" 2>> "$BINARY_DIR/tests/$TEST_NAME/client.log")
+HTTP_CODE=$(curl --cacert $X509_CA_FILE --output /dev/null -v --write-out '%{http_code}' "$XROOTD_URL/test-authed/missing.txt" 2>> "$BINARY_DIR/tests/$TEST_NAME/client.log")
 if [ "$HTTP_CODE" -ne 404 ]; then
   echo "Expected HTTP code is 404; actual was $HTTP_CODE"
   exit 1