From 4ee95a13169da91a2fb163fe32e540740b87ffd8 Mon Sep 17 00:00:00 2001 From: Ryan Hoffman Date: Thu, 7 Dec 2023 14:39:53 -0500 Subject: [PATCH 1/4] configurable S3_OBJECT_PREFIX env var preprended to object paths --- common/etc/nginx/include/s3gateway.js | 10 ++++++++++ docs/getting_started.md | 1 + 2 files changed, 11 insertions(+) diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index b1be836a..486db09b 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -69,6 +69,12 @@ const FOUR_O_FOUR_ON_EMPTY_BUCKET = utils.parseBoolean(process.env['FOUR_O_FOUR_ * @type {string} * */ const S3_STYLE = process.env['S3_STYLE']; +/** + * Prefix to be prepended to all object names, allowing the serving of a + * subset of a bucket. Default ''. + * @type {string} + */ +const S3_OBJECT_PREFIX = process.env['S3_OBJECT_PREFIX'] || ''; /** * Additional header prefixes to strip from the response before sending to the * client. This is useful for removing headers that may contain sensitive @@ -285,6 +291,10 @@ function s3uri(r) { const basePath = s3BaseUri(r); let path; + if (S3_OBJECT_PREFIX) { + uriPath = S3_OBJECT_PREFIX + uriPath + } + // Create query parameters only if directory listing is enabled. if (ALLOW_LISTING && !utils.parseBoolean(r.variables.forIndexPage)) { const queryParams = _s3DirQueryParams(uriPath, r.method); diff --git a/docs/getting_started.md b/docs/getting_started.md index 5e24732f..e04cb45e 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -21,6 +21,7 @@ running as a Container or as a Systemd service. | `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | | `AWS_SESSION_TOKEN` | No | | | Session token. | | `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | +| `S3_OBJECT_PREFIX` | No | | | Prefix to prepend to all S3 object paths | | `S3_REGION` | Yes | | | Region associated with API | | `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | | `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | From 6bd55578ecb68954cb8935d05b4b774a78a31d0a Mon Sep 17 00:00:00 2001 From: Ryan Hoffman Date: Thu, 14 Dec 2023 13:51:48 -0500 Subject: [PATCH 2/4] revert changes to s3gateway.js --- common/etc/nginx/include/s3gateway.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 486db09b..b1be836a 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -69,12 +69,6 @@ const FOUR_O_FOUR_ON_EMPTY_BUCKET = utils.parseBoolean(process.env['FOUR_O_FOUR_ * @type {string} * */ const S3_STYLE = process.env['S3_STYLE']; -/** - * Prefix to be prepended to all object names, allowing the serving of a - * subset of a bucket. Default ''. - * @type {string} - */ -const S3_OBJECT_PREFIX = process.env['S3_OBJECT_PREFIX'] || ''; /** * Additional header prefixes to strip from the response before sending to the * client. This is useful for removing headers that may contain sensitive @@ -291,10 +285,6 @@ function s3uri(r) { const basePath = s3BaseUri(r); let path; - if (S3_OBJECT_PREFIX) { - uriPath = S3_OBJECT_PREFIX + uriPath - } - // Create query parameters only if directory listing is enabled. if (ALLOW_LISTING && !utils.parseBoolean(r.variables.forIndexPage)) { const queryParams = _s3DirQueryParams(uriPath, r.method); From 11bab2264091898e099f3822c757ecb3735852f6 Mon Sep 17 00:00:00 2001 From: Ryan Hoffman Date: Thu, 14 Dec 2023 14:00:54 -0500 Subject: [PATCH 3/4] use PREFIX_LEADING_DIRECTORY_PATH to serve only a sub-"directory" of a bucket Thank you to @4141done for the suggestions! (#190) --- Dockerfile.buildkit.plus | 2 ++ Dockerfile.oss | 1 + Dockerfile.plus | 1 + common/etc/nginx/nginx.conf | 1 + common/etc/nginx/templates/default.conf.template | 6 +++--- docs/getting_started.md | 6 +++--- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile.buildkit.plus b/Dockerfile.buildkit.plus index b66df30f..36c7996f 100644 --- a/Dockerfile.buildkit.plus +++ b/Dockerfile.buildkit.plus @@ -12,6 +12,8 @@ ENV PROXY_CACHE_VALID_NOTFOUND "1m" ENV PROXY_CACHE_VALID_FORBIDDEN "30s" ENV CORS_ENABLED 0 ENV DIRECTORY_LISTING_PATH_PREFIX "" +ENV STRIP_LEADING_DIRECTORY_PATH "" +ENV PREFIX_LEADING_DIRECTORY_PATH "" COPY plus/usr /usr diff --git a/Dockerfile.oss b/Dockerfile.oss index 9c3f26a0..201db64b 100644 --- a/Dockerfile.oss +++ b/Dockerfile.oss @@ -11,6 +11,7 @@ ENV PROXY_CACHE_VALID_FORBIDDEN "30s" ENV CORS_ENABLED 0 ENV DIRECTORY_LISTING_PATH_PREFIX "" ENV STRIP_LEADING_DIRECTORY_PATH "" +ENV PREFIX_LEADING_DIRECTORY_PATH "" # We modify the nginx base image by: # 1. Adding configuration files needed for proxying private S3 buckets diff --git a/Dockerfile.plus b/Dockerfile.plus index d84f975f..80f2d15d 100644 --- a/Dockerfile.plus +++ b/Dockerfile.plus @@ -13,6 +13,7 @@ ENV PROXY_CACHE_VALID_FORBIDDEN "30s" ENV CORS_ENABLED 0 ENV DIRECTORY_LISTING_PATH_PREFIX "" ENV STRIP_LEADING_DIRECTORY_PATH "" +ENV PREFIX_LEADING_DIRECTORY_PATH "" COPY plus/etc/ssl /etc/ssl COPY plus/usr /usr diff --git a/common/etc/nginx/nginx.conf b/common/etc/nginx/nginx.conf index 75857f70..99b216a7 100644 --- a/common/etc/nginx/nginx.conf +++ b/common/etc/nginx/nginx.conf @@ -32,6 +32,7 @@ env PROXY_CACHE_VALID_FORBIDDEN; env HEADER_PREFIXES_TO_STRIP; env FOUR_O_FOUR_ON_EMPTY_BUCKET; env STRIP_LEADING_DIRECTORY_PATH; +env PREFIX_LEADING_DIRECTORY_PATH; events { worker_connections 1024; diff --git a/common/etc/nginx/templates/default.conf.template b/common/etc/nginx/templates/default.conf.template index e68413d7..d9eff188 100644 --- a/common/etc/nginx/templates/default.conf.template +++ b/common/etc/nginx/templates/default.conf.template @@ -13,10 +13,10 @@ map $request_uri $uri_full_path { "~^(?P.*?)(\?.*)*$" $path; } -# Remove a portion of request URL (if configured) +# Remove/replace a portion of request URL (if configured) map $uri_full_path $uri_path { - "~^$STRIP_LEADING_DIRECTORY_PATH(.*)" $1; - default $uri_full_path; + "~^$STRIP_LEADING_DIRECTORY_PATH(.*)" $PREFIX_LEADING_DIRECTORY_PATH$1; + default $PREFIX_LEADING_DIRECTORY_PATH$uri_full_path; } map $S3_STYLE $s3_host_hdr { diff --git a/docs/getting_started.md b/docs/getting_started.md index e04cb45e..4685f2d8 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -21,7 +21,6 @@ running as a Container or as a Systemd service. | `AWS_SECRET_ACCESS_KEY` | Yes | | | Secret access key | | `AWS_SESSION_TOKEN` | No | | | Session token. | | `S3_BUCKET_NAME` | Yes | | | Name of S3 bucket to proxy requests to | -| `S3_OBJECT_PREFIX` | No | | | Prefix to prepend to all S3 object paths | | `S3_REGION` | Yes | | | Region associated with API | | `S3_SERVER_PORT` | Yes | | | SSL/TLS port to connect to | | `S3_SERVER_PROTO` | Yes | `http`, `https` | | Protocol to used connect to S3 server | @@ -40,8 +39,9 @@ running as a Container or as a Systemd service. | `JS_TRUSTED_CERT_PATH` | No | | | Enables the `js_fetch_trusted_certificate` directive when retrieving AWS credentials and sets the path (on the container) to the specified path | | `HEADER_PREFIXES_TO_STRIP` | No | | | A list of HTTP header prefixes that exclude headers client responses. List should be specified in lower-case and a semicolon (;) should be used to as a deliminator between values. For example: `x-goog-;x-something-` | | `CORS_ENABLED` | No | `true`, `false` | `false` | Flag that enables CORS headers on GET requests and enables pre-flight OPTIONS requests. If enabled, this will add CORS headers for "fully open" cross domain requests by default, meaning all domains are allowed, similar to the settings show in [this example](https://enable-cors.org/server_nginx.html). CORS settings can be fine-tuned by overwriting the [`cors.conf.template`](/common/etc/nginx/templates/gateway/cors.conf.template) file. | -| `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) -| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). +| `CORS_ALLOWED_ORIGIN` | No | | | value to set to be returned from the CORS `Access-Control-Allow-Origin` header. This value is only used if CORS is enabled. (default: \*) | +| `STRIP_LEADING_DIRECTORY_PATH` | No | | | Removes a portion of the path in the requested URL (if configured). Useful when deploying to an ALB under a folder (eg. www.mysite.com/somepath). | +| `PREFIX_LEADING_DIRECTORY_PATH` | No | | | Prefix to prepend to all S3 object paths. Useful to serve only a subset of an S3 bucket. When used in combination with `STRIP_LEADING_DIRECTORY_PATH`, this allows the leading path to be replaced, rather than just removed. | If you are using [AWS instance profile credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html), you will need to omit the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` variables from From 464abd9ec7d271b5ec0e24e6f8a1f0d4aa954042 Mon Sep 17 00:00:00 2001 From: Javier Evans Date: Fri, 15 Dec 2023 14:28:05 -0800 Subject: [PATCH 4/4] add basic tests for PREFIX_LEADING_DIRECTORY_PATH --- standalone_ubuntu_oss_install.sh | 2 ++ test.sh | 36 ++++++++++++++++++++++---------- test/docker-compose.yaml | 1 + test/integration/test_api.sh | 15 +++++++++++++ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/standalone_ubuntu_oss_install.sh b/standalone_ubuntu_oss_install.sh index 774d4981..7d144a79 100644 --- a/standalone_ubuntu_oss_install.sh +++ b/standalone_ubuntu_oss_install.sh @@ -173,6 +173,8 @@ PROXY_CACHE_VALID_FORBIDDEN=${PROXY_CACHE_VALID_FORBIDDEN:-'30s'} CORS_ENABLED=${CORS_ENABLED:-'false'} # Configure portion of URL to be removed (optional) STRIP_LEADING_DIRECTORY_PATH=${STRIP_LEADING_DIRECTORY_PATH:-''} +# Configure portion of URL to be added to the beginning of the requested path (optional) +PREFIX_LEADING_DIRECTORY_PATH=${PREFIX_LEADING_DIRECTORY_PATH:-''} EOF # By enabling CORS, we also need to enable the OPTIONS method which diff --git a/test.sh b/test.sh index 30f5cc3d..7ea8ad27 100755 --- a/test.sh +++ b/test.sh @@ -249,10 +249,13 @@ integration_test() { printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4" printf "\033[34;1m▶\033[0m" printf "\e[1m Integration test suite with STRIP_LEADING_DIRECTORY_PATH=%s\e[22m\n" "$5" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with PREFIX_LEADING_DIRECTORY_PATH=%s\e[22m\n" "$6" + p "Starting Docker Compose Environment" # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs - - COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 STRIP_LEADING_DIRECTORY_PATH=$5 compose up -d + COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 STRIP_LEADING_DIRECTORY_PATH=$5 PREFIX_LEADING_DIRECTORY_PATH=$6 compose up -d if [ "${wait_for_it_installed}" ]; then if [ -x "${wait_for_it_cmd}" ]; then @@ -261,8 +264,8 @@ integration_test() { fi p "Starting HTTP API tests (v$1 signatures)" - echo " test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4 $5" - bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4" "$5"; + echo " test/integration/test_api.sh \"$test_server\" \"$test_dir\" $1 $2 $3 $4 $5 $6" + bash "${test_dir}/integration/test_api.sh" "${test_server}" "${test_dir}" "$1" "$2" "$3" "$4" "$5" "$6"; # We check to see if NGINX is in fact using the correct version of AWS # signatures as it was configured to do. @@ -404,41 +407,52 @@ runUnitTestWithSessionToken "s3gateway_test.js" integration_test_data p "Testing API with AWS Signature V2 and allow directory listing off" -integration_test 2 0 0 0 "" +integration_test 2 0 0 0 "" "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and allow directory listing on" -integration_test 2 1 0 0 "" +integration_test 2 1 0 0 "" "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and static site on" -integration_test 2 0 1 0 "" +integration_test 2 0 1 0 "" "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and allow directory listing on and append slash and allow index" -integration_test 2 1 1 1 "" +integration_test 2 1 1 1 "" "" compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and allow directory listing off" -integration_test 4 0 0 0 "" +integration_test 4 0 0 0 "" "" compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and allow directory listing on and appending /" -integration_test 4 1 0 1 "" +integration_test 4 1 0 1 "" "" compose stop nginx-s3-gateway # Restart with new config p "Test API with AWS Signature V4 and static site on appending /" -integration_test 4 0 1 1 "" +integration_test 4 0 1 1 "" "" compose stop nginx-s3-gateway # Restart with new config p "Testing API with AWS Signature V2 and allow directory listing off and prefix stripping on" -integration_test 2 0 0 0 /my-bucket +integration_test 2 0 0 0 /my-bucket "" + +compose stop nginx-s3-gateway # Restart with new config + +p "Test API with AWS Signature V4 and prefix leading directory path on" +integration_test 4 0 0 0 "" "/b" + +p "Test API with AWS Signature V4 and prefix leading directory path on and prefix stripping on" +integration_test 4 0 0 0 "/tostrip" "/b" + +p "Testing API with AWS Signature V2 and prefix leading directory path" +integration_test 2 0 0 0 "" "/b" p "All integration tests complete" diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index 0a162f14..b129ea8e 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -27,6 +27,7 @@ services: PROVIDE_INDEX_PAGE: APPEND_SLASH_FOR_POSSIBLE_DIRECTORY: STRIP_LEADING_DIRECTORY_PATH: + PREFIX_LEADING_DIRECTORY_PATH: AWS_SIGS_VERSION: STATIC_SITE_HOSTING: PROXY_CACHE_MAX_SIZE: "10g" diff --git a/test/integration/test_api.sh b/test/integration/test_api.sh index 26dd6fbd..082bb23a 100644 --- a/test/integration/test_api.sh +++ b/test/integration/test_api.sh @@ -26,6 +26,8 @@ allow_directory_list=$4 index_page=$5 append_slash=$6 strip_leading_directory=$7 +prefix_leading_directory_path=$8 + test_fail_exit_code=2 no_dep_exit_code=3 checksum_length=32 @@ -159,6 +161,19 @@ for (( i=1; i<=3; i++ )); do done set -o errexit +if [ -n "${prefix_leading_directory_path}" ]; then + assertHttpRequestEquals "GET" "/c/d.txt" "data/bucket-1/b/c/d.txt" + + if [ -n "${strip_leading_directory}" ]; then + # When these two flags are used together, stripped value is basically + # replaced with the specified prefix + assertHttpRequestEquals "GET" "/tostrip/c/d.txt" "data/bucket-1/b/c/d.txt" + fi + + # Exit early for this case since all tests following will fail because of the added prefix + exit 0 +fi + # Ordinary filenames assertHttpRequestEquals "HEAD" "a.txt" "200"