From 469b1150c6a64720464a77c026249bb2e59f65cf Mon Sep 17 00:00:00 2001 From: Kevin Balthaser Date: Wed, 22 Nov 2023 15:25:33 -0500 Subject: [PATCH 1/5] Add missing `x-amz-content-sha256` header when generating headers for s3 authorization. closes #518 --- packages/opal-common/opal_common/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/opal-common/opal_common/utils.py b/packages/opal-common/opal_common/utils.py index 9458e5d21..799245592 100644 --- a/packages/opal-common/opal_common/utils.py +++ b/packages/opal-common/opal_common/utils.py @@ -136,6 +136,7 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): return { "x-amz-date": amzdate, + "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "Authorization": authorization_header, } From 111701096aa7a19daafa8bf6368a4996566ce39c Mon Sep 17 00:00:00 2001 From: Kevin Balthaser Date: Tue, 28 Nov 2023 10:42:24 -0500 Subject: [PATCH 2/5] Add new configuration option to allow setting the AWS region when tracking a bundle from an S3 bucket. Originally was hard-coded to `us-east-1`. For backwards compatibility, if the new option is not provided, we will default back to `us-east-1`. --- .../docs/tutorials/track_an_api_bundle_server.mdx | 8 +++++--- .../opal-common/opal_common/sources/api_policy_source.py | 4 +++- packages/opal-common/opal_common/utils.py | 6 ++++-- packages/opal-server/opal_server/config.py | 5 +++++ .../opal-server/opal_server/policy/watcher/factory.py | 5 +++++ 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/documentation/docs/tutorials/track_an_api_bundle_server.mdx b/documentation/docs/tutorials/track_an_api_bundle_server.mdx index 918ebde60..184df814c 100644 --- a/documentation/docs/tutorials/track_an_api_bundle_server.mdx +++ b/documentation/docs/tutorials/track_an_api_bundle_server.mdx @@ -37,9 +37,11 @@ You can configure how the OPAL-server will authenticate itself with the bundle s | Variables | Description | Example | | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------- | -| POLICY_BUNDLE_SERVER_TYPE | `HTTP` (authenticated with bearer token,or nothing), `AWS-S3`(Authenticated with [AWS REST Auth](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html) | AWS-S3 | -| POLICY_BUNDLE_SERVER_TOKEN_ID | The Secret Token Id (AKA user id, AKA access-key) sent to the API bundle server. | AKIAIOSFODNN7EXAMPLE | -| POLICY_BUNDLE_SERVER_TOKEN | The Secret Token (AKA password, AKA secret-key) sent to the API bundle server. | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | +| POLICY_BUNDLE_SERVER_TYPE | `HTTP` (authenticated with bearer token,or nothing), `AWS-S3`(Authenticated with [AWS REST Auth](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html) | AWS-S3 | +| POLICY_BUNDLE_SERVER_TOKEN_ID | The Secret Token Id (AKA user id, AKA access-key) sent to the API bundle server. | AKIAIOSFODNN7EXAMPLE | +| POLICY_BUNDLE_SERVER_TOKEN | The Secret Token (AKA password, AKA secret-key) sent to the API bundle server. | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | +| POLICY_BUNDLE_SERVER_TOKEN | The Secret Token (AKA password, AKA secret-key) sent to the API bundle server. | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | +| POLICY_BUNDLE_SERVER_AWS_REGION| The AWS Region if using `AWS-S3` Defaults to `us-east-1` | us-east-1 | ## Docker compose example diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 6b04054d2..5ab5d34f5 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -50,6 +50,7 @@ def __init__( polling_interval: int = 0, token: Optional[str] = None, token_id: Optional[str] = None, + region: Optional[str] = None, bundle_server_type: Optional[PolicyBundleServerType] = None, policy_bundle_path=".", policy_bundle_git_add_pattern="*", @@ -62,6 +63,7 @@ def __init__( self.token = token self.token_id = token_id self.server_type = bundle_server_type + self.region = region self.bundle_hash = None self.etag = None self.tmp_bundle_path = Path(policy_bundle_path) @@ -136,7 +138,7 @@ def build_auth_headers(self, token=None, path=None): host = split_url.netloc path = split_url.path + "/" + path - return build_aws_rest_auth_headers(self.token_id, token, host, path) + return build_aws_rest_auth_headers(self.token_id, token, host, path, self.region) else: return {} diff --git a/packages/opal-common/opal_common/utils.py b/packages/opal-common/opal_common/utils.py index 799245592..4da855338 100644 --- a/packages/opal-common/opal_common/utils.py +++ b/packages/opal-common/opal_common/utils.py @@ -56,7 +56,7 @@ def get_authorization_header(token: str) -> Tuple[str, str]: return "Authorization", f"Bearer {token}" -def build_aws_rest_auth_headers(key_id: str, secret_key: str, host: str, path: str): +def build_aws_rest_auth_headers(key_id: str, secret_key: str, host: str, path: str, region: str): """Use the AWS signature algorithm (https://docs.aws.amazon.com/AmazonS3/la test/userguide/RESTAuthentication.html) to generate the hTTP headers. @@ -101,7 +101,9 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): + payload_hash ) - region = "us-east-1" + if not region: + region = "us-east-1" + algorithm = "AWS4-HMAC-SHA256" credential_scope = datestamp + "/" + region + "/" + "s3" + "/" + "aws4_request" diff --git a/packages/opal-server/opal_server/config.py b/packages/opal-server/opal_server/config.py index 0f2a05a00..20d7be78f 100644 --- a/packages/opal-server/opal_server/config.py +++ b/packages/opal-server/opal_server/config.py @@ -128,6 +128,11 @@ class OpalServerConfig(Confi): None, description="The id of the secret token to be sent to API bundle server", ) + POLICY_BUNDLE_SERVER_AWS_REGION = confi.str( + "POLICY_BUNDLE_SERVER_AWS_REGION", + None, + description="The AWS region of the S3 bucket", + ) POLICY_BUNDLE_TMP_PATH = confi.str( "POLICY_BUNDLE_TMP_PATH", "/tmp/bundle.tar.gz", diff --git a/packages/opal-server/opal_server/policy/watcher/factory.py b/packages/opal-server/opal_server/policy/watcher/factory.py index dabf8cf73..97c2de771 100644 --- a/packages/opal-server/opal_server/policy/watcher/factory.py +++ b/packages/opal-server/opal_server/policy/watcher/factory.py @@ -27,6 +27,7 @@ def setup_watcher_task( policy_bundle_token: str = None, policy_bundle_token_id: str = None, policy_bundle_server_type: str = None, + policy_bundle_aws_region: str = None, extensions: Optional[List[str]] = None, bundle_ignore: Optional[List[str]] = None, ) -> BasePolicyWatcherTask: @@ -115,6 +116,9 @@ def setup_watcher_task( policy_bundle_server_type = load_conf_if_none( policy_bundle_server_type, opal_server_config.POLICY_BUNDLE_SERVER_TYPE ) + policy_bundle_aws_region = load_conf_if_none( + policy_bundle_aws_region, opal_server_config.POLICY_BUNDLE_SERVER_AWS_REGION + ) watcher = ApiPolicySource( remote_source_url=remote_source_url, local_clone_path=clone_path, @@ -124,6 +128,7 @@ def setup_watcher_task( bundle_server_type=policy_bundle_server_type, policy_bundle_path=opal_server_config.POLICY_BUNDLE_TMP_PATH, policy_bundle_git_add_pattern=opal_server_config.POLICY_BUNDLE_GIT_ADD_PATTERN, + region=policy_bundle_aws_region ) else: raise ValueError("Unknown value for OPAL_POLICY_SOURCE_TYPE") From 75cce6c89961e567fc9d8f9918abf7eb8080af02 Mon Sep 17 00:00:00 2001 From: Kevin Balthaser Date: Mon, 4 Dec 2023 08:55:17 -0500 Subject: [PATCH 3/5] Move SHA256 of empty string to constant. --- packages/opal-common/opal_common/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opal-common/opal_common/utils.py b/packages/opal-common/opal_common/utils.py index 4da855338..d39fa3091 100644 --- a/packages/opal-common/opal_common/utils.py +++ b/packages/opal-common/opal_common/utils.py @@ -79,6 +79,9 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): kSigning = sign(kService, "aws4_request") return kSigning + # SHA256 of empty string. This is needed when S3 request payload is empty. + SHA256_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + t = datetime.utcnow() amzdate = t.strftime("%Y%m%dT%H%M%SZ") datestamp = t.strftime("%Y%m%d") @@ -138,7 +141,7 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): return { "x-amz-date": amzdate, - "x-amz-content-sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "x-amz-content-sha256": SHA256_EMPTY, "Authorization": authorization_header, } From 8dc364ec264db7a47b04d7aebc77368c229a1105 Mon Sep 17 00:00:00 2001 From: Kevin Balthaser Date: Thu, 21 Mar 2024 09:15:27 -0400 Subject: [PATCH 4/5] - Set default value for `POLICY_BUNDLE_SERVER_AWS_REGION` and remove redundant if statement. - Fix duplicated value in documentation. --- documentation/docs/tutorials/track_an_api_bundle_server.mdx | 1 - packages/opal-common/opal_common/utils.py | 2 -- packages/opal-server/opal_server/config.py | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/documentation/docs/tutorials/track_an_api_bundle_server.mdx b/documentation/docs/tutorials/track_an_api_bundle_server.mdx index 184df814c..ff8f5b89f 100644 --- a/documentation/docs/tutorials/track_an_api_bundle_server.mdx +++ b/documentation/docs/tutorials/track_an_api_bundle_server.mdx @@ -40,7 +40,6 @@ You can configure how the OPAL-server will authenticate itself with the bundle s | POLICY_BUNDLE_SERVER_TYPE | `HTTP` (authenticated with bearer token,or nothing), `AWS-S3`(Authenticated with [AWS REST Auth](https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html) | AWS-S3 | | POLICY_BUNDLE_SERVER_TOKEN_ID | The Secret Token Id (AKA user id, AKA access-key) sent to the API bundle server. | AKIAIOSFODNN7EXAMPLE | | POLICY_BUNDLE_SERVER_TOKEN | The Secret Token (AKA password, AKA secret-key) sent to the API bundle server. | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | -| POLICY_BUNDLE_SERVER_TOKEN | The Secret Token (AKA password, AKA secret-key) sent to the API bundle server. | wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | | POLICY_BUNDLE_SERVER_AWS_REGION| The AWS Region if using `AWS-S3` Defaults to `us-east-1` | us-east-1 | ## Docker compose example diff --git a/packages/opal-common/opal_common/utils.py b/packages/opal-common/opal_common/utils.py index d39fa3091..681202e1b 100644 --- a/packages/opal-common/opal_common/utils.py +++ b/packages/opal-common/opal_common/utils.py @@ -104,8 +104,6 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): + payload_hash ) - if not region: - region = "us-east-1" algorithm = "AWS4-HMAC-SHA256" credential_scope = datestamp + "/" + region + "/" + "s3" + "/" + "aws4_request" diff --git a/packages/opal-server/opal_server/config.py b/packages/opal-server/opal_server/config.py index 20d7be78f..ceff32927 100644 --- a/packages/opal-server/opal_server/config.py +++ b/packages/opal-server/opal_server/config.py @@ -130,7 +130,7 @@ class OpalServerConfig(Confi): ) POLICY_BUNDLE_SERVER_AWS_REGION = confi.str( "POLICY_BUNDLE_SERVER_AWS_REGION", - None, + "us-east-1", description="The AWS region of the S3 bucket", ) POLICY_BUNDLE_TMP_PATH = confi.str( From c175db246862a5da3c9277aa953ac5a26fbb0750 Mon Sep 17 00:00:00 2001 From: Kevin Balthaser Date: Thu, 21 Mar 2024 11:07:23 -0400 Subject: [PATCH 5/5] Fix pre-commit code-format issue --- .../opal-common/opal_common/sources/api_policy_source.py | 4 +++- packages/opal-common/opal_common/utils.py | 5 +++-- packages/opal-server/opal_server/policy/watcher/factory.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/opal-common/opal_common/sources/api_policy_source.py b/packages/opal-common/opal_common/sources/api_policy_source.py index 5ab5d34f5..9b6487907 100644 --- a/packages/opal-common/opal_common/sources/api_policy_source.py +++ b/packages/opal-common/opal_common/sources/api_policy_source.py @@ -138,7 +138,9 @@ def build_auth_headers(self, token=None, path=None): host = split_url.netloc path = split_url.path + "/" + path - return build_aws_rest_auth_headers(self.token_id, token, host, path, self.region) + return build_aws_rest_auth_headers( + self.token_id, token, host, path, self.region + ) else: return {} diff --git a/packages/opal-common/opal_common/utils.py b/packages/opal-common/opal_common/utils.py index 681202e1b..3897c058f 100644 --- a/packages/opal-common/opal_common/utils.py +++ b/packages/opal-common/opal_common/utils.py @@ -56,7 +56,9 @@ def get_authorization_header(token: str) -> Tuple[str, str]: return "Authorization", f"Bearer {token}" -def build_aws_rest_auth_headers(key_id: str, secret_key: str, host: str, path: str, region: str): +def build_aws_rest_auth_headers( + key_id: str, secret_key: str, host: str, path: str, region: str +): """Use the AWS signature algorithm (https://docs.aws.amazon.com/AmazonS3/la test/userguide/RESTAuthentication.html) to generate the hTTP headers. @@ -104,7 +106,6 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): + payload_hash ) - algorithm = "AWS4-HMAC-SHA256" credential_scope = datestamp + "/" + region + "/" + "s3" + "/" + "aws4_request" diff --git a/packages/opal-server/opal_server/policy/watcher/factory.py b/packages/opal-server/opal_server/policy/watcher/factory.py index 97c2de771..10fb1b19c 100644 --- a/packages/opal-server/opal_server/policy/watcher/factory.py +++ b/packages/opal-server/opal_server/policy/watcher/factory.py @@ -128,7 +128,7 @@ def setup_watcher_task( bundle_server_type=policy_bundle_server_type, policy_bundle_path=opal_server_config.POLICY_BUNDLE_TMP_PATH, policy_bundle_git_add_pattern=opal_server_config.POLICY_BUNDLE_GIT_ADD_PATTERN, - region=policy_bundle_aws_region + region=policy_bundle_aws_region, ) else: raise ValueError("Unknown value for OPAL_POLICY_SOURCE_TYPE")