From 584d7b9344b71f25c2b0c8bae953fd2005310044 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Wed, 7 Feb 2024 17:38:29 +0000 Subject: [PATCH 1/2] Fix issue with host header and canonicalURI There were actually two issues that broke authenticated buckets. The first was that we weren't properly setting the host header. The second was that we weren't generating a valid canonical URI that gets signed and sent as a part of the Authorization header. The solution to both of these was to parse the hostUrl more carefully, and in only one spot. --- src/S3Commands.cc | 12 ++++++----- src/S3Commands.hh | 55 +++++++++++++++++++---------------------------- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/S3Commands.cc b/src/S3Commands.cc index 82d17c1..84a1afe 100644 --- a/src/S3Commands.cc +++ b/src/S3Commands.cc @@ -35,6 +35,8 @@ std::string AmazonRequest::canonicalizeQueryString() { } bool AmazonRequest::parseURL( const std::string & url, + const std::string & bucket, + const std::string & object, std::string & host, std::string & path ) { auto i = url.find( "://" ); @@ -43,13 +45,13 @@ bool AmazonRequest::parseURL( const std::string & url, auto j = url.find( "/", i + 3 ); if( j == std::string::npos ) { - host = substring( url, i + 3 ); - path = "/"; + host = bucket + "." + substring( url, i + 3 ); + path = "/" + object; return true; } - host = substring( url, i + 3, j ); - path = substring( url, j ); + host = bucket + "." + substring( url, i + 3, j ); + path = substring( url, j ) + object; return true; } @@ -164,7 +166,7 @@ bool AmazonRequest::createV4Signature( const std::string & payload, std::string payloadHash; convertMessageDigestToLowercaseHex( messageDigest, mdLength, payloadHash ); if( sendContentSHA ) { - headers[ "x-amz-content-sha256" ] = payloadHash; + headers[ "X-Amz-Content-Sha256" ] = payloadHash; } // The canonical list of headers is a sorted list of lowercase header diff --git a/src/S3Commands.hh b/src/S3Commands.hh index 363eada..0aa84b4 100644 --- a/src/S3Commands.hh +++ b/src/S3Commands.hh @@ -9,21 +9,33 @@ public: const std::string & s, const std::string & akf, const std::string & skf, + const std::string & b, + const std::string & o, int sv = 4 ) : HTTPRequest( s ), accessKeyFile(akf), secretKeyFile(skf), - signatureVersion(sv) + signatureVersion(sv), + bucket(b), + object(o) { requiresSignature = true; - if (! parseURL(hostUrl, host, canonicalURI)) { + // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header). + // For example, if the incoming hostUrl (which we get from config) is "https://my-url.com:443", the bucket is "my-bucket", and + // the object is "my-object", then the host will be "my-bucket.my-url.com:443" and the canonicalURI will be "/my-object". + if (! parseURL(hostUrl, b, o, host, canonicalURI)) { errorCode = "E_INVALID_SERVICE_URL"; errorMessage = "Failed to parse host and canonicalURI from service URL."; } if( canonicalURI.empty() ) { canonicalURI = "/"; } + // Now that we have the host and canonicalURI, we can build the actual url we perform the curl against. + // Using the previous example, we'd get a new hostUrl of "https://my-bucket.my-url.com:443/my-object". + hostUrl = protocol + "://" + + host + canonicalURI; + // If we can, set the region based on the host. size_t secondDot = host.find( ".", 2 + 1 ); if( host.find( "s3." ) == 0 ) { @@ -38,6 +50,8 @@ public: return &secretKeyFile; } bool parseURL( const std::string & url, + const std::string & bucket, + const std::string & object, std::string & host, std::string & path ); @@ -57,6 +71,9 @@ protected: std::string host; std::string canonicalURI; + std::string bucket; + std::string object; + std::string region; std::string service; @@ -76,21 +93,13 @@ public: const std::string & b, const std::string & o ) : - AmazonRequest(s, akf, skf), - bucket(b), - object(o) - { - hostUrl = protocol + "://" + bucket + "." + - host + canonicalURI + object; - } + AmazonRequest(s, akf, skf, b, o){} virtual ~AmazonS3Upload(); virtual bool SendRequest( const std::string & payload, off_t offset, size_t size ); protected: - std::string bucket; - std::string object; std::string path; }; @@ -104,21 +113,11 @@ public: const std::string & b, const std::string & o ) : - AmazonRequest(s, akf, skf), - bucket(b), - object(o) - { - hostUrl = protocol + "://" + bucket + "." + - host + canonicalURI + object; - } + AmazonRequest(s, akf, skf, b, o){} virtual ~AmazonS3Download(); virtual bool SendRequest( off_t offset, size_t size ); - -protected: - std::string bucket; - std::string object; }; class AmazonS3Head : public AmazonRequest { @@ -131,21 +130,11 @@ public: const std::string & b, const std::string & o ) : - AmazonRequest(s, akf, skf), - bucket(b), - object(o) - { - hostUrl = protocol + "://" + bucket + "." + - host + canonicalURI + object; - } + AmazonRequest(s, akf, skf, b, o){} virtual ~AmazonS3Head(); virtual bool SendRequest(); - -protected: - std::string bucket; - std::string object; }; #endif /* S3_COMMANDS_H */ From e4676a2c67b5a39c9217825207ed06404550bcee Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 8 Feb 2024 15:55:51 +0000 Subject: [PATCH 2/2] Add function comment --- src/S3Commands.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/S3Commands.cc b/src/S3Commands.cc index 84a1afe..b1a9516 100644 --- a/src/S3Commands.cc +++ b/src/S3Commands.cc @@ -34,6 +34,8 @@ std::string AmazonRequest::canonicalizeQueryString() { return AWSv4Impl::canonicalizeQueryString( query_parameters ); } +// Takes in the configured `s3.service_url` and uses the bucket/object requested to generate +// the virtual host URL, as well as the canonical URI (which is the path to the object). bool AmazonRequest::parseURL( const std::string & url, const std::string & bucket, const std::string & object,