From cc46c0a764ba29e86dc59893c2bb6e23c3d859ef Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Mon, 15 Apr 2024 17:42:27 +0000 Subject: [PATCH 1/3] Initial repo-wide clang format lint --- .clang-format | 4 + src/AWSCredential.hh | 24 +- src/AWSv4-impl.cc | 239 ++++++++------- src/AWSv4-impl.hh | 30 +- src/HTTPCommands.cc | 643 +++++++++++++++++++++------------------- src/HTTPCommands.hh | 198 ++++++------- src/HTTPDirectory.hh | 51 ++-- src/HTTPFile.cc | 425 ++++++++++++-------------- src/HTTPFile.hh | 194 +++++------- src/HTTPFileSystem.cc | 274 +++++++++-------- src/HTTPFileSystem.hh | 145 +++++---- src/S3AccessInfo.cc | 22 +- src/S3AccessInfo.hh | 43 ++- src/S3Commands.cc | 403 +++++++++++++------------ src/S3Commands.hh | 242 +++++++-------- src/S3Directory.hh | 47 ++- src/S3File.cc | 460 +++++++++++++--------------- src/S3File.hh | 195 +++++------- src/S3FileSystem.cc | 341 +++++++++++---------- src/S3FileSystem.hh | 175 ++++++----- src/logging.cc | 99 ++++--- src/logging.hh | 21 +- src/shortfile.cc | 119 ++++---- src/stl_string_utils.cc | 129 ++++---- src/stl_string_utils.hh | 23 +- 25 files changed, 2226 insertions(+), 2320 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..8675285 --- /dev/null +++ b/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +UseTab: Always +TabWidth: 4 \ No newline at end of file diff --git a/src/AWSCredential.hh b/src/AWSCredential.hh index cbdd857..7cd0d6b 100644 --- a/src/AWSCredential.hh +++ b/src/AWSCredential.hh @@ -19,19 +19,15 @@ class AWSCredential { - public: - AWSCredential( - const std::string &accessKeyID, - const std::string &secretAccessKey, - const std::string &securityToken - ) : - m_access_key(accessKeyID), - m_secret_key(secretAccessKey), - m_security_token(securityToken) - {} + public: + AWSCredential(const std::string &accessKeyID, + const std::string &secretAccessKey, + const std::string &securityToken) + : m_access_key(accessKeyID), m_secret_key(secretAccessKey), + m_security_token(securityToken) {} - private: - const std::string m_access_key; - const std::string m_secret_key; - const std::string m_security_token; + private: + const std::string m_access_key; + const std::string m_secret_key; + const std::string m_security_token; }; diff --git a/src/AWSv4-impl.cc b/src/AWSv4-impl.cc index 2050330..26f306b 100644 --- a/src/AWSv4-impl.cc +++ b/src/AWSv4-impl.cc @@ -19,17 +19,18 @@ /** * Utilities for generating pre-signed URLs. * - * These were originally authored by the HTCondor team under the Apache 2.0 license which - * can also be found in the LICENSE file at the top-level directory of this project. No - * copyright statement was present in the original file. + * These were originally authored by the HTCondor team under the Apache 2.0 + * license which can also be found in the LICENSE file at the top-level + * directory of this project. No copyright statement was present in the + * original file. */ #include #include +#include "AWSv4-impl.hh" #include #include -#include "AWSv4-impl.hh" namespace AWSv4Impl { @@ -37,68 +38,65 @@ namespace AWSv4Impl { // This function should not be called for anything in query_parameters, // except for by AmazonQuery::SendRequest(). // -std::string -amazonURLEncode( const std::string & input ) { - /* - * See http://docs.amazonwebservices.com/AWSEC2/2010-11-15/DeveloperGuide/using-query-api.html - * - */ - std::string output; - for( unsigned i = 0; i < input.length(); ++i ) { - // "Do not URL encode ... A-Z, a-z, 0-9, hyphen ( - ), - // underscore ( _ ), period ( . ), and tilde ( ~ ). Percent - // encode all other characters with %XY, where X and Y are hex - // characters 0-9 and uppercase A-F. Percent encode extended - // UTF-8 characters in the form %XY%ZA..." - if( ('A' <= input[i] && input[i] <= 'Z') - || ('a' <= input[i] && input[i] <= 'z') - || ('0' <= input[i] && input[i] <= '9') - || input[i] == '-' - || input[i] == '_' - || input[i] == '.' - || input[i] == '~' ) { - char uglyHack[] = "X"; - uglyHack[0] = input[i]; - output.append( uglyHack ); - } else { - char percentEncode[4]; - snprintf(percentEncode, 4, "%%%.2hhX", input[i]); - output.append(percentEncode); - } - } - - return output; -} +std::string amazonURLEncode(const std::string &input) { + /* + * See + * http://docs.amazonwebservices.com/AWSEC2/2010-11-15/DeveloperGuide/using-query-api.html + * + */ + std::string output; + for (unsigned i = 0; i < input.length(); ++i) { + // "Do not URL encode ... A-Z, a-z, 0-9, hyphen ( - ), + // underscore ( _ ), period ( . ), and tilde ( ~ ). Percent + // encode all other characters with %XY, where X and Y are hex + // characters 0-9 and uppercase A-F. Percent encode extended + // UTF-8 characters in the form %XY%ZA..." + if (('A' <= input[i] && input[i] <= 'Z') || + ('a' <= input[i] && input[i] <= 'z') || + ('0' <= input[i] && input[i] <= '9') || input[i] == '-' || + input[i] == '_' || input[i] == '.' || input[i] == '~') { + char uglyHack[] = "X"; + uglyHack[0] = input[i]; + output.append(uglyHack); + } else { + char percentEncode[4]; + snprintf(percentEncode, 4, "%%%.2hhX", input[i]); + output.append(percentEncode); + } + } + return output; +} -std::string -pathEncode( const std::string & original ) { +std::string pathEncode(const std::string &original) { std::string segment; std::string encoded; - const char * o = original.c_str(); + const char *o = original.c_str(); size_t next = 0; size_t offset = 0; - size_t length = strlen( o ); - while( offset < length ) { - next = strcspn( o + offset, "/" ); - if( next == 0 ) { encoded += "/"; offset += 1; continue; } - - segment = std::string( o + offset, next ); - encoded += amazonURLEncode( segment ); + size_t length = strlen(o); + while (offset < length) { + next = strcspn(o + offset, "/"); + if (next == 0) { + encoded += "/"; + offset += 1; + continue; + } + + segment = std::string(o + offset, next); + encoded += amazonURLEncode(segment); offset += next; } return encoded; } - -void -convertMessageDigestToLowercaseHex( - const unsigned char * messageDigest, - unsigned int mdLength, std::string & hexEncoded ) { - char * buffer = (char *)malloc( (mdLength * 2) + 1 ); - char * ptr = buffer; +void convertMessageDigestToLowercaseHex(const unsigned char *messageDigest, + unsigned int mdLength, + std::string &hexEncoded) { + char *buffer = (char *)malloc((mdLength * 2) + 1); + char *ptr = buffer; for (unsigned int i = 0; i < mdLength; ++i, ptr += 2) { snprintf(ptr, 3, "%02x", messageDigest[i]); } @@ -106,94 +104,105 @@ convertMessageDigestToLowercaseHex( free(buffer); } -bool -doSha256( const std::string & payload, - unsigned char * messageDigest, - unsigned int * mdLength ) { - EVP_MD_CTX * mdctx = EVP_MD_CTX_create(); - if( mdctx == NULL ) { return false; } +bool doSha256(const std::string &payload, unsigned char *messageDigest, + unsigned int *mdLength) { + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + if (mdctx == NULL) { + return false; + } - if(! EVP_DigestInit_ex( mdctx, EVP_sha256(), NULL )) { - EVP_MD_CTX_destroy( mdctx ); + if (!EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL)) { + EVP_MD_CTX_destroy(mdctx); return false; } - if(! EVP_DigestUpdate( mdctx, payload.c_str(), payload.length() )) { - EVP_MD_CTX_destroy( mdctx ); + if (!EVP_DigestUpdate(mdctx, payload.c_str(), payload.length())) { + EVP_MD_CTX_destroy(mdctx); return false; } - if(! EVP_DigestFinal_ex( mdctx, messageDigest, mdLength )) { - EVP_MD_CTX_destroy( mdctx ); + if (!EVP_DigestFinal_ex(mdctx, messageDigest, mdLength)) { + EVP_MD_CTX_destroy(mdctx); return false; } - EVP_MD_CTX_destroy( mdctx ); + EVP_MD_CTX_destroy(mdctx); return true; } -bool -createSignature( const std::string & secretAccessKey, - const std::string & date, const std::string & region, - const std::string & service, const std::string & stringToSign, - std::string & signature ) { - unsigned int mdLength = 0; - unsigned char messageDigest[EVP_MAX_MD_SIZE]; +bool createSignature(const std::string &secretAccessKey, + const std::string &date, const std::string ®ion, + const std::string &service, + const std::string &stringToSign, std::string &signature) { + unsigned int mdLength = 0; + unsigned char messageDigest[EVP_MAX_MD_SIZE]; std::string saKey = "AWS4" + secretAccessKey; - const unsigned char * hmac = HMAC( EVP_sha256(), saKey.c_str(), saKey.length(), - (const unsigned char *)date.c_str(), date.length(), - messageDigest, & mdLength ); - if( hmac == NULL ) { return false; } + const unsigned char *hmac = + HMAC(EVP_sha256(), saKey.c_str(), saKey.length(), + (const unsigned char *)date.c_str(), date.length(), messageDigest, + &mdLength); + if (hmac == NULL) { + return false; + } unsigned int md2Length = 0; unsigned char messageDigest2[EVP_MAX_MD_SIZE]; - hmac = HMAC( EVP_sha256(), messageDigest, mdLength, - (const unsigned char *)region.c_str(), region.length(), messageDigest2, & md2Length ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest, mdLength, + (const unsigned char *)region.c_str(), region.length(), + messageDigest2, &md2Length); + if (hmac == NULL) { + return false; + } - hmac = HMAC( EVP_sha256(), messageDigest2, md2Length, - (const unsigned char *)service.c_str(), service.length(), messageDigest, & mdLength ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest2, md2Length, + (const unsigned char *)service.c_str(), service.length(), + messageDigest, &mdLength); + if (hmac == NULL) { + return false; + } const char c[] = "aws4_request"; - hmac = HMAC( EVP_sha256(), messageDigest, mdLength, - (const unsigned char *)c, sizeof(c) - 1, messageDigest2, & md2Length ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest, mdLength, (const unsigned char *)c, + sizeof(c) - 1, messageDigest2, &md2Length); + if (hmac == NULL) { + return false; + } - hmac = HMAC( EVP_sha256(), messageDigest2, md2Length, - (const unsigned char *)stringToSign.c_str(), stringToSign.length(), - messageDigest, & mdLength ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest2, md2Length, + (const unsigned char *)stringToSign.c_str(), + stringToSign.length(), messageDigest, &mdLength); + if (hmac == NULL) { + return false; + } - convertMessageDigestToLowercaseHex( messageDigest, mdLength, signature ); + convertMessageDigestToLowercaseHex(messageDigest, mdLength, signature); return true; } -std::string -canonicalizeQueryString( - const std::map< std::string, std::string > & query_parameters ) { - std::string canonicalQueryString; - for( auto i = query_parameters.begin(); i != query_parameters.end(); ++i ) { - // Step 1A: The map sorts the query parameters for us. Strictly - // speaking, we should encode into a different AttributeValueMap - // and then compose the string out of that, in case amazonURLEncode() - // changes the sort order, but we don't specify parameters like that. - - // Step 1B: Encode the parameter names and values. - std::string name = amazonURLEncode( i->first ); - std::string value = amazonURLEncode( i->second ); - - // Step 1C: Separate parameter names from values with '='. - canonicalQueryString += name + '=' + value; - - // Step 1D: Separate name-value pairs with '&'; - canonicalQueryString += '&'; - } - - // We'll always have a superflous trailing ampersand. - canonicalQueryString.erase( canonicalQueryString.end() - 1 ); - return canonicalQueryString; +std::string canonicalizeQueryString( + const std::map &query_parameters) { + std::string canonicalQueryString; + for (auto i = query_parameters.begin(); i != query_parameters.end(); ++i) { + // Step 1A: The map sorts the query parameters for us. Strictly + // speaking, we should encode into a different AttributeValueMap + // and then compose the string out of that, in case amazonURLEncode() + // changes the sort order, but we don't specify parameters like that. + + // Step 1B: Encode the parameter names and values. + std::string name = amazonURLEncode(i->first); + std::string value = amazonURLEncode(i->second); + + // Step 1C: Separate parameter names from values with '='. + canonicalQueryString += name + '=' + value; + + // Step 1D: Separate name-value pairs with '&'; + canonicalQueryString += '&'; + } + + // We'll always have a superflous trailing ampersand. + canonicalQueryString.erase(canonicalQueryString.end() - 1); + return canonicalQueryString; } } /* end namespace AWSv4Impl */ diff --git a/src/AWSv4-impl.hh b/src/AWSv4-impl.hh index f7c0b51..8f17e2f 100644 --- a/src/AWSv4-impl.hh +++ b/src/AWSv4-impl.hh @@ -23,27 +23,23 @@ namespace AWSv4Impl { -std::string -pathEncode( const std::string & original ); +std::string pathEncode(const std::string &original); -std::string -amazonURLEncode( const std::string & input ); +std::string amazonURLEncode(const std::string &input); std::string -canonicalizeQueryString( const std::map< std::string, std::string > & qp ); +canonicalizeQueryString(const std::map &qp); -void -convertMessageDigestToLowercaseHex( const unsigned char * messageDigest, - unsigned int mdLength, std::string & hexEncoded ); +void convertMessageDigestToLowercaseHex(const unsigned char *messageDigest, + unsigned int mdLength, + std::string &hexEncoded); -bool -doSha256( const std::string & payload, unsigned char * messageDigest, - unsigned int * mdLength ); +bool doSha256(const std::string &payload, unsigned char *messageDigest, + unsigned int *mdLength); -bool -createSignature( const std::string & secretAccessKey, - const std::string & date, const std::string & region, - const std::string & service, const std::string & stringToSign, - std::string & signature ); +bool createSignature(const std::string &secretAccessKey, + const std::string &date, const std::string ®ion, + const std::string &service, + const std::string &stringToSign, std::string &signature); -} +} // namespace AWSv4Impl diff --git a/src/HTTPCommands.cc b/src/HTTPCommands.cc index 4145760..29acc06 100644 --- a/src/HTTPCommands.cc +++ b/src/HTTPCommands.cc @@ -25,14 +25,14 @@ #include #include -#include -#include #include +#include +#include #include "HTTPCommands.hh" #include "logging.hh" -#include "stl_string_utils.hh" #include "shortfile.hh" +#include "stl_string_utils.hh" using namespace XrdHTTPServer; @@ -47,423 +47,454 @@ using namespace XrdHTTPServer; // We also make extensive use of this function in the XML parsing code, // for pretty much exactly the same reason. // -size_t appendToString( const void * ptr, size_t size, size_t nmemb, void * str ) { - if( size == 0 || nmemb == 0 ) { return 0; } +size_t appendToString(const void *ptr, size_t size, size_t nmemb, void *str) { + if (size == 0 || nmemb == 0) { + return 0; + } - std::string source( (const char *)ptr, size * nmemb ); - std::string * ssptr = (std::string *)str; - ssptr->append( source ); + std::string source((const char *)ptr, size * nmemb); + std::string *ssptr = (std::string *)str; + ssptr->append(source); - return (size * nmemb); + return (size * nmemb); } -HTTPRequest::~HTTPRequest() { } - -#define SET_CURL_SECURITY_OPTION( A, B, C ) { \ - CURLcode rv##B = curl_easy_setopt( A, B, C ); \ - if( rv##B != CURLE_OK ) { \ - this->errorCode = "E_CURL_LIB"; \ - this->errorMessage = "curl_easy_setopt( " #B " ) failed."; \ - /* dprintf( D_ALWAYS, "curl_easy_setopt( %s ) failed (%d): '%s', failing.\n", \ - #B, rv##B, curl_easy_strerror( rv##B ) ); */ \ - return false; \ - } \ -} +HTTPRequest::~HTTPRequest() {} + +#define SET_CURL_SECURITY_OPTION(A, B, C) \ + { \ + CURLcode rv##B = curl_easy_setopt(A, B, C); \ + if (rv##B != CURLE_OK) { \ + this->errorCode = "E_CURL_LIB"; \ + this->errorMessage = "curl_easy_setopt( " #B " ) failed."; \ + /* dprintf( D_ALWAYS, "curl_easy_setopt( %s ) failed (%d): '%s', \ + failing.\n", #B, rv##B, curl_easy_strerror( rv##B ) ); */ \ + return false; \ + } \ + } -bool HTTPRequest::parseProtocol( - const std::string & url, - std::string & protocol ) { - - auto i = url.find( "://" ); - if( i == std::string::npos ) { return false; } - protocol = substring( url, 0, i ); - - // This func used to parse the entire URL according - // to the Amazon canonicalURI specs, but that functionality - // has since been moved to the Amazon subclass. Now it just - // grabs the protocol. Leaving the old stuff commented for - // now, just in case... - - // auto j = url.find( "/", i + 3 ); - // if( j == std::string::npos ) { - // host = substring( url, i + 3 ); - // path = "/"; - // return true; - // } - - // host = substring( url, i + 3, j ); - // path = substring( url, j ); - return true; +bool HTTPRequest::parseProtocol(const std::string &url, std::string &protocol) { + + auto i = url.find("://"); + if (i == std::string::npos) { + return false; + } + protocol = substring(url, 0, i); + + // This func used to parse the entire URL according + // to the Amazon canonicalURI specs, but that functionality + // has since been moved to the Amazon subclass. Now it just + // grabs the protocol. Leaving the old stuff commented for + // now, just in case... + + // auto j = url.find( "/", i + 3 ); + // if( j == std::string::npos ) { + // host = substring( url, i + 3 ); + // path = "/"; + // return true; + // } + + // host = substring( url, i + 3, j ); + // path = substring( url, j ); + return true; } -bool HTTPRequest::SendHTTPRequest( const std::string & payload ) { - if( (protocol != "http") && (protocol != "https") ) { - this->errorCode = "E_INVALID_SERVICE_URL"; - this->errorMessage = "Service URL not of a known protocol (http[s])."; - m_log.Log(LogMask::Warning, "HTTPRequest::SendHTTPRequest", "Host URL '", hostUrl.c_str(), "' not of a known protocol (http[s])."); - return false; - } - - headers[ "Content-Type" ] = "binary/octet-stream"; - std::string contentLength; formatstr( contentLength, "%zu", payload.size() ); - headers[ "Content-Length" ] = contentLength; - // Another undocumented CURL feature: transfer-encoding is "chunked" - // by default for "PUT", which we really don't want. - headers[ "Transfer-Encoding" ] = ""; - - return sendPreparedRequest( protocol, hostUrl, payload ); +bool HTTPRequest::SendHTTPRequest(const std::string &payload) { + if ((protocol != "http") && (protocol != "https")) { + this->errorCode = "E_INVALID_SERVICE_URL"; + this->errorMessage = "Service URL not of a known protocol (http[s])."; + m_log.Log(LogMask::Warning, "HTTPRequest::SendHTTPRequest", + "Host URL '", hostUrl.c_str(), + "' not of a known protocol (http[s])."); + return false; + } + + headers["Content-Type"] = "binary/octet-stream"; + std::string contentLength; + formatstr(contentLength, "%zu", payload.size()); + headers["Content-Length"] = contentLength; + // Another undocumented CURL feature: transfer-encoding is "chunked" + // by default for "PUT", which we really don't want. + headers["Transfer-Encoding"] = ""; + + return sendPreparedRequest(protocol, hostUrl, payload); } -int -debug_callback( CURL *, curl_infotype ci, char * data, size_t size, void * ) { - switch( ci ) { - default: - break; +int debug_callback(CURL *, curl_infotype ci, char *data, size_t size, void *) { + switch (ci) { + default: + break; - case CURLINFO_TEXT: - break; + case CURLINFO_TEXT: + break; - case CURLINFO_HEADER_IN: - break; + case CURLINFO_HEADER_IN: + break; - case CURLINFO_HEADER_OUT: - break; + case CURLINFO_HEADER_OUT: + break; - case CURLINFO_DATA_IN: - break; + case CURLINFO_DATA_IN: + break; - case CURLINFO_DATA_OUT: - break; + case CURLINFO_DATA_OUT: + break; - case CURLINFO_SSL_DATA_IN: - break; + case CURLINFO_SSL_DATA_IN: + break; - case CURLINFO_SSL_DATA_OUT: - break; + case CURLINFO_SSL_DATA_OUT: + break; } return 0; } -size_t -read_callback( char * buffer, size_t size, size_t n, void * v ) { +size_t read_callback(char *buffer, size_t size, size_t n, void *v) { // This can be static because only one curl_easy_perform() can be // running at a time. static size_t sentSoFar = 0; - std::string * payload = (std::string *)v; + std::string *payload = (std::string *)v; - if( sentSoFar == payload->size() ) { + if (sentSoFar == payload->size()) { sentSoFar = 0; return 0; } size_t request = size * n; - if( request > payload->size() ) { request = payload->size(); } + if (request > payload->size()) { + request = payload->size(); + } - if( sentSoFar + request > payload->size() ) { + if (sentSoFar + request > payload->size()) { request = payload->size() - sentSoFar; } - memcpy( buffer, payload->data() + sentSoFar, request ); + memcpy(buffer, payload->data() + sentSoFar, request); sentSoFar += request; return request; } -bool HTTPRequest::sendPreparedRequest( - const std::string & protocol, - const std::string & uri, - const std::string & payload ) { - - CURLcode rv = curl_global_init( CURL_GLOBAL_ALL ); - if( rv != 0 ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_global_init() failed."; - return false; - } - - std::unique_ptr curl(curl_easy_init(), &curl_easy_cleanup); - - if( curl.get() == NULL ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_init() failed."; - return false; - } - - char errorBuffer[CURL_ERROR_SIZE]; - rv = curl_easy_setopt( curl.get(), CURLOPT_ERRORBUFFER, errorBuffer ); - if( rv != CURLE_OK ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_ERRORBUFFER ) failed."; - return false; - } - - rv = curl_easy_setopt( curl.get(), CURLOPT_URL, uri.c_str() ); - if( rv != CURLE_OK ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_URL ) failed."; - return false; - } - - if( httpVerb == "HEAD" ) { - rv = curl_easy_setopt( curl.get(), CURLOPT_NOBODY, 1 ); - if( rv != CURLE_OK ) { +bool HTTPRequest::sendPreparedRequest(const std::string &protocol, + const std::string &uri, + const std::string &payload) { + + CURLcode rv = curl_global_init(CURL_GLOBAL_ALL); + if (rv != 0) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_global_init() failed."; + return false; + } + + std::unique_ptr curl( + curl_easy_init(), &curl_easy_cleanup); + + if (curl.get() == NULL) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_init() failed."; + return false; + } + + char errorBuffer[CURL_ERROR_SIZE]; + rv = curl_easy_setopt(curl.get(), CURLOPT_ERRORBUFFER, errorBuffer); + if (rv != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_setopt( CURLOPT_ERRORBUFFER ) failed."; + return false; + } + + rv = curl_easy_setopt(curl.get(), CURLOPT_URL, uri.c_str()); + if (rv != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_setopt( CURLOPT_URL ) failed."; + return false; + } + + if (httpVerb == "HEAD") { + rv = curl_easy_setopt(curl.get(), CURLOPT_NOBODY, 1); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; this->errorMessage = "curl_easy_setopt( CURLOPT_HEAD ) failed."; return false; } - } + } - if( httpVerb == "POST" ) { - rv = curl_easy_setopt( curl.get(), CURLOPT_POST, 1 ); - if( rv != CURLE_OK ) { + if (httpVerb == "POST") { + rv = curl_easy_setopt(curl.get(), CURLOPT_POST, 1); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; this->errorMessage = "curl_easy_setopt( CURLOPT_POST ) failed."; return false; } - rv = curl_easy_setopt( curl.get(), CURLOPT_POSTFIELDS, payload.c_str() ); - if( rv != CURLE_OK ) { + rv = curl_easy_setopt(curl.get(), CURLOPT_POSTFIELDS, payload.c_str()); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_POSTFIELDS ) failed."; + this->errorMessage = + "curl_easy_setopt( CURLOPT_POSTFIELDS ) failed."; return false; } } - if( httpVerb == "PUT" ) { - rv = curl_easy_setopt( curl.get(), CURLOPT_UPLOAD, 1 ); - if( rv != CURLE_OK ) { + if (httpVerb == "PUT") { + rv = curl_easy_setopt(curl.get(), CURLOPT_UPLOAD, 1); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; this->errorMessage = "curl_easy_setopt( CURLOPT_UPLOAD ) failed."; return false; } - rv = curl_easy_setopt( curl.get(), CURLOPT_READDATA, & payload ); - if( rv != CURLE_OK ) { + rv = curl_easy_setopt(curl.get(), CURLOPT_READDATA, &payload); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; this->errorMessage = "curl_easy_setopt( CURLOPT_READDATA ) failed."; return false; } - rv = curl_easy_setopt( curl.get(), CURLOPT_READFUNCTION, read_callback ); - if( rv != CURLE_OK ) { + rv = curl_easy_setopt(curl.get(), CURLOPT_READFUNCTION, read_callback); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_READFUNCTION ) failed."; + this->errorMessage = + "curl_easy_setopt( CURLOPT_READFUNCTION ) failed."; return false; } } - rv = curl_easy_setopt( curl.get(), CURLOPT_NOPROGRESS, 1 ); - if( rv != CURLE_OK ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_NOPROGRESS ) failed."; - return false; - } - - if ( includeResponseHeader ) { - rv = curl_easy_setopt( curl.get(), CURLOPT_HEADER, 1 ); - if( rv != CURLE_OK ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_HEADER ) failed."; - return false; - } - } - - rv = curl_easy_setopt( curl.get(), CURLOPT_WRITEFUNCTION, & appendToString ); - if( rv != CURLE_OK ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_WRITEFUNCTION ) failed."; - return false; - } - - rv = curl_easy_setopt( curl.get(), CURLOPT_WRITEDATA, & this->resultString ); - if( rv != CURLE_OK ) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_WRITEDATA ) failed."; - return false; - } - - if (curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1) != CURLE_OK) { - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_setopt( CURLOPT_FOLLOWLOCATION ) failed."; - return false; - } - - // - // Set security options. - // - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_SSL_VERIFYPEER, 1 ); - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_SSL_VERIFYHOST, 2 ); - - // NB: Contrary to libcurl's manual, it doesn't strdup() strings passed - // to it, so they MUST remain in scope until after we call - // curl_easy_cleanup(). Otherwise, curl_perform() will fail with - // a completely bogus error, number 60, claiming that there's a - // 'problem with the SSL CA cert'. - std::string CAFile = ""; - std::string CAPath = ""; - - char * x509_ca_dir = getenv( "X509_CERT_DIR" ); - if( x509_ca_dir != NULL ) { - CAPath = x509_ca_dir; - } - - char * x509_ca_file = getenv( "X509_CERT_FILE" ); - if( x509_ca_file != NULL ) { - CAFile = x509_ca_file; - } - - if( CAPath.empty() ) { - char * soap_ssl_ca_dir = getenv( "GAHP_SSL_CADIR" ); - if( soap_ssl_ca_dir != NULL ) { - CAPath = soap_ssl_ca_dir; - } - } - - if( CAFile.empty() ) { - char * soap_ssl_ca_file = getenv( "GAHP_SSL_CAFILE" ); - if( soap_ssl_ca_file != NULL ) { - CAFile = soap_ssl_ca_file; - } - } - - if( ! CAPath.empty() ) { - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_CAPATH, CAPath.c_str() ); - } - - if( ! CAFile.empty() ) { - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_CAINFO, CAFile.c_str() ); - } - - if( setenv( "OPENSSL_ALLOW_PROXY", "1", 0 ) != 0 ) { - } - - // - // Configure for x.509 operation. - // - - if( protocol == "x509" && requiresSignature ) { - const std::string* accessKeyFilePtr = this->getAccessKey(); - const std::string* secretKeyFilePtr = this->getSecretKey(); - if (accessKeyFilePtr && secretKeyFilePtr) { - - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_SSLKEYTYPE, "PEM" ); - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_SSLKEY, *secretKeyFilePtr->c_str() ); - - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_SSLCERTTYPE, "PEM" ); - SET_CURL_SECURITY_OPTION( curl.get(), CURLOPT_SSLCERT, *accessKeyFilePtr->c_str() ); - } - } + rv = curl_easy_setopt(curl.get(), CURLOPT_NOPROGRESS, 1); + if (rv != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_setopt( CURLOPT_NOPROGRESS ) failed."; + return false; + } + + if (includeResponseHeader) { + rv = curl_easy_setopt(curl.get(), CURLOPT_HEADER, 1); + if (rv != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_setopt( CURLOPT_HEADER ) failed."; + return false; + } + } + + rv = curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, &appendToString); + if (rv != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = + "curl_easy_setopt( CURLOPT_WRITEFUNCTION ) failed."; + return false; + } + + rv = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &this->resultString); + if (rv != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_setopt( CURLOPT_WRITEDATA ) failed."; + return false; + } + + if (curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1) != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = + "curl_easy_setopt( CURLOPT_FOLLOWLOCATION ) failed."; + return false; + } + + // + // Set security options. + // + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_SSL_VERIFYPEER, 1); + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_SSL_VERIFYHOST, 2); + + // NB: Contrary to libcurl's manual, it doesn't strdup() strings passed + // to it, so they MUST remain in scope until after we call + // curl_easy_cleanup(). Otherwise, curl_perform() will fail with + // a completely bogus error, number 60, claiming that there's a + // 'problem with the SSL CA cert'. + std::string CAFile = ""; + std::string CAPath = ""; + + char *x509_ca_dir = getenv("X509_CERT_DIR"); + if (x509_ca_dir != NULL) { + CAPath = x509_ca_dir; + } + + char *x509_ca_file = getenv("X509_CERT_FILE"); + if (x509_ca_file != NULL) { + CAFile = x509_ca_file; + } + + if (CAPath.empty()) { + char *soap_ssl_ca_dir = getenv("GAHP_SSL_CADIR"); + if (soap_ssl_ca_dir != NULL) { + CAPath = soap_ssl_ca_dir; + } + } + + if (CAFile.empty()) { + char *soap_ssl_ca_file = getenv("GAHP_SSL_CAFILE"); + if (soap_ssl_ca_file != NULL) { + CAFile = soap_ssl_ca_file; + } + } + + if (!CAPath.empty()) { + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_CAPATH, CAPath.c_str()); + } + + if (!CAFile.empty()) { + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_CAINFO, CAFile.c_str()); + } + + if (setenv("OPENSSL_ALLOW_PROXY", "1", 0) != 0) { + } + + // + // Configure for x.509 operation. + // + + if (protocol == "x509" && requiresSignature) { + const std::string *accessKeyFilePtr = this->getAccessKey(); + const std::string *secretKeyFilePtr = this->getSecretKey(); + if (accessKeyFilePtr && secretKeyFilePtr) { + + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_SSLKEYTYPE, "PEM"); + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_SSLKEY, + *secretKeyFilePtr->c_str()); + + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_SSLCERTTYPE, "PEM"); + SET_CURL_SECURITY_OPTION(curl.get(), CURLOPT_SSLCERT, + *accessKeyFilePtr->c_str()); + } + } std::string headerPair; - struct curl_slist * header_slist = NULL; - for( auto i = headers.begin(); i != headers.end(); ++i ) { - formatstr( headerPair, "%s: %s", i->first.c_str(), i->second.c_str() ); - header_slist = curl_slist_append( header_slist, headerPair.c_str() ); - if( header_slist == NULL ) { + struct curl_slist *header_slist = NULL; + for (auto i = headers.begin(); i != headers.end(); ++i) { + formatstr(headerPair, "%s: %s", i->first.c_str(), i->second.c_str()); + header_slist = curl_slist_append(header_slist, headerPair.c_str()); + if (header_slist == NULL) { this->errorCode = "E_CURL_LIB"; this->errorMessage = "curl_slist_append() failed."; return false; } } - rv = curl_easy_setopt( curl.get(), CURLOPT_HTTPHEADER, header_slist ); - if( rv != CURLE_OK ) { + rv = curl_easy_setopt(curl.get(), CURLOPT_HTTPHEADER, header_slist); + if (rv != CURLE_OK) { this->errorCode = "E_CURL_LIB"; this->errorMessage = "curl_easy_setopt( CURLOPT_HTTPHEADER ) failed."; - if( header_slist ) { curl_slist_free_all( header_slist ); } + if (header_slist) { + curl_slist_free_all(header_slist); + } return false; } retry: - rv = curl_easy_perform( curl.get() ); - - if( rv != 0 ) { - - this->errorCode = "E_CURL_IO"; - std::ostringstream error; - error << "curl_easy_perform() failed (" << rv << "): '" << curl_easy_strerror( rv ) << "'."; - this->errorMessage = error.str(); - if( header_slist ) { curl_slist_free_all( header_slist ); } - - return false; - } - - responseCode = 0; - rv = curl_easy_getinfo( curl.get(), CURLINFO_RESPONSE_CODE, & responseCode ); - if( rv != CURLE_OK ) { - // So we contacted the server but it returned such gibberish that - // CURL couldn't identify the response code. Let's assume that's - // bad news. Since we're already terminally failing the request, - // don't bother to check if this was our last chance at retrying. - - this->errorCode = "E_CURL_LIB"; - this->errorMessage = "curl_easy_getinfo() failed."; - if( header_slist ) { curl_slist_free_all( header_slist ); } - - return false; - } - - if( responseCode == 503 && (resultString.find( "RequestLimitExceeded" ) != std::string::npos) ) { - resultString.clear(); - goto retry; - } - - if( header_slist ) { curl_slist_free_all( header_slist ); } - - if( responseCode != this->expectedResponseCode ) { - formatstr( this->errorCode, "E_HTTP_RESPONSE_NOT_EXPECTED (response %lu != expected %lu)", responseCode, this->expectedResponseCode ); - this->errorMessage = resultString; - if( this->errorMessage.empty() ) { - formatstr( this->errorMessage, "HTTP response was %lu, not %lu, and no body was returned.", responseCode, this->expectedResponseCode ); - } - return false; - } - - return true; + rv = curl_easy_perform(curl.get()); + + if (rv != 0) { + + this->errorCode = "E_CURL_IO"; + std::ostringstream error; + error << "curl_easy_perform() failed (" << rv << "): '" + << curl_easy_strerror(rv) << "'."; + this->errorMessage = error.str(); + if (header_slist) { + curl_slist_free_all(header_slist); + } + + return false; + } + + responseCode = 0; + rv = curl_easy_getinfo(curl.get(), CURLINFO_RESPONSE_CODE, &responseCode); + if (rv != CURLE_OK) { + // So we contacted the server but it returned such gibberish that + // CURL couldn't identify the response code. Let's assume that's + // bad news. Since we're already terminally failing the request, + // don't bother to check if this was our last chance at retrying. + + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_getinfo() failed."; + if (header_slist) { + curl_slist_free_all(header_slist); + } + + return false; + } + + if (responseCode == 503 && + (resultString.find("RequestLimitExceeded") != + std::string::npos)) { + resultString.clear(); + goto retry; + } + + if (header_slist) { + curl_slist_free_all(header_slist); + } + + if (responseCode != this->expectedResponseCode) { + formatstr(this->errorCode, + "E_HTTP_RESPONSE_NOT_EXPECTED (response %lu != expected %lu)", + responseCode, this->expectedResponseCode); + this->errorMessage = resultString; + if (this->errorMessage.empty()) { + formatstr( + this->errorMessage, + "HTTP response was %lu, not %lu, and no body was returned.", + responseCode, this->expectedResponseCode); + } + return false; + } + + return true; } // --------------------------------------------------------------------------- -HTTPUpload::~HTTPUpload() { } +HTTPUpload::~HTTPUpload() {} -bool HTTPUpload::SendRequest( const std::string & payload, off_t offset, size_t size ) { - if( offset != 0 || size != 0 ) { +bool HTTPUpload::SendRequest(const std::string &payload, off_t offset, + size_t size) { + if (offset != 0 || size != 0) { std::string range; - formatstr( range, "bytes=%lld-%lld", static_cast(offset), static_cast(offset + size - 1)); + formatstr(range, "bytes=%lld-%lld", static_cast(offset), + static_cast(offset + size - 1)); headers["Range"] = range.c_str(); } httpVerb = "PUT"; - return SendHTTPRequest( payload ); + return SendHTTPRequest(payload); } // --------------------------------------------------------------------------- -HTTPDownload::~HTTPDownload() { } +HTTPDownload::~HTTPDownload() {} -bool HTTPDownload::SendRequest( off_t offset, size_t size ) { - if( offset != 0 || size != 0 ) { +bool HTTPDownload::SendRequest(off_t offset, size_t size) { + if (offset != 0 || size != 0) { std::string range; - formatstr( range, "bytes=%lld-%lld", static_cast(offset), static_cast(offset + size - 1)); + formatstr(range, "bytes=%lld-%lld", static_cast(offset), + static_cast(offset + size - 1)); headers["Range"] = range.c_str(); this->expectedResponseCode = 206; } httpVerb = "GET"; std::string noPayloadAllowed; - return SendHTTPRequest( noPayloadAllowed ); + return SendHTTPRequest(noPayloadAllowed); } // --------------------------------------------------------------------------- -HTTPHead::~HTTPHead() { } +HTTPHead::~HTTPHead() {} bool HTTPHead::SendRequest() { httpVerb = "HEAD"; includeResponseHeader = true; std::string noPayloadAllowed; - return SendHTTPRequest( noPayloadAllowed ); + return SendHTTPRequest(noPayloadAllowed); } // --------------------------------------------------------------------------- diff --git a/src/HTTPCommands.hh b/src/HTTPCommands.hh index f12b869..614caae 100644 --- a/src/HTTPCommands.hh +++ b/src/HTTPCommands.hh @@ -24,123 +24,101 @@ class XrdSysError; class HTTPRequest { -public: - HTTPRequest( - const std::string & hostUrl, - XrdSysError &log - ) : - hostUrl(hostUrl), - requiresSignature(false), - responseCode(0), - includeResponseHeader(false), - httpVerb("POST"), - m_log(log) - { - // Parse the URL and populate - // What to do if the function returns false? - // TODO: Figure out best way to deal with this - if (! parseProtocol(hostUrl, protocol)) { - errorCode = "E_INVALID_HOST_URL"; - errorMessage = "Failed to parse protocol from host/service URL."; - } - } - virtual ~HTTPRequest(); - - virtual const std::string* getAccessKey() const { return nullptr; } - virtual const std::string* getSecretKey() const { return nullptr; } - - virtual bool parseProtocol( - const std::string & url, - std::string & protocol); - - virtual bool SendHTTPRequest( const std::string & payload ); - - unsigned long getResponseCode() const { return responseCode; } - const std::string & getResultString() const { return resultString; } - -protected: - - bool sendPreparedRequest( const std::string & protocol, - const std::string & uri, - const std::string & payload ); - - typedef std::map< std::string, std::string > AttributeValueMap; - AttributeValueMap query_parameters; - AttributeValueMap headers; - - std::string hostUrl; - std::string protocol; - - bool requiresSignature; - struct timespec signatureTime; - - std::string errorMessage; - std::string errorCode; - - std::string resultString; - unsigned long responseCode; - unsigned long expectedResponseCode = 200; - bool includeResponseHeader; - - std::string httpVerb; - - XrdSysError &m_log; + public: + HTTPRequest(const std::string &hostUrl, XrdSysError &log) + : hostUrl(hostUrl), requiresSignature(false), responseCode(0), + includeResponseHeader(false), httpVerb("POST"), m_log(log) { + // Parse the URL and populate + // What to do if the function returns false? + // TODO: Figure out best way to deal with this + if (!parseProtocol(hostUrl, protocol)) { + errorCode = "E_INVALID_HOST_URL"; + errorMessage = "Failed to parse protocol from host/service URL."; + } + } + virtual ~HTTPRequest(); + + virtual const std::string *getAccessKey() const { return nullptr; } + virtual const std::string *getSecretKey() const { return nullptr; } + + virtual bool parseProtocol(const std::string &url, std::string &protocol); + + virtual bool SendHTTPRequest(const std::string &payload); + + unsigned long getResponseCode() const { return responseCode; } + const std::string &getResultString() const { return resultString; } + + protected: + bool sendPreparedRequest(const std::string &protocol, + const std::string &uri, + const std::string &payload); + + typedef std::map AttributeValueMap; + AttributeValueMap query_parameters; + AttributeValueMap headers; + + std::string hostUrl; + std::string protocol; + + bool requiresSignature; + struct timespec signatureTime; + + std::string errorMessage; + std::string errorCode; + + std::string resultString; + unsigned long responseCode; + unsigned long expectedResponseCode = 200; + bool includeResponseHeader; + + std::string httpVerb; + + XrdSysError &m_log; }; class HTTPUpload : public HTTPRequest { -public: - HTTPUpload( - const std::string & h, - const std::string & o, - XrdSysError &log - ) : - HTTPRequest(h, log), - object(o) - { hostUrl = hostUrl + "/" + object; } - - virtual ~HTTPUpload(); - - virtual bool SendRequest( const std::string & payload, off_t offset, size_t size ); - -protected: - std::string object; - std::string path; + public: + HTTPUpload(const std::string &h, const std::string &o, XrdSysError &log) + : HTTPRequest(h, log), object(o) { + hostUrl = hostUrl + "/" + object; + } + + virtual ~HTTPUpload(); + + virtual bool SendRequest(const std::string &payload, off_t offset, + size_t size); + + protected: + std::string object; + std::string path; }; class HTTPDownload : public HTTPRequest { -public: - HTTPDownload( - const std::string & h, - const std::string & o, - XrdSysError &log - ) : - HTTPRequest(h, log), - object(o) - { hostUrl = hostUrl + "/" + object; } - - virtual ~HTTPDownload(); - - virtual bool SendRequest( off_t offset, size_t size ); - -protected: - std::string object; + public: + HTTPDownload(const std::string &h, const std::string &o, XrdSysError &log) + : HTTPRequest(h, log), object(o) { + hostUrl = hostUrl + "/" + object; + } + + virtual ~HTTPDownload(); + + virtual bool SendRequest(off_t offset, size_t size); + + protected: + std::string object; }; class HTTPHead : public HTTPRequest { -public: - HTTPHead( - const std::string & h, - const std::string & o, - XrdSysError &log - ) : - HTTPRequest(h, log), - object(o) - { hostUrl = hostUrl + "/" + object; } - - virtual ~HTTPHead(); - - virtual bool SendRequest(); - -protected: - std::string object; + public: + HTTPHead(const std::string &h, const std::string &o, XrdSysError &log) + : HTTPRequest(h, log), object(o) { + hostUrl = hostUrl + "/" + object; + } + + virtual ~HTTPHead(); + + virtual bool SendRequest(); + + protected: + std::string object; }; diff --git a/src/HTTPDirectory.hh b/src/HTTPDirectory.hh index af2188f..7e1068c 100644 --- a/src/HTTPDirectory.hh +++ b/src/HTTPDirectory.hh @@ -18,42 +18,27 @@ #pragma once -#include "XrdOuc/XrdOucEnv.hh" #include "XrdOss/XrdOss.hh" +#include "XrdOuc/XrdOucEnv.hh" class XrdSysError; class HTTPDirectory : public XrdOssDF { -public: - HTTPDirectory(XrdSysError &log) : - m_log(log) - { - } - - virtual ~HTTPDirectory() {} - - virtual int - Opendir(const char *path, - XrdOucEnv &env) override - { - return -ENOSYS; - } - - virtual int Readdir(char *buff, int blen) override - { - return -ENOSYS; - } - - virtual int StatRet(struct stat *statStruct) override - { - return -ENOSYS; - } - - virtual int Close(long long *retsz=0) override - { - return -ENOSYS; - } - -protected: - XrdSysError &m_log; + public: + HTTPDirectory(XrdSysError &log) : m_log(log) {} + + virtual ~HTTPDirectory() {} + + virtual int Opendir(const char *path, XrdOucEnv &env) override { + return -ENOSYS; + } + + virtual int Readdir(char *buff, int blen) override { return -ENOSYS; } + + virtual int StatRet(struct stat *statStruct) override { return -ENOSYS; } + + virtual int Close(long long *retsz = 0) override { return -ENOSYS; } + + protected: + XrdSysError &m_log; }; diff --git a/src/HTTPFile.cc b/src/HTTPFile.cc index 8beb948..364f2fe 100644 --- a/src/HTTPFile.cc +++ b/src/HTTPFile.cc @@ -16,19 +16,19 @@ * ***************************************************************/ +#include "HTTPFile.hh" #include "HTTPCommands.hh" #include "HTTPFileSystem.hh" -#include "HTTPFile.hh" #include "logging.hh" #include "stl_string_utils.hh" -#include #include #include #include #include #include #include +#include #include #include @@ -41,16 +41,12 @@ using namespace XrdHTTPServer; -HTTPFileSystem* g_http_oss = nullptr; +HTTPFileSystem *g_http_oss = nullptr; XrdVERSIONINFO(XrdOssGetFileSystem, HTTP); -HTTPFile::HTTPFile(XrdSysError &log, HTTPFileSystem *oss) : - m_log(log), - m_oss(oss), - content_length(0), - last_modified(0) -{} +HTTPFile::HTTPFile(XrdSysError &log, HTTPFileSystem *oss) + : m_log(log), m_oss(oss), content_length(0), last_modified(0) {} // Ensures that path is of the form /storagePrefix/object and returns // the resulting object value. The storagePrefix does not necessarily begin @@ -60,251 +56,222 @@ HTTPFile::HTTPFile(XrdSysError &log, HTTPFileSystem *oss) : // /foo/bar, /foo/bar/baz -> baz // storage.com/foo, /storage.com/foo/bar -> bar // /baz, /foo/bar -> error -int -parse_path(const std::string &storagePrefixStr, const char *pathStr, std::string &object) { - const std::filesystem::path storagePath(pathStr); - const std::filesystem::path storagePrefix(storagePrefixStr); - - auto prefixComponents = storagePrefix.begin(); - auto pathComponents = storagePath.begin(); - - std::filesystem::path full; - std::filesystem::path prefix; - - pathComponents++; - if (!storagePrefixStr.empty() && storagePrefixStr[0] == '/') { - prefixComponents++; - } - - while (prefixComponents != storagePrefix.end() && *prefixComponents == *pathComponents ) { - full /= *prefixComponents++; - prefix /= *pathComponents++; - } - - // Check that nothing diverged before reaching end of service name - if (prefixComponents != storagePrefix.end()) { - return -ENOENT; - } - - std::filesystem::path obj_path; - while (pathComponents != storagePath.end()) { - obj_path /= *pathComponents++; - } - - object = obj_path.string(); - return 0; +int parse_path(const std::string &storagePrefixStr, const char *pathStr, + std::string &object) { + const std::filesystem::path storagePath(pathStr); + const std::filesystem::path storagePrefix(storagePrefixStr); + + auto prefixComponents = storagePrefix.begin(); + auto pathComponents = storagePath.begin(); + + std::filesystem::path full; + std::filesystem::path prefix; + + pathComponents++; + if (!storagePrefixStr.empty() && storagePrefixStr[0] == '/') { + prefixComponents++; + } + + while (prefixComponents != storagePrefix.end() && + *prefixComponents == *pathComponents) { + full /= *prefixComponents++; + prefix /= *pathComponents++; + } + + // Check that nothing diverged before reaching end of service name + if (prefixComponents != storagePrefix.end()) { + return -ENOENT; + } + + std::filesystem::path obj_path; + while (pathComponents != storagePath.end()) { + obj_path /= *pathComponents++; + } + + object = obj_path.string(); + return 0; } - -int -HTTPFile::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) -{ - auto configured_hostname = m_oss->getHTTPHostName(); - auto configured_hostUrl = m_oss->getHTTPHostUrl(); - const auto &configured_url_base = m_oss->getHTTPUrlBase(); - if (!configured_url_base.empty()) { - configured_hostUrl = configured_url_base; - configured_hostname = m_oss->getStoragePrefix(); - } - - // - // Check the path for validity. - // - std::string object; - int rv = parse_path(configured_hostname, path, object); - - if( rv != 0 ) { return rv; } - - // We used to query S3 here to see if the object existed, but of course - // if you're creating a file on upload, you don't care. - - this->object = object; - this->hostname = configured_hostname; - this->hostUrl = configured_hostUrl; - - return 0; +int HTTPFile::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) { + auto configured_hostname = m_oss->getHTTPHostName(); + auto configured_hostUrl = m_oss->getHTTPHostUrl(); + const auto &configured_url_base = m_oss->getHTTPUrlBase(); + if (!configured_url_base.empty()) { + configured_hostUrl = configured_url_base; + configured_hostname = m_oss->getStoragePrefix(); + } + + // + // Check the path for validity. + // + std::string object; + int rv = parse_path(configured_hostname, path, object); + + if (rv != 0) { + return rv; + } + + // We used to query S3 here to see if the object existed, but of course + // if you're creating a file on upload, you don't care. + + this->object = object; + this->hostname = configured_hostname; + this->hostUrl = configured_hostUrl; + + return 0; } - -ssize_t -HTTPFile::Read(void *buffer, off_t offset, size_t size) -{ - HTTPDownload download( - this->hostUrl, - this->object, - m_log - ); - m_log.Log(LogMask::Debug, "HTTPFile::Read", "About to perform download from HTTPFile::Read(): hostname / object:", hostname.c_str(), object.c_str()); - - if (!download.SendRequest(offset, size)) { - std::stringstream ss; - ss << "Failed to send GetObject command: " << download.getResponseCode() << "'" << download.getResultString() << "'"; - m_log.Log(LogMask::Warning, "HTTPFile::Read", ss.str().c_str()); - return 0; - } - - const std::string & bytes = download.getResultString(); - memcpy( buffer, bytes.data(), bytes.size() ); - return bytes.size(); +ssize_t HTTPFile::Read(void *buffer, off_t offset, size_t size) { + HTTPDownload download(this->hostUrl, this->object, m_log); + m_log.Log( + LogMask::Debug, "HTTPFile::Read", + "About to perform download from HTTPFile::Read(): hostname / object:", + hostname.c_str(), object.c_str()); + + if (!download.SendRequest(offset, size)) { + std::stringstream ss; + ss << "Failed to send GetObject command: " << download.getResponseCode() + << "'" << download.getResultString() << "'"; + m_log.Log(LogMask::Warning, "HTTPFile::Read", ss.str().c_str()); + return 0; + } + + const std::string &bytes = download.getResultString(); + memcpy(buffer, bytes.data(), bytes.size()); + return bytes.size(); } - -int -HTTPFile::Fstat(struct stat *buff) -{ - m_log.Log(LogMask::Debug, "HTTPFile::Fstat", "About to perform HTTPFile::Fstat():", hostUrl.c_str(), object.c_str()); - HTTPHead head( - hostUrl, - object, - m_log - ); - - if (!head.SendRequest()) { - // SendRequest() returns false for all errors, including ones - // where the server properly responded with something other - // than code 200. If xrootd wants us to distinguish between - // these cases, head.getResponseCode() is initialized to 0, so - // we can check. - std::stringstream ss; - ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" << head.getResultString() << "'"; - m_log.Log(LogMask::Warning, "HTTPFile::Fstat", ss.str().c_str()); - return -ENOENT; - } - - - std::string headers = head.getResultString(); - - std::string line; - size_t current_newline = 0; - size_t next_newline = std::string::npos; - size_t last_character = headers.size(); - while( current_newline != std::string::npos && current_newline != last_character - 1 ) { - next_newline = headers.find( "\r\n", current_newline + 2); - std::string line = substring( headers, current_newline + 2, next_newline ); - - size_t colon = line.find(":"); - if( colon != std::string::npos && colon != line.size() ) { - std::string attr = substring( line, 0, colon ); - toLower(attr); // Some servers might not follow conventional capitalization schemes - std::string value = substring( line, colon + 1 ); - trim(value); - - if( attr == "content-length" ) { - this->content_length = std::stol(value); - } else if( attr == "last-modified" ) { - struct tm t; - char * eos = strptime( value.c_str(), - "%a, %d %b %Y %T %Z", - & t ); - if( eos == & value.c_str()[value.size()] ) { - time_t epoch = timegm(& t); - if( epoch != -1 ) { - this->last_modified = epoch; - } - } - } - } - - current_newline = next_newline; - } - - - buff->st_mode = 0600 | S_IFREG; - buff->st_nlink = 1; - buff->st_uid = 1; - buff->st_gid = 1; - buff->st_size = this->content_length; - buff->st_mtime = this->last_modified; - buff->st_atime = 0; - buff->st_ctime = 0; - buff->st_dev = 0; - buff->st_ino = 0; - - return 0; +int HTTPFile::Fstat(struct stat *buff) { + m_log.Log(LogMask::Debug, "HTTPFile::Fstat", + "About to perform HTTPFile::Fstat():", hostUrl.c_str(), + object.c_str()); + HTTPHead head(hostUrl, object, m_log); + + if (!head.SendRequest()) { + // SendRequest() returns false for all errors, including ones + // where the server properly responded with something other + // than code 200. If xrootd wants us to distinguish between + // these cases, head.getResponseCode() is initialized to 0, so + // we can check. + std::stringstream ss; + ss << "Failed to send HeadObject command: " << head.getResponseCode() + << "'" << head.getResultString() << "'"; + m_log.Log(LogMask::Warning, "HTTPFile::Fstat", ss.str().c_str()); + return -ENOENT; + } + + std::string headers = head.getResultString(); + + std::string line; + size_t current_newline = 0; + size_t next_newline = std::string::npos; + size_t last_character = headers.size(); + while (current_newline != std::string::npos && + current_newline != last_character - 1) { + next_newline = headers.find("\r\n", current_newline + 2); + std::string line = + substring(headers, current_newline + 2, next_newline); + + size_t colon = line.find(":"); + if (colon != std::string::npos && colon != line.size()) { + std::string attr = substring(line, 0, colon); + toLower(attr); // Some servers might not follow conventional + // capitalization schemes + std::string value = substring(line, colon + 1); + trim(value); + + if (attr == "content-length") { + this->content_length = std::stol(value); + } else if (attr == "last-modified") { + struct tm t; + char *eos = strptime(value.c_str(), "%a, %d %b %Y %T %Z", &t); + if (eos == &value.c_str()[value.size()]) { + time_t epoch = timegm(&t); + if (epoch != -1) { + this->last_modified = epoch; + } + } + } + } + + current_newline = next_newline; + } + + buff->st_mode = 0600 | S_IFREG; + buff->st_nlink = 1; + buff->st_uid = 1; + buff->st_gid = 1; + buff->st_size = this->content_length; + buff->st_mtime = this->last_modified; + buff->st_atime = 0; + buff->st_ctime = 0; + buff->st_dev = 0; + buff->st_ino = 0; + + return 0; } - -ssize_t -HTTPFile::Write(const void *buffer, off_t offset, size_t size) -{ - HTTPUpload upload( - this->hostUrl, - this->object, - m_log - ); - - std::string payload( (char *)buffer, size ); - if(! upload.SendRequest( payload, offset, size )) { - m_log.Emsg( "Open", "upload.SendRequest() failed" ); - return -ENOENT; - } else { - m_log.Emsg( "Open", "upload.SendRequest() succeeded" ); - return 0; - } +ssize_t HTTPFile::Write(const void *buffer, off_t offset, size_t size) { + HTTPUpload upload(this->hostUrl, this->object, m_log); + + std::string payload((char *)buffer, size); + if (!upload.SendRequest(payload, offset, size)) { + m_log.Emsg("Open", "upload.SendRequest() failed"); + return -ENOENT; + } else { + m_log.Emsg("Open", "upload.SendRequest() succeeded"); + return 0; + } } - -int HTTPFile::Close(long long *retsz) -{ - m_log.Emsg("Close", "Closed our HTTP file"); - return 0; +int HTTPFile::Close(long long *retsz) { + m_log.Emsg("Close", "Closed our HTTP file"); + return 0; } - extern "C" { /* - This function is called when we are wrapping something. + This function is called when we are wrapping something. */ -XrdOss *XrdOssAddStorageSystem2(XrdOss *curr_oss, - XrdSysLogger *Logger, - const char *config_fn, - const char *parms, - XrdOucEnv *envP) -{ - XrdSysError log(Logger, "s3_"); - - log.Emsg("Initialize", "HTTP filesystem cannot be stacked with other filesystems"); - return nullptr; +XrdOss *XrdOssAddStorageSystem2(XrdOss *curr_oss, XrdSysLogger *Logger, + const char *config_fn, const char *parms, + XrdOucEnv *envP) { + XrdSysError log(Logger, "s3_"); + + log.Emsg("Initialize", + "HTTP filesystem cannot be stacked with other filesystems"); + return nullptr; } - /* - This function is called when it is the top level file system and we are not - wrapping anything + This function is called when it is the top level file system and we are not + wrapping anything */ -XrdOss *XrdOssGetStorageSystem2(XrdOss *native_oss, - XrdSysLogger *Logger, - const char *config_fn, - const char *parms, - XrdOucEnv *envP) -{ - XrdSysError log(Logger, "httpserver_"); - - envP->Export("XRDXROOTD_NOPOSC", "1"); - - try { - g_http_oss = new HTTPFileSystem(Logger, config_fn, envP); - return g_http_oss; - } catch (std::runtime_error &re) { - log.Emsg("Initialize", "Encountered a runtime failure", re.what()); - return nullptr; - } +XrdOss *XrdOssGetStorageSystem2(XrdOss *native_oss, XrdSysLogger *Logger, + const char *config_fn, const char *parms, + XrdOucEnv *envP) { + XrdSysError log(Logger, "httpserver_"); + + envP->Export("XRDXROOTD_NOPOSC", "1"); + + try { + g_http_oss = new HTTPFileSystem(Logger, config_fn, envP); + return g_http_oss; + } catch (std::runtime_error &re) { + log.Emsg("Initialize", "Encountered a runtime failure", re.what()); + return nullptr; + } } - -XrdOss *XrdOssGetStorageSystem(XrdOss *native_oss, - XrdSysLogger *Logger, - const char *config_fn, - const char *parms) -{ - return XrdOssGetStorageSystem2(native_oss, Logger, config_fn, parms, nullptr); +XrdOss *XrdOssGetStorageSystem(XrdOss *native_oss, XrdSysLogger *Logger, + const char *config_fn, const char *parms) { + return XrdOssGetStorageSystem2(native_oss, Logger, config_fn, parms, + nullptr); } - } // end extern "C" - -XrdVERSIONINFO(XrdOssGetStorageSystem, HTTPserver); +XrdVERSIONINFO(XrdOssGetStorageSystem, HTTPserver); XrdVERSIONINFO(XrdOssGetStorageSystem2, HTTPserver); XrdVERSIONINFO(XrdOssAddStorageSystem2, HTTPserver); diff --git a/src/HTTPFile.hh b/src/HTTPFile.hh index 53c1774..df10aca 100644 --- a/src/HTTPFile.hh +++ b/src/HTTPFile.hh @@ -18,130 +18,90 @@ #pragma once +#include "HTTPFileSystem.hh" +#include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdSec/XrdSecEntity.hh" #include "XrdSec/XrdSecEntityAttr.hh" #include "XrdVersion.hh" -#include "XrdOss/XrdOss.hh" -#include "HTTPFileSystem.hh" #include -int parse_path( const std::string & hostname, const char * path,std::string & object ); +int parse_path(const std::string &hostname, const char *path, + std::string &object); class HTTPFile : public XrdOssDF { -public: - HTTPFile(XrdSysError &log, HTTPFileSystem *oss); - - virtual ~HTTPFile() {} - - virtual int Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) override; - - int Fchmod(mode_t mode) override - { - return -ENOSYS; - } - - void Flush() override - { - } - - virtual int Fstat(struct stat *buf) override; - - int Fsync() override - { - return -ENOSYS; - } - - int Fsync(XrdSfsAio *aiop) override - { - return -ENOSYS; - } - - int Ftruncate(unsigned long long size) override - { - return -ENOSYS; - } - - off_t getMmap(void **addr) override - { - return 0; - } - - int isCompressed(char *cxidp=0) override - { - return -ENOSYS; - } - - ssize_t pgRead (void* buffer, off_t offset, size_t rdlen, - uint32_t* csvec, uint64_t opts) override - { - return -ENOSYS; - } - - int pgRead (XrdSfsAio* aioparm, uint64_t opts) override - { - return -ENOSYS; - } - - ssize_t pgWrite(void* buffer, off_t offset, size_t wrlen, - uint32_t* csvec, uint64_t opts) override - { - return -ENOSYS; - } - - int pgWrite(XrdSfsAio* aioparm, uint64_t opts) override - { - return -ENOSYS; - } - - ssize_t Read(off_t offset, size_t size) override - { - return -ENOSYS; - } - - virtual ssize_t Read(void *buffer, off_t offset, size_t size) override; - - int Read(XrdSfsAio *aiop) override - { - return -ENOSYS; - } - - ssize_t ReadRaw(void *buffer, off_t offset, size_t size) override - { - return -ENOSYS; - } - - ssize_t ReadV(XrdOucIOVec *readV, int rdvcnt) override - { - return -ENOSYS; - } - - virtual ssize_t Write(const void *buffer, off_t offset, size_t size) override; - - int Write(XrdSfsAio *aiop) override - { - return -ENOSYS; - } - - ssize_t WriteV(XrdOucIOVec *writeV, int wrvcnt) override - { - return -ENOSYS; - } - - virtual int Close(long long *retsz=0) override; // upstream is abstract definition - - size_t getContentLength() { return content_length; } - time_t getLastModified() { return last_modified; } - -private: - XrdSysError &m_log; - HTTPFileSystem *m_oss; - - std::string hostname; - std::string hostUrl; - std::string object; - - size_t content_length; - time_t last_modified; + public: + HTTPFile(XrdSysError &log, HTTPFileSystem *oss); + + virtual ~HTTPFile() {} + + virtual int Open(const char *path, int Oflag, mode_t Mode, + XrdOucEnv &env) override; + + int Fchmod(mode_t mode) override { return -ENOSYS; } + + void Flush() override {} + + virtual int Fstat(struct stat *buf) override; + + int Fsync() override { return -ENOSYS; } + + int Fsync(XrdSfsAio *aiop) override { return -ENOSYS; } + + int Ftruncate(unsigned long long size) override { return -ENOSYS; } + + off_t getMmap(void **addr) override { return 0; } + + int isCompressed(char *cxidp = 0) override { return -ENOSYS; } + + ssize_t pgRead(void *buffer, off_t offset, size_t rdlen, uint32_t *csvec, + uint64_t opts) override { + return -ENOSYS; + } + + int pgRead(XrdSfsAio *aioparm, uint64_t opts) override { return -ENOSYS; } + + ssize_t pgWrite(void *buffer, off_t offset, size_t wrlen, uint32_t *csvec, + uint64_t opts) override { + return -ENOSYS; + } + + int pgWrite(XrdSfsAio *aioparm, uint64_t opts) override { return -ENOSYS; } + + ssize_t Read(off_t offset, size_t size) override { return -ENOSYS; } + + virtual ssize_t Read(void *buffer, off_t offset, size_t size) override; + + int Read(XrdSfsAio *aiop) override { return -ENOSYS; } + + ssize_t ReadRaw(void *buffer, off_t offset, size_t size) override { + return -ENOSYS; + } + + ssize_t ReadV(XrdOucIOVec *readV, int rdvcnt) override { return -ENOSYS; } + + virtual ssize_t Write(const void *buffer, off_t offset, + size_t size) override; + + int Write(XrdSfsAio *aiop) override { return -ENOSYS; } + + ssize_t WriteV(XrdOucIOVec *writeV, int wrvcnt) override { return -ENOSYS; } + + virtual int + Close(long long *retsz = 0) override; // upstream is abstract definition + + size_t getContentLength() { return content_length; } + time_t getLastModified() { return last_modified; } + + private: + XrdSysError &m_log; + HTTPFileSystem *m_oss; + + std::string hostname; + std::string hostUrl; + std::string object; + + size_t content_length; + time_t last_modified; }; diff --git a/src/HTTPFileSystem.cc b/src/HTTPFileSystem.cc index 979d8ed..5e17b9d 100644 --- a/src/HTTPFileSystem.cc +++ b/src/HTTPFileSystem.cc @@ -16,9 +16,9 @@ * ***************************************************************/ +#include "HTTPFileSystem.hh" #include "HTTPDirectory.hh" #include "HTTPFile.hh" -#include "HTTPFileSystem.hh" #include "logging.hh" #include @@ -27,169 +27,159 @@ #include #include -#include -#include #include +#include +#include #include -#include #include +#include #include #include "stl_string_utils.hh" using namespace XrdHTTPServer; -HTTPFileSystem::HTTPFileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP) : - m_env(envP), - m_log(lp, "httpserver_") -{ - m_log.Say("------ Initializing the HTTP filesystem plugin."); - if (!Config(lp, configfn)) { - throw std::runtime_error("Failed to configure HTTP filesystem plugin."); - } +HTTPFileSystem::HTTPFileSystem(XrdSysLogger *lp, const char *configfn, + XrdOucEnv *envP) + : m_env(envP), m_log(lp, "httpserver_") { + m_log.Say("------ Initializing the HTTP filesystem plugin."); + if (!Config(lp, configfn)) { + throw std::runtime_error("Failed to configure HTTP filesystem plugin."); + } } - -HTTPFileSystem::~HTTPFileSystem() { -} - - -bool -HTTPFileSystem::handle_required_config( - const std::string & name_from_config, - const char * desired_name, - const std::string & source, - std::string & target -) { - if( name_from_config != desired_name ) { return true; } - - if( source.empty() ) { - std::string error; - formatstr( error, "%s must specify a value", desired_name ); - m_log.Emsg( "Config", error.c_str() ); - return false; - } - - std::stringstream ss; - ss << "Setting " << desired_name << "=" << source; - m_log.Log(LogMask::Debug, "Config", ss.str().c_str()); - target = source; - return true; +HTTPFileSystem::~HTTPFileSystem() {} + +bool HTTPFileSystem::handle_required_config(const std::string &name_from_config, + const char *desired_name, + const std::string &source, + std::string &target) { + if (name_from_config != desired_name) { + return true; + } + + if (source.empty()) { + std::string error; + formatstr(error, "%s must specify a value", desired_name); + m_log.Emsg("Config", error.c_str()); + return false; + } + + std::stringstream ss; + ss << "Setting " << desired_name << "=" << source; + m_log.Log(LogMask::Debug, "Config", ss.str().c_str()); + target = source; + return true; } - -bool -HTTPFileSystem::Config(XrdSysLogger *lp, const char *configfn) -{ - XrdOucEnv myEnv; - XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); - - int cfgFD = open(configfn, O_RDONLY, 0); - if (cfgFD < 0) { - m_log.Emsg("Config", errno, "open config file", configfn); - return false; - } - - char * temporary; - std::string value; - std::string attribute; - Config.Attach(cfgFD); - while ((temporary = Config.GetMyFirstWord())) { - attribute = temporary; - if (attribute == "httpserver.trace") { - if (!XrdHTTPServer::ConfigLog(Config, m_log)) { - m_log.Emsg("Config", "Failed to configure the log level"); - } - continue; - } - - temporary = Config.GetWord(); - if(! temporary) { continue; } - value = temporary; - - if (!handle_required_config(attribute, "httpserver.host_name", value, http_host_name) || - !handle_required_config(attribute, "httpserver.host_url", value, http_host_url) || - !handle_required_config(attribute, "httpserver.url_base", value, m_url_base) || - !handle_required_config(attribute, "httpserver.storage_prefix", value, m_storage_prefix)) - { - Config.Close(); - return false; - } - } - - if (m_url_base.empty()) { - if (http_host_name.empty()) { - m_log.Emsg("Config", "httpserver.host_name not specified; this or httpserver.url_base are required"); - return false; - } - if (http_host_url.empty()) { - m_log.Emsg("Config", "httpserver.host_url not specified; this or httpserver.url_base are required"); - return false; - } - } - - int retc = Config.LastError(); - if( retc ) { - m_log.Emsg("Config", -retc, "read config file", configfn); - Config.Close(); - return false; - } - - Config.Close(); - return true; +bool HTTPFileSystem::Config(XrdSysLogger *lp, const char *configfn) { + XrdOucEnv myEnv; + XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); + + int cfgFD = open(configfn, O_RDONLY, 0); + if (cfgFD < 0) { + m_log.Emsg("Config", errno, "open config file", configfn); + return false; + } + + char *temporary; + std::string value; + std::string attribute; + Config.Attach(cfgFD); + while ((temporary = Config.GetMyFirstWord())) { + attribute = temporary; + if (attribute == "httpserver.trace") { + if (!XrdHTTPServer::ConfigLog(Config, m_log)) { + m_log.Emsg("Config", "Failed to configure the log level"); + } + continue; + } + + temporary = Config.GetWord(); + if (!temporary) { + continue; + } + value = temporary; + + if (!handle_required_config(attribute, "httpserver.host_name", value, + http_host_name) || + !handle_required_config(attribute, "httpserver.host_url", value, + http_host_url) || + !handle_required_config(attribute, "httpserver.url_base", value, + m_url_base) || + !handle_required_config(attribute, "httpserver.storage_prefix", + value, m_storage_prefix)) { + Config.Close(); + return false; + } + } + + if (m_url_base.empty()) { + if (http_host_name.empty()) { + m_log.Emsg("Config", "httpserver.host_name not specified; this or " + "httpserver.url_base are required"); + return false; + } + if (http_host_url.empty()) { + m_log.Emsg("Config", "httpserver.host_url not specified; this or " + "httpserver.url_base are required"); + return false; + } + } + + int retc = Config.LastError(); + if (retc) { + m_log.Emsg("Config", -retc, "read config file", configfn); + Config.Close(); + return false; + } + + Config.Close(); + return true; } - // Object Allocation Functions // -XrdOssDF * -HTTPFileSystem::newDir(const char *user) -{ - return new HTTPDirectory(m_log); +XrdOssDF *HTTPFileSystem::newDir(const char *user) { + return new HTTPDirectory(m_log); } - -XrdOssDF * -HTTPFileSystem::newFile(const char *user) -{ - return new HTTPFile(m_log, this); +XrdOssDF *HTTPFileSystem::newFile(const char *user) { + return new HTTPFile(m_log, this); } - -int -HTTPFileSystem::Stat(const char *path, struct stat *buff, - int opts, XrdOucEnv *env) -{ - std::string error; - - m_log.Emsg("Stat", "Stat'ing path", path); - - HTTPFile httpFile(m_log, this); - int rv = httpFile.Open( path, 0, (mode_t)0, *env ); - if (rv) { - m_log.Emsg("Stat", "Failed to open path:", path); - } - // Assume that HTTPFile::FStat() doesn't write to buff unless it succeeds. - rv = httpFile.Fstat( buff ); - if( rv != 0 ) { - formatstr( error, "File %s not found.", path ); - m_log.Emsg( "Stat", error.c_str() ); - return -ENOENT; - } - - return 0; +int HTTPFileSystem::Stat(const char *path, struct stat *buff, int opts, + XrdOucEnv *env) { + std::string error; + + m_log.Emsg("Stat", "Stat'ing path", path); + + HTTPFile httpFile(m_log, this); + int rv = httpFile.Open(path, 0, (mode_t)0, *env); + if (rv) { + m_log.Emsg("Stat", "Failed to open path:", path); + } + // Assume that HTTPFile::FStat() doesn't write to buff unless it succeeds. + rv = httpFile.Fstat(buff); + if (rv != 0) { + formatstr(error, "File %s not found.", path); + m_log.Emsg("Stat", error.c_str()); + return -ENOENT; + } + + return 0; } -int -HTTPFileSystem::Create( const char *tid, const char *path, mode_t mode, - XrdOucEnv &env, int opts ) -{ - // Is path valid? - std::string object; - std::string hostname = this->getHTTPHostName(); - int rv = parse_path( hostname, path, object ); - if( rv != 0 ) { return rv; } - - return 0; +int HTTPFileSystem::Create(const char *tid, const char *path, mode_t mode, + XrdOucEnv &env, int opts) { + // Is path valid? + std::string object; + std::string hostname = this->getHTTPHostName(); + int rv = parse_path(hostname, path, object); + if (rv != 0) { + return rv; + } + + return 0; } diff --git a/src/HTTPFileSystem.hh b/src/HTTPFileSystem.hh index 6d19134..eebc43e 100644 --- a/src/HTTPFileSystem.hh +++ b/src/HTTPFileSystem.hh @@ -27,73 +27,94 @@ #include class HTTPFileSystem : public XrdOss { -public: + public: + HTTPFileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP); + virtual ~HTTPFileSystem(); - HTTPFileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP); - virtual ~HTTPFileSystem(); + virtual bool Config(XrdSysLogger *lp, const char *configfn); - virtual bool - Config(XrdSysLogger *lp, const char *configfn); + XrdOssDF *newDir(const char *user = 0); + XrdOssDF *newFile(const char *user = 0); - XrdOssDF *newDir(const char *user=0); - XrdOssDF *newFile(const char *user=0); + int Chmod(const char *path, mode_t mode, XrdOucEnv *env = 0) { + return -ENOSYS; + } + void Connect(XrdOucEnv &env) {} + int Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env, + int opts = 0); + void Disc(XrdOucEnv &env) {} + void EnvInfo(XrdOucEnv *env) {} + uint64_t Features() { return 0; } + int FSctl(int cmd, int alen, const char *args, char **resp = 0) { + return -ENOSYS; + } + int Init(XrdSysLogger *lp, const char *cfn) { return 0; } + int Init(XrdSysLogger *lp, const char *cfn, XrdOucEnv *en) { return 0; } + int Mkdir(const char *path, mode_t mode, int mkpath = 0, + XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Reloc(const char *tident, const char *path, const char *cgName, + const char *anchor = 0) { + return -ENOSYS; + } + int Remdir(const char *path, int Opts = 0, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Rename(const char *oPath, const char *nPath, XrdOucEnv *oEnvP = 0, + XrdOucEnv *nEnvP = 0) { + return -ENOSYS; + } + int Stat(const char *path, struct stat *buff, int opts = 0, + XrdOucEnv *env = 0); + int Stats(char *buff, int blen) { return -ENOSYS; } + int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int StatLS(XrdOucEnv &env, const char *path, char *buff, int &blen) { + return -ENOSYS; + } + int StatPF(const char *path, struct stat *buff, int opts) { + return -ENOSYS; + } + int StatPF(const char *path, struct stat *buff) { return -ENOSYS; } + int StatVS(XrdOssVSInfo *vsP, const char *sname = 0, int updt = 0) { + return -ENOSYS; + } + int StatXA(const char *path, char *buff, int &blen, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int StatXP(const char *path, unsigned long long &attr, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Truncate(const char *path, unsigned long long fsize, + XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Unlink(const char *path, int Opts = 0, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Lfn2Pfn(const char *Path, char *buff, int blen) { return -ENOSYS; } + const char *Lfn2Pfn(const char *Path, char *buff, int blen, int &rc) { + return nullptr; + } - int Chmod(const char * path, mode_t mode, XrdOucEnv *env=0) {return -ENOSYS;} - void Connect(XrdOucEnv &env) {} - int Create(const char *tid, const char *path, mode_t mode, - XrdOucEnv &env, int opts=0); - void Disc(XrdOucEnv &env) {} - void EnvInfo(XrdOucEnv *env) {} - uint64_t Features() {return 0;} - int FSctl(int cmd, int alen, const char *args, char **resp=0) {return -ENOSYS;} - int Init(XrdSysLogger *lp, const char *cfn) {return 0;} - int Init(XrdSysLogger *lp, const char *cfn, XrdOucEnv *en) {return 0;} - int Mkdir(const char *path, mode_t mode, int mkpath=0, - XrdOucEnv *env=0) {return -ENOSYS;} - int Reloc(const char *tident, const char *path, - const char *cgName, const char *anchor=0) {return -ENOSYS;} - int Remdir(const char *path, int Opts=0, XrdOucEnv *env=0) {return -ENOSYS;} - int Rename(const char *oPath, const char *nPath, - XrdOucEnv *oEnvP=0, XrdOucEnv *nEnvP=0) {return -ENOSYS;} - int Stat(const char *path, struct stat *buff, - int opts=0, XrdOucEnv *env=0); - int Stats(char *buff, int blen) {return -ENOSYS;} - int StatFS(const char *path, char *buff, int &blen, - XrdOucEnv *env=0) {return -ENOSYS;} - int StatLS(XrdOucEnv &env, const char *path, - char *buff, int &blen) {return -ENOSYS;} - int StatPF(const char *path, struct stat *buff, int opts) {return -ENOSYS;} - int StatPF(const char *path, struct stat *buff) {return -ENOSYS;} - int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0) {return -ENOSYS;} - int StatXA(const char *path, char *buff, int &blen, - XrdOucEnv *env=0) {return -ENOSYS;} - int StatXP(const char *path, unsigned long long &attr, - XrdOucEnv *env=0) {return -ENOSYS;} - int Truncate(const char *path, unsigned long long fsize, - XrdOucEnv *env=0) {return -ENOSYS;} - int Unlink(const char *path, int Opts=0, XrdOucEnv *env=0) {return -ENOSYS;} - int Lfn2Pfn(const char *Path, char *buff, int blen) {return -ENOSYS;} - const char *Lfn2Pfn(const char *Path, char *buff, int blen, int &rc) {return nullptr;} + const std::string &getHTTPHostName() const { return http_host_name; } + const std::string &getHTTPHostUrl() const { return http_host_url; } + const std::string &getHTTPUrlBase() const { return m_url_base; } + const std::string &getStoragePrefix() const { return m_storage_prefix; } - const std::string &getHTTPHostName() const {return http_host_name;} - const std::string &getHTTPHostUrl() const {return http_host_url;} - const std::string &getHTTPUrlBase() const {return m_url_base;} - const std::string &getStoragePrefix() const {return m_storage_prefix;} + protected: + XrdOucEnv *m_env; + XrdSysError m_log; -protected: - XrdOucEnv *m_env; - XrdSysError m_log; + bool handle_required_config(const std::string &name_from_config, + const char *desired_name, + const std::string &source, std::string &target); - bool handle_required_config( - const std::string & name_from_config, - const char * desired_name, - const std::string & source, - std::string & target - ); - -private: - std::string http_host_name; - std::string http_host_url; - std::string m_url_base; - std::string m_storage_prefix; + private: + std::string http_host_name; + std::string http_host_url; + std::string m_url_base; + std::string m_storage_prefix; }; diff --git a/src/S3AccessInfo.cc b/src/S3AccessInfo.cc index 783076d..8664833 100644 --- a/src/S3AccessInfo.cc +++ b/src/S3AccessInfo.cc @@ -5,47 +5,47 @@ #include "S3AccessInfo.hh" const std::string &S3AccessInfo::getS3BucketName() const { - return s3_bucket_name; + return s3_bucket_name; } void S3AccessInfo::setS3BucketName(const std::string &s3BucketName) { - s3_bucket_name = s3BucketName; + s3_bucket_name = s3BucketName; } const std::string &S3AccessInfo::getS3ServiceName() const { - return s3_service_name; + return s3_service_name; } void S3AccessInfo::setS3ServiceName(const std::string &s3ServiceName) { - s3_service_name = s3ServiceName; + s3_service_name = s3ServiceName; } const std::string &S3AccessInfo::getS3Region() const { return s3_region; } void S3AccessInfo::setS3Region(const std::string &s3Region) { - s3_region = s3Region; + s3_region = s3Region; } const std::string &S3AccessInfo::getS3ServiceUrl() const { - return s3_service_url; + return s3_service_url; } void S3AccessInfo::setS3ServiceUrl(const std::string &s3ServiceUrl) { - s3_service_url = s3ServiceUrl; + s3_service_url = s3ServiceUrl; } const std::string &S3AccessInfo::getS3AccessKeyFile() const { - return s3_access_key_file; + return s3_access_key_file; } void S3AccessInfo::setS3AccessKeyFile(const std::string &s3AccessKeyFile) { - s3_access_key_file = s3AccessKeyFile; + s3_access_key_file = s3AccessKeyFile; } const std::string &S3AccessInfo::getS3SecretKeyFile() const { - return s3_secret_key_file; + return s3_secret_key_file; } void S3AccessInfo::setS3SecretKeyFile(const std::string &s3SecretKeyFile) { - s3_secret_key_file = s3SecretKeyFile; + s3_secret_key_file = s3SecretKeyFile; } diff --git a/src/S3AccessInfo.hh b/src/S3AccessInfo.hh index 853bee7..2770ee0 100644 --- a/src/S3AccessInfo.hh +++ b/src/S3AccessInfo.hh @@ -5,42 +5,41 @@ #ifndef XROOTD_S3_HTTP_S3ACCESSINFO_HH #define XROOTD_S3_HTTP_S3ACCESSINFO_HH - #include class S3AccessInfo { -public: - const std::string &getS3BucketName() const; + public: + const std::string &getS3BucketName() const; - void setS3BucketName(const std::string &s3BucketName); + void setS3BucketName(const std::string &s3BucketName); - const std::string &getS3ServiceName() const; + const std::string &getS3ServiceName() const; - void setS3ServiceName(const std::string &s3ServiceName); + void setS3ServiceName(const std::string &s3ServiceName); - const std::string &getS3Region() const; + const std::string &getS3Region() const; - void setS3Region(const std::string &s3Region); + void setS3Region(const std::string &s3Region); - const std::string &getS3ServiceUrl() const; + const std::string &getS3ServiceUrl() const; - void setS3ServiceUrl(const std::string &s3ServiceUrl); + void setS3ServiceUrl(const std::string &s3ServiceUrl); - const std::string &getS3AccessKeyFile() const; + const std::string &getS3AccessKeyFile() const; - void setS3AccessKeyFile(const std::string &s3AccessKeyFile); + void setS3AccessKeyFile(const std::string &s3AccessKeyFile); - const std::string &getS3SecretKeyFile() const; + const std::string &getS3SecretKeyFile() const; - void setS3SecretKeyFile(const std::string &s3SecretKeyFile); + void setS3SecretKeyFile(const std::string &s3SecretKeyFile); -private: - std::string s3_bucket_name; - std::string s3_service_name; - std::string s3_region; - std::string s3_service_url; - std::string s3_access_key_file; - std::string s3_secret_key_file; + private: + std::string s3_bucket_name; + std::string s3_service_name; + std::string s3_region; + std::string s3_service_url; + std::string s3_access_key_file; + std::string s3_secret_key_file; }; -#endif //XROOTD_S3_HTTP_S3ACCESSINFO_HH +#endif // XROOTD_S3_HTTP_S3ACCESSINFO_HH diff --git a/src/S3Commands.cc b/src/S3Commands.cc index 8a445ba..f5883dc 100644 --- a/src/S3Commands.cc +++ b/src/S3Commands.cc @@ -16,14 +16,14 @@ * ***************************************************************/ -#include "AWSv4-impl.hh" #include "S3Commands.hh" +#include "AWSv4-impl.hh" #include "shortfile.hh" #include "stl_string_utils.hh" -#include -#include #include +#include +#include #include #include @@ -33,120 +33,128 @@ #include #include -AmazonRequest::~AmazonRequest() { } +AmazonRequest::~AmazonRequest() {} bool AmazonRequest::SendRequest() { - query_parameters.insert( std::make_pair( "Version", "2012-10-01" ) ); + query_parameters.insert(std::make_pair("Version", "2012-10-01")); - switch( signatureVersion ) { - case 4: - return sendV4Request( canonicalizeQueryString() ); - default: - this->errorCode = "E_INTERNAL"; - this->errorMessage = "Invalid signature version."; - // dprintf( D_ALWAYS, "Invalid signature version (%d), failing.\n", signatureVersion ); - return false; + switch (signatureVersion) { + case 4: + return sendV4Request(canonicalizeQueryString()); + default: + this->errorCode = "E_INTERNAL"; + this->errorMessage = "Invalid signature version."; + // dprintf( D_ALWAYS, "Invalid signature version (%d), failing.\n", + // signatureVersion ); + return false; } } std::string AmazonRequest::canonicalizeQueryString() { - return AWSv4Impl::canonicalizeQueryString( query_parameters ); + return AWSv4Impl::canonicalizeQueryString(query_parameters); } -// Takes in the configured `s3.service_url` and uses the bucket/object requested to generate -// the virtual host URL, as well as the canonical URI (which is the path to the object). -bool AmazonRequest::parseURL( const std::string & url, - std::string & path ) { - auto i = url.find( "://" ); - if( i == std::string::npos ) { return false; } - - auto j = url.find( "/", i + 3 ); - if( j == std::string::npos ) { - if (style == "path") { - // If we're configured for path-style requests, then the host is everything between +// Takes in the configured `s3.service_url` and uses the bucket/object requested +// to generate the virtual host URL, as well as the canonical URI (which is the +// path to the object). +bool AmazonRequest::parseURL(const std::string &url, std::string &path) { + auto i = url.find("://"); + if (i == std::string::npos) { + return false; + } + + auto j = url.find("/", i + 3); + if (j == std::string::npos) { + if (style == "path") { + // If we're configured for path-style requests, then the host is + // everything between // :// and the last / - host = substring( url, i + 3 ); + host = substring(url, i + 3); // Likewise, the path is going to be /bucket/object path = "/" + bucket + "/" + object; - } else { - // In virtual-style requests, the host should be determined as everything between + } else { + // In virtual-style requests, the host should be determined as + // everything between // :// up until the last /, but with appended to the front. - host = bucket + "." + substring( url, i + 3 ); + host = bucket + "." + substring(url, i + 3); path = "/" + object; - } - - return true; - } + } + + return true; + } if (style == "path") { - host = substring( url, i + 3, j ); - path = substring( url, j ) + "/" + bucket + "/" + object; + host = substring(url, i + 3, j); + path = substring(url, j) + "/" + bucket + "/" + object; } else { - host = bucket + "." + substring( url, i + 3, j ); - path = substring( url, j ) + object; + host = bucket + "." + substring(url, i + 3, j); + path = substring(url, j) + object; } - return true; + return true; } -void convertMessageDigestToLowercaseHex( - const unsigned char * messageDigest, - unsigned int mdLength, - std::string & hexEncoded ) { - AWSv4Impl::convertMessageDigestToLowercaseHex( messageDigest, - mdLength, hexEncoded ); +void convertMessageDigestToLowercaseHex(const unsigned char *messageDigest, + unsigned int mdLength, + std::string &hexEncoded) { + AWSv4Impl::convertMessageDigestToLowercaseHex(messageDigest, mdLength, + hexEncoded); } - -bool doSha256( const std::string & payload, - unsigned char * messageDigest, - unsigned int * mdLength ) { - return AWSv4Impl::doSha256( payload, messageDigest, mdLength ); +bool doSha256(const std::string &payload, unsigned char *messageDigest, + unsigned int *mdLength) { + return AWSv4Impl::doSha256(payload, messageDigest, mdLength); } -std::string pathEncode( const std::string & original ) { - return AWSv4Impl::pathEncode( original ); +std::string pathEncode(const std::string &original) { + return AWSv4Impl::pathEncode(original); } -bool AmazonRequest::createV4Signature( const std::string & payload, - std::string & authorizationValue, - bool sendContentSHA ) { +bool AmazonRequest::createV4Signature(const std::string &payload, + std::string &authorizationValue, + bool sendContentSHA) { // If we're using temporary credentials, we need to add the token // header here as well. We set saKey and keyID here (well before // necessary) since we'll get them for free when we get the token. std::string keyID; std::string saKey; std::string token; - if (!this->secretKeyFile.empty()) { // Some origins may exist in front of unauthenticated buckets - if( ! readShortFile( this->secretKeyFile, saKey ) ) { + if (!this->secretKeyFile.empty()) { // Some origins may exist in front of + // unauthenticated buckets + if (!readShortFile(this->secretKeyFile, saKey)) { this->errorCode = "E_FILE_IO"; - this->errorMessage = "Unable to read from secretkey file '" + this->secretKeyFile + "'."; + this->errorMessage = "Unable to read from secretkey file '" + + this->secretKeyFile + "'."; return false; } - trim( saKey ); - } - else { - requiresSignature = false; // If we don't create a signature, it must not be needed... + trim(saKey); + } else { + requiresSignature = + false; // If we don't create a signature, it must not be needed... return true; // If there was no saKey, we need not generate a signature } - if (!this->accessKeyFile.empty()) { // Some origins may exist in front of unauthenticated buckets - if( ! readShortFile( this->accessKeyFile, keyID ) ) { + if (!this->accessKeyFile.empty()) { // Some origins may exist in front of + // unauthenticated buckets + if (!readShortFile(this->accessKeyFile, keyID)) { this->errorCode = "E_FILE_IO"; - this->errorMessage = "Unable to read from accesskey file '" + this->accessKeyFile + "'."; + this->errorMessage = "Unable to read from accesskey file '" + + this->accessKeyFile + "'."; return false; } - trim( keyID ); - } - else { + trim(keyID); + } else { this->errorCode = "E_FILE_IO"; - this->errorMessage = "The secretkey file was read, but I can't read from accesskey file '" + this->secretKeyFile + "'."; - return false; + this->errorMessage = "The secretkey file was read, but I can't read " + "from accesskey file '" + + this->secretKeyFile + "'."; + return false; } - - time_t now; time( & now ); - struct tm brokenDownTime; gmtime_r( & now, & brokenDownTime ); + time_t now; + time(&now); + struct tm brokenDownTime; + gmtime_r(&now, &brokenDownTime); // // Create task 1's inputs. @@ -158,7 +166,7 @@ bool AmazonRequest::createV4Signature( const std::string & payload, // But that sounds like a lot of work, so until something we do actually // requires it, I'll just assume the path is already normalized. - canonicalURI = pathEncode( canonicalURI ); + canonicalURI = pathEncode(canonicalURI); // The canonical query string is the alphabetically sorted list of // URI-encoded parameter names '=' values, separated by '&'s. That @@ -168,21 +176,21 @@ bool AmazonRequest::createV4Signature( const std::string & payload, // This function doesn't (currently) support query parameters, // but no current caller attempts to use them. - assert( (httpVerb != "GET") || query_parameters.size() == 0 ); + assert((httpVerb != "GET") || query_parameters.size() == 0); // The canonical headers must include the Host header, so add that // now if we don't have it. - if( headers.find( "Host" ) == headers.end() ) { - headers[ "Host" ] = host; + if (headers.find("Host") == headers.end()) { + headers["Host"] = host; } // S3 complains if x-amz-date isn't signed, so do this early. char dt[] = "YYYYMMDDThhmmssZ"; - strftime( dt, sizeof(dt), "%Y%m%dT%H%M%SZ", & brokenDownTime ); - headers[ "X-Amz-Date" ] = dt; + strftime(dt, sizeof(dt), "%Y%m%dT%H%M%SZ", &brokenDownTime); + headers["X-Amz-Date"] = dt; char d[] = "YYYYMMDD"; - strftime( d, sizeof(d), "%Y%m%d", & brokenDownTime ); + strftime(d, sizeof(d), "%Y%m%d", &brokenDownTime); // S3 complains if x-amz-content-sha256 isn't signed, which makes sense, // so do this early. @@ -191,49 +199,54 @@ bool AmazonRequest::createV4Signature( const std::string & payload, // (SHA256) hash value of the payload. unsigned int mdLength = 0; unsigned char messageDigest[EVP_MAX_MD_SIZE]; - if(! doSha256( payload, messageDigest, & mdLength )) { + if (!doSha256(payload, messageDigest, &mdLength)) { this->errorCode = "E_INTERNAL"; this->errorMessage = "Unable to hash payload."; // dprintf( D_ALWAYS, "Unable to hash payload, failing.\n" ); return false; } std::string payloadHash; - convertMessageDigestToLowercaseHex( messageDigest, mdLength, payloadHash ); - if( sendContentSHA ) { - headers[ "X-Amz-Content-Sha256" ] = payloadHash; + convertMessageDigestToLowercaseHex(messageDigest, mdLength, payloadHash); + if (sendContentSHA) { + headers["X-Amz-Content-Sha256"] = payloadHash; } // The canonical list of headers is a sorted list of lowercase header // names paired via ':' with the trimmed header value, each pair // terminated with a newline. AmazonRequest::AttributeValueMap transformedHeaders; - for( auto i = headers.begin(); i != headers.end(); ++i ) { + for (auto i = headers.begin(); i != headers.end(); ++i) { std::string header = i->first; - std::transform( header.begin(), header.end(), header.begin(), & tolower ); + std::transform(header.begin(), header.end(), header.begin(), &tolower); std::string value = i->second; // We need to leave empty headers alone so that they can be used // to disable CURL stupidity later. - if( value.size() == 0 ) { + if (value.size() == 0) { continue; } // Eliminate trailing spaces. unsigned j = value.length() - 1; - while( value[j] == ' ' ) { --j; } - if( j != value.length() - 1 ) { value.erase( j + 1 ); } + while (value[j] == ' ') { + --j; + } + if (j != value.length() - 1) { + value.erase(j + 1); + } // Eliminate leading spaces. - for( j = 0; value[j] == ' '; ++j ) { } - value.erase( 0, j ); + for (j = 0; value[j] == ' '; ++j) { + } + value.erase(0, j); // Convert internal runs of spaces into single spaces. unsigned left = 1; unsigned right = 1; bool inSpaces = false; - while( right < value.length() ) { - if(! inSpaces) { - if( value[right] == ' ' ) { + while (right < value.length()) { + if (!inSpaces) { + if (value[right] == ' ') { inSpaces = true; left = right; ++right; @@ -241,78 +254,77 @@ bool AmazonRequest::createV4Signature( const std::string & payload, ++right; } } else { - if( value[right] == ' ' ) { + if (value[right] == ' ') { ++right; } else { inSpaces = false; - value.erase( left, right - left - 1 ); + value.erase(left, right - left - 1); right = left + 1; } } } - transformedHeaders[ header ] = value; + transformedHeaders[header] = value; } // The canonical list of signed headers is trivial to generate while // generating the list of headers. std::string signedHeaders; std::string canonicalHeaders; - for( auto i = transformedHeaders.begin(); i != transformedHeaders.end(); ++i ) { + for (auto i = transformedHeaders.begin(); i != transformedHeaders.end(); + ++i) { canonicalHeaders += i->first + ":" + i->second + "\n"; signedHeaders += i->first + ";"; } - signedHeaders.erase( signedHeaders.end() - 1 ); + signedHeaders.erase(signedHeaders.end() - 1); // Task 1: create the canonical request. - std::string canonicalRequest = httpVerb + "\n" - + canonicalURI + "\n" - + canonicalQueryString + "\n" - + canonicalHeaders + "\n" - + signedHeaders + "\n" - + payloadHash; + std::string canonicalRequest = + httpVerb + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash; // // Create task 2's inputs. // // Hash the canonical request the way we did the payload. - if(! doSha256( canonicalRequest, messageDigest, & mdLength )) { + if (!doSha256(canonicalRequest, messageDigest, &mdLength)) { this->errorCode = "E_INTERNAL"; this->errorMessage = "Unable to hash canonical request."; return false; } std::string canonicalRequestHash; - convertMessageDigestToLowercaseHex( messageDigest, mdLength, canonicalRequestHash ); + convertMessageDigestToLowercaseHex(messageDigest, mdLength, + canonicalRequestHash); std::string s = this->service; - if( s.empty() ) { - size_t i = host.find( "." ); - if( i != std::string::npos ) { - s = host.substr( 0, i ); + if (s.empty()) { + size_t i = host.find("."); + if (i != std::string::npos) { + s = host.substr(0, i); } else { s = host; } } std::string r = this->region; - if( r.empty() ) { - size_t i = host.find( "." ); - size_t j = host.find( ".", i + 1 ); - if( j != std::string::npos ) { - r = host.substr( i + 1, j - i - 1 ); + if (r.empty()) { + size_t i = host.find("."); + size_t j = host.find(".", i + 1); + if (j != std::string::npos) { + r = host.substr(i + 1, j - i - 1); } else { r = host; } } - // Task 2: create the string to sign. std::string credentialScope; - formatstr( credentialScope, "%s/%s/%s/aws4_request", d, r.c_str(), s.c_str() ); + formatstr(credentialScope, "%s/%s/%s/aws4_request", d, r.c_str(), + s.c_str()); std::string stringToSign; - formatstr( stringToSign, "AWS4-HMAC-SHA256\n%s\n%s\n%s", - dt, credentialScope.c_str(), canonicalRequestHash.c_str() ); + formatstr(stringToSign, "AWS4-HMAC-SHA256\n%s\n%s\n%s", dt, + credentialScope.c_str(), canonicalRequestHash.c_str()); // // Creating task 3's inputs was done when we checked to see if we needed @@ -321,124 +333,147 @@ bool AmazonRequest::createV4Signature( const std::string & payload, // Task 3: calculate the signature. saKey = "AWS4" + saKey; - const unsigned char * hmac = HMAC( EVP_sha256(), saKey.c_str(), saKey.length(), - (unsigned char *)d, sizeof(d) - 1, - messageDigest, & mdLength ); - if( hmac == NULL ) { return false; } + const unsigned char *hmac = + HMAC(EVP_sha256(), saKey.c_str(), saKey.length(), (unsigned char *)d, + sizeof(d) - 1, messageDigest, &mdLength); + if (hmac == NULL) { + return false; + } unsigned int md2Length = 0; unsigned char messageDigest2[EVP_MAX_MD_SIZE]; - hmac = HMAC( EVP_sha256(), messageDigest, mdLength, - (const unsigned char *)r.c_str(), r.length(), messageDigest2, & md2Length ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest, mdLength, + (const unsigned char *)r.c_str(), r.length(), messageDigest2, + &md2Length); + if (hmac == NULL) { + return false; + } - hmac = HMAC( EVP_sha256(), messageDigest2, md2Length, - (const unsigned char *)s.c_str(), s.length(), messageDigest, & mdLength ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest2, md2Length, + (const unsigned char *)s.c_str(), s.length(), messageDigest, + &mdLength); + if (hmac == NULL) { + return false; + } const char c[] = "aws4_request"; - hmac = HMAC( EVP_sha256(), messageDigest, mdLength, - (const unsigned char *)c, sizeof(c) - 1, messageDigest2, & md2Length ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest, mdLength, (const unsigned char *)c, + sizeof(c) - 1, messageDigest2, &md2Length); + if (hmac == NULL) { + return false; + } - hmac = HMAC( EVP_sha256(), messageDigest2, md2Length, - (const unsigned char *)stringToSign.c_str(), stringToSign.length(), - messageDigest, & mdLength ); - if( hmac == NULL ) { return false; } + hmac = HMAC(EVP_sha256(), messageDigest2, md2Length, + (const unsigned char *)stringToSign.c_str(), + stringToSign.length(), messageDigest, &mdLength); + if (hmac == NULL) { + return false; + } std::string signature; - convertMessageDigestToLowercaseHex( messageDigest, mdLength, signature ); + convertMessageDigestToLowercaseHex(messageDigest, mdLength, signature); - formatstr( authorizationValue, "AWS4-HMAC-SHA256 Credential=%s/%s," - " SignedHeaders=%s, Signature=%s", - keyID.c_str(), credentialScope.c_str(), - signedHeaders.c_str(), signature.c_str() ); + formatstr(authorizationValue, + "AWS4-HMAC-SHA256 Credential=%s/%s," + " SignedHeaders=%s, Signature=%s", + keyID.c_str(), credentialScope.c_str(), signedHeaders.c_str(), + signature.c_str()); return true; } -bool AmazonRequest::sendV4Request( const std::string & payload, bool sendContentSHA ) { - if( (protocol != "http") && (protocol != "https") ) { - this->errorCode = "E_INVALID_SERVICE_URL"; - this->errorMessage = "Service URL not of a known protocol (http[s])."; - return false; - } - - if(! sendContentSHA) { - // dprintf( D_FULLDEBUG, "Payload is '%s'\n", payload.c_str() ); - } - - std::string authorizationValue; - if(! createV4Signature( payload, authorizationValue, sendContentSHA )) { - if( this->errorCode.empty() ) { this->errorCode = "E_INTERNAL"; } - if( this->errorMessage.empty() ) { this->errorMessage = "Failed to create v4 signature."; } - return false; - } - - // When accessing an unauthenticated bucket, providing an auth header will cause errors - if (!authorizationValue.empty()) { - headers[ "Authorization" ] = authorizationValue; - } - - return sendPreparedRequest( protocol, hostUrl, payload ); +bool AmazonRequest::sendV4Request(const std::string &payload, + bool sendContentSHA) { + if ((protocol != "http") && (protocol != "https")) { + this->errorCode = "E_INVALID_SERVICE_URL"; + this->errorMessage = "Service URL not of a known protocol (http[s])."; + return false; + } + + if (!sendContentSHA) { + // dprintf( D_FULLDEBUG, "Payload is '%s'\n", payload.c_str() ); + } + + std::string authorizationValue; + if (!createV4Signature(payload, authorizationValue, sendContentSHA)) { + if (this->errorCode.empty()) { + this->errorCode = "E_INTERNAL"; + } + if (this->errorMessage.empty()) { + this->errorMessage = "Failed to create v4 signature."; + } + return false; + } + + // When accessing an unauthenticated bucket, providing an auth header will + // cause errors + if (!authorizationValue.empty()) { + headers["Authorization"] = authorizationValue; + } + + return sendPreparedRequest(protocol, hostUrl, payload); } // It's stated in the API documentation that you can upload to any region // via us-east-1, which is moderately crazy. -bool AmazonRequest::SendS3Request( const std::string & payload ) { - headers[ "Content-Type" ] = "binary/octet-stream"; - std::string contentLength; formatstr( contentLength, "%zu", payload.size() ); - headers[ "Content-Length" ] = contentLength; +bool AmazonRequest::SendS3Request(const std::string &payload) { + headers["Content-Type"] = "binary/octet-stream"; + std::string contentLength; + formatstr(contentLength, "%zu", payload.size()); + headers["Content-Length"] = contentLength; // Another undocumented CURL feature: transfer-encoding is "chunked" // by default for "PUT", which we really don't want. - headers[ "Transfer-Encoding" ] = ""; + headers["Transfer-Encoding"] = ""; service = "s3"; - if( region.empty() ) { + if (region.empty()) { region = "us-east-1"; } - return sendV4Request( payload, true ); + return sendV4Request(payload, true); } // --------------------------------------------------------------------------- -AmazonS3Upload::~AmazonS3Upload() { } +AmazonS3Upload::~AmazonS3Upload() {} -bool AmazonS3Upload::SendRequest( const std::string & payload, off_t offset, size_t size ) { - if( offset != 0 || size != 0 ) { +bool AmazonS3Upload::SendRequest(const std::string &payload, off_t offset, + size_t size) { + if (offset != 0 || size != 0) { std::string range; - formatstr(range, "bytes=%lld-%lld", static_cast(offset), static_cast(offset + size - 1)); + formatstr(range, "bytes=%lld-%lld", static_cast(offset), + static_cast(offset + size - 1)); headers["Range"] = range.c_str(); } httpVerb = "PUT"; - return SendS3Request( payload ); + return SendS3Request(payload); } // --------------------------------------------------------------------------- -AmazonS3Download::~AmazonS3Download() { } +AmazonS3Download::~AmazonS3Download() {} -bool AmazonS3Download::SendRequest( off_t offset, size_t size ) { - if( offset != 0 || size != 0 ) { +bool AmazonS3Download::SendRequest(off_t offset, size_t size) { + if (offset != 0 || size != 0) { std::string range; - formatstr(range, "bytes=%lld-%lld",static_cast(offset), static_cast(offset + size - 1)); + formatstr(range, "bytes=%lld-%lld", static_cast(offset), + static_cast(offset + size - 1)); headers["Range"] = range.c_str(); this->expectedResponseCode = 206; } httpVerb = "GET"; std::string noPayloadAllowed; - return SendS3Request( noPayloadAllowed ); + return SendS3Request(noPayloadAllowed); } // --------------------------------------------------------------------------- -AmazonS3Head::~AmazonS3Head() { } +AmazonS3Head::~AmazonS3Head() {} bool AmazonS3Head::SendRequest() { httpVerb = "HEAD"; includeResponseHeader = true; std::string noPayloadAllowed; - return SendS3Request( noPayloadAllowed ); + return SendS3Request(noPayloadAllowed); } // --------------------------------------------------------------------------- diff --git a/src/S3Commands.hh b/src/S3Commands.hh index 49a2aba..dee72ee 100644 --- a/src/S3Commands.hh +++ b/src/S3Commands.hh @@ -23,141 +23,127 @@ #include class AmazonRequest : public HTTPRequest { -public: - AmazonRequest( - const std::string & s, - const std::string & akf, - const std::string & skf, - const std::string & b, - const std::string & o, - const std::string & style, - int sv, - XrdSysError &log - ) : - HTTPRequest(s, log), - accessKeyFile(akf), - secretKeyFile(skf), - signatureVersion(sv), - bucket(b), - object(o), - style(style) - { - requiresSignature = true; - // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header). - // For example, if the incoming hostUrl (which we get from config) is "https://my-url.com:443", the bucket is "my-bucket", and - // the object is "my-object", then the host will be "my-bucket.my-url.com:443" and the canonicalURI will be "/my-object". - if (! parseURL(hostUrl, canonicalURI)) { - errorCode = "E_INVALID_SERVICE_URL"; - errorMessage = "Failed to parse host and canonicalURI from service URL."; - } - - if( canonicalURI.empty() ) { canonicalURI = "/"; } - - // Now that we have the host and canonicalURI, we can build the actual url we perform the curl against. - // Using the previous example, we'd get a new hostUrl of - // --> "https://my-bucket.my-url.com:443/my-object" for virtual style requests, and - // --> "https://my-url.com:443/my-bucket/my-object" for path style requests. - hostUrl = protocol + "://" + host + canonicalURI; - - // If we can, set the region based on the host. - size_t secondDot = host.find( ".", 2 + 1 ); - if( host.find( "s3." ) == 0 ) { - region = host.substr( 3, secondDot - 2 - 1 ); - } - } - virtual ~AmazonRequest(); - - virtual const std::string* getAccessKey() const { - return &accessKeyFile; } - virtual const std::string* getSecretKey() const { - return &secretKeyFile; } - - bool parseURL( const std::string & url, - std::string & path ); - - virtual bool SendRequest(); - virtual bool SendS3Request( const std::string & payload ); - -protected: - bool sendV4Request( const std::string & payload, bool sendContentSHA = false ); - - std::string accessKeyFile; - std::string secretKeyFile; - - int signatureVersion; - - std::string host; - std::string canonicalURI; - - std::string bucket; - std::string object; - - std::string region; - std::string service; - - std::string style; -private: - bool createV4Signature( const std::string & payload, std::string & authorizationHeader, bool sendContentSHA = false ); - - std::string canonicalizeQueryString(); + public: + AmazonRequest(const std::string &s, const std::string &akf, + const std::string &skf, const std::string &b, + const std::string &o, const std::string &style, int sv, + XrdSysError &log) + : HTTPRequest(s, log), accessKeyFile(akf), secretKeyFile(skf), + signatureVersion(sv), bucket(b), object(o), style(style) { + requiresSignature = true; + // Start off by parsing the hostUrl, which we use in conjunction with + // the bucket to fill in the host (for setting host header). For + // example, if the incoming hostUrl (which we get from config) is + // "https://my-url.com:443", the bucket is "my-bucket", and the object + // is "my-object", then the host will be "my-bucket.my-url.com:443" and + // the canonicalURI will be "/my-object". + if (!parseURL(hostUrl, canonicalURI)) { + errorCode = "E_INVALID_SERVICE_URL"; + errorMessage = + "Failed to parse host and canonicalURI from service URL."; + } + + if (canonicalURI.empty()) { + canonicalURI = "/"; + } + + // Now that we have the host and canonicalURI, we can build the actual + // url we perform the curl against. Using the previous example, we'd get + // a new hostUrl of + // --> "https://my-bucket.my-url.com:443/my-object" for virtual style + // requests, and + // --> "https://my-url.com:443/my-bucket/my-object" for path style + // requests. + hostUrl = protocol + "://" + host + canonicalURI; + + // If we can, set the region based on the host. + size_t secondDot = host.find(".", 2 + 1); + if (host.find("s3.") == 0) { + region = host.substr(3, secondDot - 2 - 1); + } + } + virtual ~AmazonRequest(); + + virtual const std::string *getAccessKey() const { return &accessKeyFile; } + virtual const std::string *getSecretKey() const { return &secretKeyFile; } + + bool parseURL(const std::string &url, std::string &path); + + virtual bool SendRequest(); + virtual bool SendS3Request(const std::string &payload); + + protected: + bool sendV4Request(const std::string &payload, bool sendContentSHA = false); + + std::string accessKeyFile; + std::string secretKeyFile; + + int signatureVersion; + + std::string host; + std::string canonicalURI; + + std::string bucket; + std::string object; + + std::string region; + std::string service; + + std::string style; + + private: + bool createV4Signature(const std::string &payload, + std::string &authorizationHeader, + bool sendContentSHA = false); + + std::string canonicalizeQueryString(); }; class AmazonS3Upload : public AmazonRequest { - using AmazonRequest::SendRequest; -public: - AmazonS3Upload( - const std::string & s, - const std::string & akf, - const std::string & skf, - const std::string & b, - const std::string & o, - const std::string & style, - XrdSysError &log - ) : - AmazonRequest(s, akf, skf, b, o, style, 4, log) {} - - virtual ~AmazonS3Upload(); - - virtual bool SendRequest( const std::string & payload, off_t offset, size_t size ); - -protected: - std::string path; + using AmazonRequest::SendRequest; + + public: + AmazonS3Upload(const std::string &s, const std::string &akf, + const std::string &skf, const std::string &b, + const std::string &o, const std::string &style, + XrdSysError &log) + : AmazonRequest(s, akf, skf, b, o, style, 4, log) {} + + virtual ~AmazonS3Upload(); + + virtual bool SendRequest(const std::string &payload, off_t offset, + size_t size); + + protected: + std::string path; }; class AmazonS3Download : public AmazonRequest { - using AmazonRequest::SendRequest; -public: - AmazonS3Download( - const std::string & s, - const std::string & akf, - const std::string & skf, - const std::string & b, - const std::string & o, - const std::string & style, - XrdSysError &log - ) : - AmazonRequest(s, akf, skf, b, o, style, 4, log){} - - virtual ~AmazonS3Download(); - - virtual bool SendRequest( off_t offset, size_t size ); + using AmazonRequest::SendRequest; + + public: + AmazonS3Download(const std::string &s, const std::string &akf, + const std::string &skf, const std::string &b, + const std::string &o, const std::string &style, + XrdSysError &log) + : AmazonRequest(s, akf, skf, b, o, style, 4, log) {} + + virtual ~AmazonS3Download(); + + virtual bool SendRequest(off_t offset, size_t size); }; class AmazonS3Head : public AmazonRequest { - using AmazonRequest::SendRequest; -public: - AmazonS3Head( - const std::string & s, - const std::string & akf, - const std::string & skf, - const std::string & b, - const std::string & o, - const std::string & style, - XrdSysError &log - ) : - AmazonRequest(s, akf, skf, b, o, style, 4, log){} - - virtual ~AmazonS3Head(); - - virtual bool SendRequest(); + using AmazonRequest::SendRequest; + + public: + AmazonS3Head(const std::string &s, const std::string &akf, + const std::string &skf, const std::string &b, + const std::string &o, const std::string &style, + XrdSysError &log) + : AmazonRequest(s, akf, skf, b, o, style, 4, log) {} + + virtual ~AmazonS3Head(); + + virtual bool SendRequest(); }; diff --git a/src/S3Directory.hh b/src/S3Directory.hh index f99b9c4..3287eda 100644 --- a/src/S3Directory.hh +++ b/src/S3Directory.hh @@ -25,34 +25,21 @@ // behaviors for either HTTP or S3 variants in the future. class S3Directory : public HTTPDirectory { -public: - S3Directory(XrdSysError &log) : - HTTPDirectory(log) - // m_log(log) - { - } - - virtual ~S3Directory() {} - - virtual int - Opendir(const char *path, - XrdOucEnv &env) override - { - return -ENOSYS; - } - - int Readdir(char *buff, int blen) override - { - return -ENOSYS; - } - - int StatRet(struct stat *statStruct) override - { - return -ENOSYS; - } - - int Close(long long *retsz=0) override - { - return -ENOSYS; - } + public: + S3Directory(XrdSysError &log) + : HTTPDirectory(log) + // m_log(log) + {} + + virtual ~S3Directory() {} + + virtual int Opendir(const char *path, XrdOucEnv &env) override { + return -ENOSYS; + } + + int Readdir(char *buff, int blen) override { return -ENOSYS; } + + int StatRet(struct stat *statStruct) override { return -ENOSYS; } + + int Close(long long *retsz = 0) override { return -ENOSYS; } }; diff --git a/src/S3File.cc b/src/S3File.cc index c2f1217..8415c49 100644 --- a/src/S3File.cc +++ b/src/S3File.cc @@ -16,9 +16,9 @@ * ***************************************************************/ +#include "S3File.hh" #include "S3Commands.hh" #include "S3FileSystem.hh" -#include "S3File.hh" #include "logging.hh" #include "stl_string_utils.hh" @@ -41,285 +41,241 @@ using namespace XrdHTTPServer; -S3FileSystem* g_s3_oss = nullptr; +S3FileSystem *g_s3_oss = nullptr; XrdVERSIONINFO(XrdOssGetFileSystem, S3); -S3File::S3File(XrdSysError &log, S3FileSystem *oss) : - m_log(log), - m_oss(oss), - content_length(0), - last_modified(0) -{} - - -int -parse_path( const S3FileSystem & fs, const char * fullPath, std::string & exposedPath, std::string & object ) { - // - // Check the path for validity. - // - std::filesystem::path p(fullPath); - auto pathComponents = p.begin(); - - // Iterate through components of the fullPath until we either find a match - // or we've reached the end of the path. - std::filesystem::path currentPath = *pathComponents; - while (pathComponents != p.end()) { - if (fs.exposedPathExists(currentPath.string())) { - exposedPath = currentPath.string(); - break; - } - ++pathComponents; - if (pathComponents != p.end()) { - currentPath /= *pathComponents; - } else { - return -ENOENT; - } - } - - // Objects names may contain path separators. - ++pathComponents; - if( pathComponents == p.end() ) { return -ENOENT; } - - std::filesystem::path objectPath = *pathComponents++; - for( ; pathComponents != p.end(); ++pathComponents ) { - objectPath /= (* pathComponents); - } - object = objectPath.string(); - - fprintf( stderr, "object = %s\n", object.c_str() ); - - return 0; +S3File::S3File(XrdSysError &log, S3FileSystem *oss) + : m_log(log), m_oss(oss), content_length(0), last_modified(0) {} + +int parse_path(const S3FileSystem &fs, const char *fullPath, + std::string &exposedPath, std::string &object) { + // + // Check the path for validity. + // + std::filesystem::path p(fullPath); + auto pathComponents = p.begin(); + + // Iterate through components of the fullPath until we either find a match + // or we've reached the end of the path. + std::filesystem::path currentPath = *pathComponents; + while (pathComponents != p.end()) { + if (fs.exposedPathExists(currentPath.string())) { + exposedPath = currentPath.string(); + break; + } + ++pathComponents; + if (pathComponents != p.end()) { + currentPath /= *pathComponents; + } else { + return -ENOENT; + } + } + + // Objects names may contain path separators. + ++pathComponents; + if (pathComponents == p.end()) { + return -ENOENT; + } + + std::filesystem::path objectPath = *pathComponents++; + for (; pathComponents != p.end(); ++pathComponents) { + objectPath /= (*pathComponents); + } + object = objectPath.string(); + + fprintf(stderr, "object = %s\n", object.c_str()); + + return 0; } - -int -S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) -{ - std::string exposedPath, object; - int rv = parse_path( * m_oss, path, exposedPath, object ); - if( rv != 0 ) { return rv; } - if(!m_oss->exposedPathExists(exposedPath)) return -ENOENT; - - std::string configured_s3_region = m_oss->getS3Region(exposedPath); - std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath); - std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath); - std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath); - std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath); - - // We used to query S3 here to see if the object existed, but of course - // if you're creating a file on upload, you don't care. - - this->s3_object_name = object; - this->s3_bucket_name = configured_s3_bucket_name; - this->s3_service_url = configured_s3_service_url; - this->s3_access_key = configured_s3_access_key; - this->s3_secret_key = configured_s3_secret_key; - std::string configured_s3_url_style = m_oss->getS3URLStyle(); - - - // We used to query S3 here to see if the object existed, but of course - // if you're creating a file on upload, you don't care. - - this->s3_object_name = object; - this->s3_bucket_name = configured_s3_bucket_name; - this->s3_service_url = configured_s3_service_url; - this->s3_access_key = configured_s3_access_key; - this->s3_secret_key = configured_s3_secret_key; - this->s3_url_style = configured_s3_url_style; - return 0; +int S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) { + std::string exposedPath, object; + int rv = parse_path(*m_oss, path, exposedPath, object); + if (rv != 0) { + return rv; + } + if (!m_oss->exposedPathExists(exposedPath)) + return -ENOENT; + + std::string configured_s3_region = m_oss->getS3Region(exposedPath); + std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath); + std::string configured_s3_access_key = + m_oss->getS3AccessKeyFile(exposedPath); + std::string configured_s3_secret_key = + m_oss->getS3SecretKeyFile(exposedPath); + std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath); + + // We used to query S3 here to see if the object existed, but of course + // if you're creating a file on upload, you don't care. + + this->s3_object_name = object; + this->s3_bucket_name = configured_s3_bucket_name; + this->s3_service_url = configured_s3_service_url; + this->s3_access_key = configured_s3_access_key; + this->s3_secret_key = configured_s3_secret_key; + std::string configured_s3_url_style = m_oss->getS3URLStyle(); + + // We used to query S3 here to see if the object existed, but of course + // if you're creating a file on upload, you don't care. + + this->s3_object_name = object; + this->s3_bucket_name = configured_s3_bucket_name; + this->s3_service_url = configured_s3_service_url; + this->s3_access_key = configured_s3_access_key; + this->s3_secret_key = configured_s3_secret_key; + this->s3_url_style = configured_s3_url_style; + return 0; } - -ssize_t -S3File::Read(void *buffer, off_t offset, size_t size) -{ - AmazonS3Download download( - this->s3_service_url, - this->s3_access_key, - this->s3_secret_key, - this->s3_bucket_name, - this->s3_object_name, - this->s3_url_style, - m_log - ); - - if(! download.SendRequest( offset, size ) ) { - std::stringstream ss; - ss << "Failed to send GetObject command: " << download.getResponseCode() << "'" << download.getResultString() << "'"; - m_log.Log(LogMask::Warning, "S3File::Read", ss.str().c_str()); - return 0; - } - - const std::string & bytes = download.getResultString(); - memcpy( buffer, bytes.data(), bytes.size() ); - return bytes.size(); +ssize_t S3File::Read(void *buffer, off_t offset, size_t size) { + AmazonS3Download download(this->s3_service_url, this->s3_access_key, + this->s3_secret_key, this->s3_bucket_name, + this->s3_object_name, this->s3_url_style, m_log); + + if (!download.SendRequest(offset, size)) { + std::stringstream ss; + ss << "Failed to send GetObject command: " << download.getResponseCode() + << "'" << download.getResultString() << "'"; + m_log.Log(LogMask::Warning, "S3File::Read", ss.str().c_str()); + return 0; + } + + const std::string &bytes = download.getResultString(); + memcpy(buffer, bytes.data(), bytes.size()); + return bytes.size(); } - -int -S3File::Fstat(struct stat *buff) -{ - AmazonS3Head head( - this->s3_service_url, - this->s3_access_key, - this->s3_secret_key, - this->s3_bucket_name, - this->s3_object_name, - this->s3_url_style, - m_log - ); - - if(! head.SendRequest()) { - // SendRequest() returns false for all errors, including ones - // where the server properly responded with something other - // than code 200. If xrootd wants us to distinguish between - // these cases, head.getResponseCode() is initialized to 0, so - // we can check. - std::stringstream ss; - ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" << head.getResultString() << "'"; - m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str()); - return -ENOENT; - } - - - std::string headers = head.getResultString(); - - std::string line; - size_t current_newline = 0; - size_t next_newline = std::string::npos; - size_t last_character = headers.size(); - while( current_newline != std::string::npos && current_newline != last_character - 1 ) { - next_newline = headers.find( "\r\n", current_newline + 2); - line = substring( headers, current_newline + 2, next_newline ); - - size_t colon = line.find(":"); - if( colon != std::string::npos && colon != line.size() ) { - std::string attr = substring( line, 0, colon ); - std::string value = substring( line, colon + 1 ); - trim(value); - toLower(attr); - - if( attr == "content-length" ) { - this->content_length = std::stol(value); - } else if( attr == "last-modified" ) { - struct tm t; - char * eos = strptime( value.c_str(), - "%a, %d %b %Y %T %Z", - & t ); - if( eos == & value.c_str()[value.size()] ) { - time_t epoch = timegm(& t); - if( epoch != -1 ) { - this->last_modified = epoch; - } - } - } - } - - current_newline = next_newline; - } - - - buff->st_mode = 0600 | S_IFREG; - buff->st_nlink = 1; - buff->st_uid = 1; - buff->st_gid = 1; - buff->st_size = this->content_length; - buff->st_mtime = this->last_modified; - buff->st_atime = 0; - buff->st_ctime = 0; - buff->st_dev = 0; - buff->st_ino = 0; - - return 0; +int S3File::Fstat(struct stat *buff) { + AmazonS3Head head(this->s3_service_url, this->s3_access_key, + this->s3_secret_key, this->s3_bucket_name, + this->s3_object_name, this->s3_url_style, m_log); + + if (!head.SendRequest()) { + // SendRequest() returns false for all errors, including ones + // where the server properly responded with something other + // than code 200. If xrootd wants us to distinguish between + // these cases, head.getResponseCode() is initialized to 0, so + // we can check. + std::stringstream ss; + ss << "Failed to send HeadObject command: " << head.getResponseCode() + << "'" << head.getResultString() << "'"; + m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str()); + return -ENOENT; + } + + std::string headers = head.getResultString(); + + std::string line; + size_t current_newline = 0; + size_t next_newline = std::string::npos; + size_t last_character = headers.size(); + while (current_newline != std::string::npos && + current_newline != last_character - 1) { + next_newline = headers.find("\r\n", current_newline + 2); + line = substring(headers, current_newline + 2, next_newline); + + size_t colon = line.find(":"); + if (colon != std::string::npos && colon != line.size()) { + std::string attr = substring(line, 0, colon); + std::string value = substring(line, colon + 1); + trim(value); + toLower(attr); + + if (attr == "content-length") { + this->content_length = std::stol(value); + } else if (attr == "last-modified") { + struct tm t; + char *eos = strptime(value.c_str(), "%a, %d %b %Y %T %Z", &t); + if (eos == &value.c_str()[value.size()]) { + time_t epoch = timegm(&t); + if (epoch != -1) { + this->last_modified = epoch; + } + } + } + } + + current_newline = next_newline; + } + + buff->st_mode = 0600 | S_IFREG; + buff->st_nlink = 1; + buff->st_uid = 1; + buff->st_gid = 1; + buff->st_size = this->content_length; + buff->st_mtime = this->last_modified; + buff->st_atime = 0; + buff->st_ctime = 0; + buff->st_dev = 0; + buff->st_ino = 0; + + return 0; } - -ssize_t -S3File::Write(const void *buffer, off_t offset, size_t size) -{ - AmazonS3Upload upload( - this->s3_service_url, - this->s3_access_key, - this->s3_secret_key, - this->s3_bucket_name, - this->s3_object_name, - this->s3_url_style, - m_log - ); - - std::string payload( (char *)buffer, size ); - if(! upload.SendRequest( payload, offset, size )) { - m_log.Emsg( "Open", "upload.SendRequest() failed" ); - return -ENOENT; - } else { - m_log.Emsg( "Open", "upload.SendRequest() succeeded" ); - return 0; - } +ssize_t S3File::Write(const void *buffer, off_t offset, size_t size) { + AmazonS3Upload upload(this->s3_service_url, this->s3_access_key, + this->s3_secret_key, this->s3_bucket_name, + this->s3_object_name, this->s3_url_style, m_log); + + std::string payload((char *)buffer, size); + if (!upload.SendRequest(payload, offset, size)) { + m_log.Emsg("Open", "upload.SendRequest() failed"); + return -ENOENT; + } else { + m_log.Emsg("Open", "upload.SendRequest() succeeded"); + return 0; + } } - -int S3File::Close(long long *retsz) -{ - m_log.Emsg("Close", "Closed our S3 file"); - return 0; +int S3File::Close(long long *retsz) { + m_log.Emsg("Close", "Closed our S3 file"); + return 0; } - extern "C" { /* - This function is called when we are wrapping something. + This function is called when we are wrapping something. */ -XrdOss *XrdOssAddStorageSystem2(XrdOss *curr_oss, - XrdSysLogger *Logger, - const char *config_fn, - const char *parms, - XrdOucEnv *envP) -{ - XrdSysError log(Logger, "s3_"); - - log.Emsg("Initialize", "S3 filesystem cannot be stacked with other filesystems"); - return nullptr; +XrdOss *XrdOssAddStorageSystem2(XrdOss *curr_oss, XrdSysLogger *Logger, + const char *config_fn, const char *parms, + XrdOucEnv *envP) { + XrdSysError log(Logger, "s3_"); + + log.Emsg("Initialize", + "S3 filesystem cannot be stacked with other filesystems"); + return nullptr; } - /* - This function is called when it is the top level file system and we are not - wrapping anything + This function is called when it is the top level file system and we are not + wrapping anything */ -XrdOss *XrdOssGetStorageSystem2(XrdOss *native_oss, - XrdSysLogger *Logger, - const char *config_fn, - const char *parms, - XrdOucEnv *envP) -{ - XrdSysError log(Logger, "s3_"); - - envP->Export("XRDXROOTD_NOPOSC", "1"); - - try { - g_s3_oss = new S3FileSystem(Logger, config_fn, envP); - return g_s3_oss; - } catch (std::runtime_error &re) { - log.Emsg("Initialize", "Encountered a runtime failure", re.what()); - return nullptr; - } +XrdOss *XrdOssGetStorageSystem2(XrdOss *native_oss, XrdSysLogger *Logger, + const char *config_fn, const char *parms, + XrdOucEnv *envP) { + XrdSysError log(Logger, "s3_"); + + envP->Export("XRDXROOTD_NOPOSC", "1"); + + try { + g_s3_oss = new S3FileSystem(Logger, config_fn, envP); + return g_s3_oss; + } catch (std::runtime_error &re) { + log.Emsg("Initialize", "Encountered a runtime failure", re.what()); + return nullptr; + } } - -XrdOss *XrdOssGetStorageSystem(XrdOss *native_oss, - XrdSysLogger *Logger, - const char *config_fn, - const char *parms) -{ - return XrdOssGetStorageSystem2(native_oss, Logger, config_fn, parms, nullptr); +XrdOss *XrdOssGetStorageSystem(XrdOss *native_oss, XrdSysLogger *Logger, + const char *config_fn, const char *parms) { + return XrdOssGetStorageSystem2(native_oss, Logger, config_fn, parms, + nullptr); } - } // end extern "C" - -XrdVERSIONINFO(XrdOssGetStorageSystem, s3); +XrdVERSIONINFO(XrdOssGetStorageSystem, s3); XrdVERSIONINFO(XrdOssGetStorageSystem2, s3); XrdVERSIONINFO(XrdOssAddStorageSystem2, s3); diff --git a/src/S3File.hh b/src/S3File.hh index d59c6ad..a3e12f3 100644 --- a/src/S3File.hh +++ b/src/S3File.hh @@ -20,132 +20,89 @@ #include "S3FileSystem.hh" -#include #include +#include #include #include #include #include -int parse_path( const S3FileSystem & fs, const char * path, std::string & exposedPath, std::string & object ); +int parse_path(const S3FileSystem &fs, const char *path, + std::string &exposedPath, std::string &object); class S3File : public XrdOssDF { -public: - S3File(XrdSysError &log, S3FileSystem *oss); - - virtual ~S3File() {} - - int Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) override; - - int Fchmod(mode_t mode) override - { - return -ENOSYS; - } - - void Flush() override - { - } - - int Fstat(struct stat *buf) override; - - int Fsync() override - { - return -ENOSYS; - } - - int Fsync(XrdSfsAio *aiop) override - { - return -ENOSYS; - } - - int Ftruncate(unsigned long long size) override - { - return -ENOSYS; - } - - off_t getMmap(void **addr) override - { - return 0; - } - - int isCompressed(char *cxidp=0) override - { - return -ENOSYS; - } - - ssize_t pgRead (void* buffer, off_t offset, size_t rdlen, - uint32_t* csvec, uint64_t opts) override - { - return -ENOSYS; - } - - int pgRead (XrdSfsAio* aioparm, uint64_t opts) override - { - return -ENOSYS; - } - - ssize_t pgWrite(void* buffer, off_t offset, size_t wrlen, - uint32_t* csvec, uint64_t opts) override - { - return -ENOSYS; - } - - int pgWrite(XrdSfsAio* aioparm, uint64_t opts) override - { - return -ENOSYS; - } - - ssize_t Read(off_t offset, size_t size) override - { - return -ENOSYS; - } - - ssize_t Read(void *buffer, off_t offset, size_t size) override; - - int Read(XrdSfsAio *aiop) override - { - return -ENOSYS; - } - - ssize_t ReadRaw(void *buffer, off_t offset, size_t size) override - { - return -ENOSYS; - } - - ssize_t ReadV(XrdOucIOVec *readV, int rdvcnt) override - { - return -ENOSYS; - } - - ssize_t Write(const void *buffer, off_t offset, size_t size) override; - - int Write(XrdSfsAio *aiop) override - { - return -ENOSYS; - } - - ssize_t WriteV(XrdOucIOVec *writeV, int wrvcnt) override - { - return -ENOSYS; - } - - int Close(long long *retsz=0) override; - - size_t getContentLength() { return content_length; } - time_t getLastModified() { return last_modified; } - -private: - XrdSysError &m_log; - S3FileSystem *m_oss; - - std::string s3_service_url; - std::string s3_bucket_name; - std::string s3_object_name; - std::string s3_access_key; - std::string s3_secret_key; - std::string s3_url_style; - - size_t content_length; - time_t last_modified; + public: + S3File(XrdSysError &log, S3FileSystem *oss); + + virtual ~S3File() {} + + int Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) override; + + int Fchmod(mode_t mode) override { return -ENOSYS; } + + void Flush() override {} + + int Fstat(struct stat *buf) override; + + int Fsync() override { return -ENOSYS; } + + int Fsync(XrdSfsAio *aiop) override { return -ENOSYS; } + + int Ftruncate(unsigned long long size) override { return -ENOSYS; } + + off_t getMmap(void **addr) override { return 0; } + + int isCompressed(char *cxidp = 0) override { return -ENOSYS; } + + ssize_t pgRead(void *buffer, off_t offset, size_t rdlen, uint32_t *csvec, + uint64_t opts) override { + return -ENOSYS; + } + + int pgRead(XrdSfsAio *aioparm, uint64_t opts) override { return -ENOSYS; } + + ssize_t pgWrite(void *buffer, off_t offset, size_t wrlen, uint32_t *csvec, + uint64_t opts) override { + return -ENOSYS; + } + + int pgWrite(XrdSfsAio *aioparm, uint64_t opts) override { return -ENOSYS; } + + ssize_t Read(off_t offset, size_t size) override { return -ENOSYS; } + + ssize_t Read(void *buffer, off_t offset, size_t size) override; + + int Read(XrdSfsAio *aiop) override { return -ENOSYS; } + + ssize_t ReadRaw(void *buffer, off_t offset, size_t size) override { + return -ENOSYS; + } + + ssize_t ReadV(XrdOucIOVec *readV, int rdvcnt) override { return -ENOSYS; } + + ssize_t Write(const void *buffer, off_t offset, size_t size) override; + + int Write(XrdSfsAio *aiop) override { return -ENOSYS; } + + ssize_t WriteV(XrdOucIOVec *writeV, int wrvcnt) override { return -ENOSYS; } + + int Close(long long *retsz = 0) override; + + size_t getContentLength() { return content_length; } + time_t getLastModified() { return last_modified; } + + private: + XrdSysError &m_log; + S3FileSystem *m_oss; + + std::string s3_service_url; + std::string s3_bucket_name; + std::string s3_object_name; + std::string s3_access_key; + std::string s3_secret_key; + std::string s3_url_style; + + size_t content_length; + time_t last_modified; }; diff --git a/src/S3FileSystem.cc b/src/S3FileSystem.cc index 6e98cce..3a4e939 100644 --- a/src/S3FileSystem.cc +++ b/src/S3FileSystem.cc @@ -16,8 +16,6 @@ * ***************************************************************/ -#include "S3Directory.hh" -#include "S3File.hh" #include "S3FileSystem.hh" #include "S3AccessInfo.hh" #include "S3Directory.hh" @@ -34,183 +32,198 @@ #include #include -#include #include +#include #include -S3FileSystem::S3FileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP) : - m_env(envP), - m_log(lp, "s3_") -{ - m_log.Say("------ Initializing the S3 filesystem plugin."); - if (!Config(lp, configfn)) { - throw std::runtime_error("Failed to configure S3 filesystem plugin."); - } -} - - -S3FileSystem::~S3FileSystem() { +S3FileSystem::S3FileSystem(XrdSysLogger *lp, const char *configfn, + XrdOucEnv *envP) + : m_env(envP), m_log(lp, "s3_") { + m_log.Say("------ Initializing the S3 filesystem plugin."); + if (!Config(lp, configfn)) { + throw std::runtime_error("Failed to configure S3 filesystem plugin."); + } } - -bool -S3FileSystem::handle_required_config( - const char * desired_name, - const std::string & source) { - if( source.empty() ) { - std::string error; - formatstr( error, "%s must specify a value", desired_name ); - m_log.Emsg( "Config", error.c_str() ); - return false; - } - return true; +S3FileSystem::~S3FileSystem() {} + +bool S3FileSystem::handle_required_config(const char *desired_name, + const std::string &source) { + if (source.empty()) { + std::string error; + formatstr(error, "%s must specify a value", desired_name); + m_log.Emsg("Config", error.c_str()); + return false; + } + return true; } - -bool -S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) -{ - XrdOucEnv myEnv; - XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); - - int cfgFD = open(configfn, O_RDONLY, 0); - if (cfgFD < 0) { - m_log.Emsg("Config", errno, "open config file", configfn); - return false; - } - - char * temporary; - std::string value; - std::string attribute; - Config.Attach(cfgFD); - S3AccessInfo *newAccessInfo = new S3AccessInfo(); - std::string exposedPath; - while ((temporary = Config.GetMyFirstWord())) { - attribute = temporary; - temporary = Config.GetWord(); - if(attribute == "s3.end") { - s3_access_map[exposedPath] = newAccessInfo; - if(newAccessInfo->getS3ServiceName().empty()) { - m_log.Emsg("Config", "s3.service_name not specified"); - return false; - } - if(newAccessInfo->getS3Region().empty()) { - m_log.Emsg("Config", "s3.region not specified"); - return false; - } - newAccessInfo = new S3AccessInfo(); - exposedPath = ""; - continue; - } - if(! temporary) { continue; } - value = temporary; - - if(!handle_required_config("s3.path_name",value) ) { Config.Close(); return false; } - if(!handle_required_config("s3.bucket_name",value) ) { Config.Close(); return false; } - if(!handle_required_config("s3.service_name",value) ) { Config.Close(); return false; } - if(!handle_required_config("s3.region", value ) ) { Config.Close(); return false; } - if(!handle_required_config("s3.service_url", value) ) { Config.Close(); return false; } - if(!handle_required_config("s3.access_key_file", value) ) { Config.Close(); return false; } - if(!handle_required_config("s3.secret_key_file", value) ) { Config.Close(); return false; } - if(!handle_required_config("s3.url_style", value) ) { Config.Close(); return false; } - - if (attribute == "s3.path_name") { - // Normalize paths so that they all start with / - if (value[0] != '/') { - exposedPath = "/" + value; - } else { - exposedPath = value; - } - } - else if(attribute == "s3.bucket_name") newAccessInfo->setS3BucketName(value); - else if(attribute == "s3.service_name") newAccessInfo->setS3ServiceName(value); - else if(attribute == "s3.region") newAccessInfo->setS3Region(value); - else if(attribute == "s3.access_key_file") newAccessInfo->setS3AccessKeyFile(value); - else if(attribute == "s3.secret_key_file") newAccessInfo->setS3SecretKeyFile(value); - else if(attribute == "s3.service_url") newAccessInfo->setS3ServiceUrl(value); - else if(attribute == "s3.url_style") this->s3_url_style = value; - - } - - if( this->s3_url_style.empty() ) { - m_log.Emsg("Config", "s3.url_style not specified"); - return false; - } else { - // We want this to be case-insensitive. - toLower( this->s3_url_style ); - } - if( this->s3_url_style != "virtual" && this->s3_url_style != "path" ) { - m_log.Emsg("Config", "invalid s3.url_style specified. Must be 'virtual' or 'path'"); - return false; - } - - int retc = Config.LastError(); - if( retc ) { - m_log.Emsg("Config", -retc, "read config file", configfn); - Config.Close(); - return false; - } - - Config.Close(); - return true; +bool S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) { + XrdOucEnv myEnv; + XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); + + int cfgFD = open(configfn, O_RDONLY, 0); + if (cfgFD < 0) { + m_log.Emsg("Config", errno, "open config file", configfn); + return false; + } + + char *temporary; + std::string value; + std::string attribute; + Config.Attach(cfgFD); + S3AccessInfo *newAccessInfo = new S3AccessInfo(); + std::string exposedPath; + while ((temporary = Config.GetMyFirstWord())) { + attribute = temporary; + temporary = Config.GetWord(); + if (attribute == "s3.end") { + s3_access_map[exposedPath] = newAccessInfo; + if (newAccessInfo->getS3ServiceName().empty()) { + m_log.Emsg("Config", "s3.service_name not specified"); + return false; + } + if (newAccessInfo->getS3Region().empty()) { + m_log.Emsg("Config", "s3.region not specified"); + return false; + } + newAccessInfo = new S3AccessInfo(); + exposedPath = ""; + continue; + } + if (!temporary) { + continue; + } + value = temporary; + + if (!handle_required_config("s3.path_name", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.bucket_name", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.service_name", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.region", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.service_url", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.access_key_file", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.secret_key_file", value)) { + Config.Close(); + return false; + } + if (!handle_required_config("s3.url_style", value)) { + Config.Close(); + return false; + } + + if (attribute == "s3.path_name") { + // Normalize paths so that they all start with / + if (value[0] != '/') { + exposedPath = "/" + value; + } else { + exposedPath = value; + } + } else if (attribute == "s3.bucket_name") + newAccessInfo->setS3BucketName(value); + else if (attribute == "s3.service_name") + newAccessInfo->setS3ServiceName(value); + else if (attribute == "s3.region") + newAccessInfo->setS3Region(value); + else if (attribute == "s3.access_key_file") + newAccessInfo->setS3AccessKeyFile(value); + else if (attribute == "s3.secret_key_file") + newAccessInfo->setS3SecretKeyFile(value); + else if (attribute == "s3.service_url") + newAccessInfo->setS3ServiceUrl(value); + else if (attribute == "s3.url_style") + this->s3_url_style = value; + } + + if (this->s3_url_style.empty()) { + m_log.Emsg("Config", "s3.url_style not specified"); + return false; + } else { + // We want this to be case-insensitive. + toLower(this->s3_url_style); + } + if (this->s3_url_style != "virtual" && this->s3_url_style != "path") { + m_log.Emsg( + "Config", + "invalid s3.url_style specified. Must be 'virtual' or 'path'"); + return false; + } + + int retc = Config.LastError(); + if (retc) { + m_log.Emsg("Config", -retc, "read config file", configfn); + Config.Close(); + return false; + } + + Config.Close(); + return true; } - // Object Allocation Functions // -XrdOssDF * -S3FileSystem::newDir(const char *user) -{ - return new S3Directory(m_log); +XrdOssDF *S3FileSystem::newDir(const char *user) { + return new S3Directory(m_log); } - -XrdOssDF * -S3FileSystem::newFile(const char *user) -{ - return new S3File(m_log, this); +XrdOssDF *S3FileSystem::newFile(const char *user) { + return new S3File(m_log, this); } - -int -S3FileSystem::Stat(const char *path, struct stat *buff, - int opts, XrdOucEnv *env) -{ - std::string error; - - m_log.Emsg("Stat", "Stat'ing path", path); - - S3File s3file(m_log, this); - int rv = s3file.Open( path, 0, (mode_t)0, *env ); - if (rv) { - m_log.Emsg("Stat", "Failed to open path:", path); - } - // Assume that S3File::FStat() doesn't write to buff unless it succeeds. - rv = s3file.Fstat( buff ); - if( rv != 0 ) { - formatstr( error, "File %s not found.", path ); - m_log.Emsg( "Stat", error.c_str() ); - return -ENOENT; - } - - return 0; +int S3FileSystem::Stat(const char *path, struct stat *buff, int opts, + XrdOucEnv *env) { + std::string error; + + m_log.Emsg("Stat", "Stat'ing path", path); + + S3File s3file(m_log, this); + int rv = s3file.Open(path, 0, (mode_t)0, *env); + if (rv) { + m_log.Emsg("Stat", "Failed to open path:", path); + } + // Assume that S3File::FStat() doesn't write to buff unless it succeeds. + rv = s3file.Fstat(buff); + if (rv != 0) { + formatstr(error, "File %s not found.", path); + m_log.Emsg("Stat", error.c_str()); + return -ENOENT; + } + + return 0; } -int -S3FileSystem::Create( const char *tid, const char *path, mode_t mode, - XrdOucEnv &env, int opts ) -{ - // Is path valid? - std::string exposedPath, object; - int rv = parse_path( * this, path, exposedPath, object ); - if( rv != 0 ) { return rv; } - - // - // We could instead invoke the upload mchinery directly to create a - // 0-byte file, but it seems smarter to remove a round-trip (in - // S3File::Open(), checking if the file exists) than to add one - // (here, creating the file if it doesn't exist). - // - - return 0; +int S3FileSystem::Create(const char *tid, const char *path, mode_t mode, + XrdOucEnv &env, int opts) { + // Is path valid? + std::string exposedPath, object; + int rv = parse_path(*this, path, exposedPath, object); + if (rv != 0) { + return rv; + } + + // + // We could instead invoke the upload mchinery directly to create a + // 0-byte file, but it seems smarter to remove a round-trip (in + // S3File::Open(), checking if the file exists) than to add one + // (here, creating the file if it doesn't exist). + // + + return 0; } diff --git a/src/S3FileSystem.hh b/src/S3FileSystem.hh index e32272b..5cdee6a 100644 --- a/src/S3FileSystem.hh +++ b/src/S3FileSystem.hh @@ -18,91 +18,120 @@ #pragma once +#include "S3AccessInfo.hh" +#include #include #include #include -#include -#include "S3AccessInfo.hh" +#include #include #include -#include - class S3FileSystem : public XrdOss { -public: - - S3FileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP); - virtual ~S3FileSystem(); - - bool - Config(XrdSysLogger *lp, const char *configfn); + public: + S3FileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP); + virtual ~S3FileSystem(); - XrdOssDF *newDir(const char *user=0); - XrdOssDF *newFile(const char *user=0); + bool Config(XrdSysLogger *lp, const char *configfn); - int Chmod(const char * path, mode_t mode, XrdOucEnv *env=0) {return -ENOSYS;} - void Connect(XrdOucEnv &env) {} - int Create(const char *tid, const char *path, mode_t mode, - XrdOucEnv &env, int opts=0); - void Disc(XrdOucEnv &env) {} - void EnvInfo(XrdOucEnv *env) {} - uint64_t Features() {return 0;} - int FSctl(int cmd, int alen, const char *args, char **resp=0) {return -ENOSYS;} - int Init(XrdSysLogger *lp, const char *cfn) {return 0;} - int Init(XrdSysLogger *lp, const char *cfn, XrdOucEnv *en) {return 0;} - int Mkdir(const char *path, mode_t mode, int mkpath=0, - XrdOucEnv *env=0) {return -ENOSYS;} - int Reloc(const char *tident, const char *path, - const char *cgName, const char *anchor=0) {return -ENOSYS;} - int Remdir(const char *path, int Opts=0, XrdOucEnv *env=0) {return -ENOSYS;} - int Rename(const char *oPath, const char *nPath, - XrdOucEnv *oEnvP=0, XrdOucEnv *nEnvP=0) {return -ENOSYS;} - int Stat(const char *path, struct stat *buff, - int opts=0, XrdOucEnv *env=0); - int Stats(char *buff, int blen) {return -ENOSYS;} - int StatFS(const char *path, char *buff, int &blen, - XrdOucEnv *env=0) {return -ENOSYS;} - int StatLS(XrdOucEnv &env, const char *path, - char *buff, int &blen) {return -ENOSYS;} - int StatPF(const char *path, struct stat *buff, int opts) {return -ENOSYS;} - int StatPF(const char *path, struct stat *buff) {return -ENOSYS;} - int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0) {return -ENOSYS;} - int StatXA(const char *path, char *buff, int &blen, - XrdOucEnv *env=0) {return -ENOSYS;} - int StatXP(const char *path, unsigned long long &attr, - XrdOucEnv *env=0) {return -ENOSYS;} - int Truncate(const char *path, unsigned long long fsize, - XrdOucEnv *env=0) {return -ENOSYS;} - int Unlink(const char *path, int Opts=0, XrdOucEnv *env=0) {return -ENOSYS;} - int Lfn2Pfn(const char *Path, char *buff, int blen) {return -ENOSYS;} - const char *Lfn2Pfn(const char *Path, char *buff, int blen, int &rc) {return nullptr;} + XrdOssDF *newDir(const char *user = 0); + XrdOssDF *newFile(const char *user = 0); - bool exposedPathExists(const std::string &exposedPath) const - { return s3_access_map.count(exposedPath) > 0;} - const std::string & getS3ServiceName (const std::string &exposedPath) - const { return s3_access_map.at(exposedPath)->getS3ServiceName(); } - const std::string &getS3Region(const std::string &exposedPath) - const { return s3_access_map.at(exposedPath)->getS3Region(); } - const std::string &getS3ServiceURL(const std::string &exposedPath) - const { return s3_access_map.at(exposedPath)->getS3ServiceUrl(); } - const std::string &getS3BucketName(const std::string &exposedPath) - const { return s3_access_map.at(exposedPath)->getS3BucketName(); } - const std::string &getS3AccessKeyFile(const std::string &exposedPath) - const { return s3_access_map.at(exposedPath)->getS3AccessKeyFile(); } - const std::string &getS3SecretKeyFile(const std::string &exposedPath) - const { return s3_access_map.at(exposedPath)->getS3SecretKeyFile(); } - const std::string & getS3URLStyle() const { return s3_url_style; } + int Chmod(const char *path, mode_t mode, XrdOucEnv *env = 0) { + return -ENOSYS; + } + void Connect(XrdOucEnv &env) {} + int Create(const char *tid, const char *path, mode_t mode, XrdOucEnv &env, + int opts = 0); + void Disc(XrdOucEnv &env) {} + void EnvInfo(XrdOucEnv *env) {} + uint64_t Features() { return 0; } + int FSctl(int cmd, int alen, const char *args, char **resp = 0) { + return -ENOSYS; + } + int Init(XrdSysLogger *lp, const char *cfn) { return 0; } + int Init(XrdSysLogger *lp, const char *cfn, XrdOucEnv *en) { return 0; } + int Mkdir(const char *path, mode_t mode, int mkpath = 0, + XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Reloc(const char *tident, const char *path, const char *cgName, + const char *anchor = 0) { + return -ENOSYS; + } + int Remdir(const char *path, int Opts = 0, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Rename(const char *oPath, const char *nPath, XrdOucEnv *oEnvP = 0, + XrdOucEnv *nEnvP = 0) { + return -ENOSYS; + } + int Stat(const char *path, struct stat *buff, int opts = 0, + XrdOucEnv *env = 0); + int Stats(char *buff, int blen) { return -ENOSYS; } + int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int StatLS(XrdOucEnv &env, const char *path, char *buff, int &blen) { + return -ENOSYS; + } + int StatPF(const char *path, struct stat *buff, int opts) { + return -ENOSYS; + } + int StatPF(const char *path, struct stat *buff) { return -ENOSYS; } + int StatVS(XrdOssVSInfo *vsP, const char *sname = 0, int updt = 0) { + return -ENOSYS; + } + int StatXA(const char *path, char *buff, int &blen, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int StatXP(const char *path, unsigned long long &attr, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Truncate(const char *path, unsigned long long fsize, + XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Unlink(const char *path, int Opts = 0, XrdOucEnv *env = 0) { + return -ENOSYS; + } + int Lfn2Pfn(const char *Path, char *buff, int blen) { return -ENOSYS; } + const char *Lfn2Pfn(const char *Path, char *buff, int blen, int &rc) { + return nullptr; + } -private: - XrdOucEnv *m_env; - XrdSysError m_log; + bool exposedPathExists(const std::string &exposedPath) const { + return s3_access_map.count(exposedPath) > 0; + } + const std::string &getS3ServiceName(const std::string &exposedPath) const { + return s3_access_map.at(exposedPath)->getS3ServiceName(); + } + const std::string &getS3Region(const std::string &exposedPath) const { + return s3_access_map.at(exposedPath)->getS3Region(); + } + const std::string &getS3ServiceURL(const std::string &exposedPath) const { + return s3_access_map.at(exposedPath)->getS3ServiceUrl(); + } + const std::string &getS3BucketName(const std::string &exposedPath) const { + return s3_access_map.at(exposedPath)->getS3BucketName(); + } + const std::string & + getS3AccessKeyFile(const std::string &exposedPath) const { + return s3_access_map.at(exposedPath)->getS3AccessKeyFile(); + } + const std::string & + getS3SecretKeyFile(const std::string &exposedPath) const { + return s3_access_map.at(exposedPath)->getS3SecretKeyFile(); + } + const std::string &getS3URLStyle() const { return s3_url_style; } - bool handle_required_config( - const char * desired_name, - const std::string & source - ); - std::map s3_access_map; - std::string s3_url_style; + private: + XrdOucEnv *m_env; + XrdSysError m_log; + bool handle_required_config(const char *desired_name, + const std::string &source); + std::map s3_access_map; + std::string s3_url_style; }; diff --git a/src/logging.cc b/src/logging.cc index 845e08c..d013507 100644 --- a/src/logging.cc +++ b/src/logging.cc @@ -25,49 +25,62 @@ using namespace XrdHTTPServer; -std::string -XrdHTTPServer::LogMaskToString(int mask) { - if (mask == LogMask::All) {return "all";} - - bool has_entry = false; - std::stringstream ss; - if (mask & LogMask::Debug) { - ss << "debug"; - has_entry = true; - } - if (mask & LogMask::Info) { - ss << (has_entry ? ", " : "") << "info"; - has_entry = true; - } - if (mask & LogMask::Warning) { - ss << (has_entry ? ", " : "") << "warning"; - has_entry = true; - } - if (mask & LogMask::Error) { - ss << (has_entry ? ", " : "") << "error"; - has_entry = true; - } - return ss.str(); +std::string XrdHTTPServer::LogMaskToString(int mask) { + if (mask == LogMask::All) { + return "all"; + } + + bool has_entry = false; + std::stringstream ss; + if (mask & LogMask::Debug) { + ss << "debug"; + has_entry = true; + } + if (mask & LogMask::Info) { + ss << (has_entry ? ", " : "") << "info"; + has_entry = true; + } + if (mask & LogMask::Warning) { + ss << (has_entry ? ", " : "") << "warning"; + has_entry = true; + } + if (mask & LogMask::Error) { + ss << (has_entry ? ", " : "") << "error"; + has_entry = true; + } + return ss.str(); } -bool -XrdHTTPServer::ConfigLog(XrdOucStream &conf, XrdSysError &log) { - std::string map_filename; - log.setMsgMask(0); - char *val = nullptr; - if (!(val = conf.GetToken())) { - log.Emsg("Config", "httpserver.trace requires an argument. Usage: httpserver.trace [all|error|warning|info|debug|none]"); - return false; - } - do { - if (!strcmp(val, "all")) {log.setMsgMask(log.getMsgMask() | LogMask::All);} - else if (!strcmp(val, "error")) {log.setMsgMask(log.getMsgMask() | LogMask::Error);} - else if (!strcmp(val, "warning")) {log.setMsgMask(log.getMsgMask() | LogMask::Warning);} - else if (!strcmp(val, "info")) {log.setMsgMask(log.getMsgMask() | LogMask::Info);} - else if (!strcmp(val, "debug")) {log.setMsgMask(log.getMsgMask() | LogMask::Debug);} - else if (!strcmp(val, "none")) {log.setMsgMask(0);} - else {log.Emsg("Config", "scitokens.trace encountered an unknown directive:", val); return false;} - } while ((val = conf.GetToken())); - log.Emsg("Config", "Logging levels enabled -", XrdHTTPServer::LogMaskToString(log.getMsgMask()).c_str()); - return true; +bool XrdHTTPServer::ConfigLog(XrdOucStream &conf, XrdSysError &log) { + std::string map_filename; + log.setMsgMask(0); + char *val = nullptr; + if (!(val = conf.GetToken())) { + log.Emsg("Config", + "httpserver.trace requires an argument. Usage: " + "httpserver.trace [all|error|warning|info|debug|none]"); + return false; + } + do { + if (!strcmp(val, "all")) { + log.setMsgMask(log.getMsgMask() | LogMask::All); + } else if (!strcmp(val, "error")) { + log.setMsgMask(log.getMsgMask() | LogMask::Error); + } else if (!strcmp(val, "warning")) { + log.setMsgMask(log.getMsgMask() | LogMask::Warning); + } else if (!strcmp(val, "info")) { + log.setMsgMask(log.getMsgMask() | LogMask::Info); + } else if (!strcmp(val, "debug")) { + log.setMsgMask(log.getMsgMask() | LogMask::Debug); + } else if (!strcmp(val, "none")) { + log.setMsgMask(0); + } else { + log.Emsg("Config", + "scitokens.trace encountered an unknown directive:", val); + return false; + } + } while ((val = conf.GetToken())); + log.Emsg("Config", "Logging levels enabled -", + XrdHTTPServer::LogMaskToString(log.getMsgMask()).c_str()); + return true; } diff --git a/src/logging.hh b/src/logging.hh index 41a5750..6426bc0 100644 --- a/src/logging.hh +++ b/src/logging.hh @@ -25,19 +25,20 @@ class XrdSysError; namespace XrdHTTPServer { -enum LogMask { - Debug = 0x01, - Info = 0x02, - Warning = 0x04, - Error = 0x08, - All = 0xff -}; - -// Given a bitset based on LogMask, return a human-readable string of the set logging levels. +enum LogMask { + Debug = 0x01, + Info = 0x02, + Warning = 0x04, + Error = 0x08, + All = 0xff +}; + +// Given a bitset based on LogMask, return a human-readable string of the set +// logging levels. std::string LogMaskToString(int mask); // Given an xrootd configuration object that matched on httpserver.trace, parse // the remainder of the line and configure the logger appropriately. bool ConfigLog(XrdOucStream &conf, XrdSysError &log); -} +} // namespace XrdHTTPServer diff --git a/src/shortfile.cc b/src/shortfile.cc index 0931ea2..ce6b022 100644 --- a/src/shortfile.cc +++ b/src/shortfile.cc @@ -20,71 +20,66 @@ #include -#include -#include +#include #include +#include +#include #include -#include - - -ssize_t -full_read( int fd, void * ptr, size_t nbytes ) { - ssize_t nleft, nread; - - nleft = nbytes; - while (nleft > 0) { - REISSUE_READ: - nread = read(fd, ptr, nleft); - if (nread < 0) { - /* error happened, ignore if EINTR, otherwise inform the caller */ - if (errno == EINTR) { - goto REISSUE_READ; - } - /* The caller has no idea how much was actually read in this - scenario and the file offset is undefined */ - return -1; - } else if (nread == 0) { - /* We've reached the end of file marker, so stop looping. */ - break; - } - - nleft -= nread; - ptr = ((char *)ptr) + nread; - } - - /* return how much was actually read, which could include 0 in an - EOF situation */ - return (nbytes - nleft); +ssize_t full_read(int fd, void *ptr, size_t nbytes) { + ssize_t nleft, nread; + + nleft = nbytes; + while (nleft > 0) { + REISSUE_READ: + nread = read(fd, ptr, nleft); + if (nread < 0) { + /* error happened, ignore if EINTR, otherwise inform the caller */ + if (errno == EINTR) { + goto REISSUE_READ; + } + + /* The caller has no idea how much was actually read in this + scenario and the file offset is undefined */ + return -1; + } else if (nread == 0) { + /* We've reached the end of file marker, so stop looping. */ + break; + } + + nleft -= nread; + ptr = ((char *)ptr) + nread; + } + + /* return how much was actually read, which could include 0 in an + EOF situation */ + return (nbytes - nleft); } - - -bool -readShortFile( const std::string & fileName, std::string & contents ) { - int fd = open( fileName.c_str(), O_RDONLY, 0600 ); - - if( fd < 0 ) { - return false; - } - - struct stat statbuf; - int rv = fstat( fd, & statbuf ); - if( rv < 0 ) { - return false; - } - unsigned long fileSize = statbuf.st_size; - - char * rawBuffer = (char *)malloc( fileSize + 1 ); - assert( rawBuffer != NULL ); - unsigned long totalRead = full_read( fd, rawBuffer, fileSize ); - close( fd ); - if( totalRead != fileSize ) { - free( rawBuffer ); - return false; - } - contents.assign( rawBuffer, fileSize ); - free( rawBuffer ); - - return true; +bool readShortFile(const std::string &fileName, std::string &contents) { + int fd = open(fileName.c_str(), O_RDONLY, 0600); + + if (fd < 0) { + return false; + } + + struct stat statbuf; + int rv = fstat(fd, &statbuf); + if (rv < 0) { + return false; + } + unsigned long fileSize = statbuf.st_size; + + char *rawBuffer = (char *)malloc(fileSize + 1); + assert(rawBuffer != NULL); + unsigned long totalRead = full_read(fd, rawBuffer, fileSize); + close(fd); + if (totalRead != fileSize) { + free(rawBuffer); + return false; + } + contents.assign(rawBuffer, fileSize); + free(rawBuffer); + + return true; } diff --git a/src/stl_string_utils.cc b/src/stl_string_utils.cc index 4d1d07a..4623b6e 100644 --- a/src/stl_string_utils.cc +++ b/src/stl_string_utils.cc @@ -24,80 +24,78 @@ #include -std::string -substring( const std::string & str, size_t left, size_t right ) { - if( right == std::string::npos ) { - return str.substr( left ); - } else { - return str.substr( left, right - left ); - } +std::string substring(const std::string &str, size_t left, size_t right) { + if (right == std::string::npos) { + return str.substr(left); + } else { + return str.substr(left, right - left); + } } -void -trim( std::string &str ) { - if( str.empty() ) { +void trim(std::string &str) { + if (str.empty()) { return; } - unsigned begin = 0; - while ( begin < str.length() && isspace(str[begin]) ) { + unsigned begin = 0; + while (begin < str.length() && isspace(str[begin])) { ++begin; } - int end = (int)str.length() - 1; - while ( end >= 0 && isspace(str[end]) ) { + int end = (int)str.length() - 1; + while (end >= 0 && isspace(str[end])) { --end; } - if ( begin != 0 || end != (int)(str.length()) - 1 ) { + if (begin != 0 || end != (int)(str.length()) - 1) { str = str.substr(begin, (end - begin) + 1); } } -void -toLower( std::string &str ) { - std::transform(str.begin(), str.end(), str.begin(), ::tolower); +void toLower(std::string &str) { + std::transform(str.begin(), str.end(), str.begin(), ::tolower); } -int -vformatstr_impl(std::string& s, bool concat, const char* format, va_list pargs) { - char fixbuf[512]; - const int fixlen = sizeof(fixbuf)/sizeof(fixbuf[0]); +int vformatstr_impl(std::string &s, bool concat, const char *format, + va_list pargs) { + char fixbuf[512]; + const int fixlen = sizeof(fixbuf) / sizeof(fixbuf[0]); int n; #if !defined(va_copy) n = vsnprintf(fixbuf, fixlen, format, pargs); #else - va_list args; + va_list args; va_copy(args, pargs); n = vsnprintf(fixbuf, fixlen, format, args); va_end(args); #endif - // In this case, fixed buffer was sufficient so we're done. - // Return number of chars written. - if (n < fixlen) { + // In this case, fixed buffer was sufficient so we're done. + // Return number of chars written. + if (n < fixlen) { if (concat) { s.append(fixbuf, n); } else { s.assign(fixbuf, n); } - return n; - } - - // Otherwise, the fixed buffer was not large enough, but return from - // vsnprintf() tells us how much memory we need now. - n += 1; - char* varbuf = NULL; - // Handle 'new' behavior mode of returning NULL or throwing exception - try { - varbuf = new char[n]; - } catch (...) { - varbuf = NULL; - } - // if (NULL == varbuf) { EXCEPT("Failed to allocate char buffer of %d chars", n); } + return n; + } + + // Otherwise, the fixed buffer was not large enough, but return from + // vsnprintf() tells us how much memory we need now. + n += 1; + char *varbuf = NULL; + // Handle 'new' behavior mode of returning NULL or throwing exception + try { + varbuf = new char[n]; + } catch (...) { + varbuf = NULL; + } + // if (NULL == varbuf) { EXCEPT("Failed to allocate char buffer of %d + // chars", n); } assert(NULL == varbuf); - // re-print, using buffer of sufficient size + // re-print, using buffer of sufficient size #if !defined(va_copy) int nn = vsnprintf(varbuf, n, format, pargs); #else @@ -106,48 +104,45 @@ vformatstr_impl(std::string& s, bool concat, const char* format, va_list pargs) va_end(args); #endif - // Sanity check. This ought not to happen. Ever. - // if (nn >= n) EXCEPT("Insufficient buffer size (%d) for printing %d chars", n, nn); - assert(nn >= n); + // Sanity check. This ought not to happen. Ever. + // if (nn >= n) EXCEPT("Insufficient buffer size (%d) for printing %d + // chars", n, nn); + assert(nn >= n); - // safe to do string assignment + // safe to do string assignment if (concat) { s.append(varbuf, nn); } else { s.assign(varbuf, nn); } - // clean up our allocated buffer - delete[] varbuf; + // clean up our allocated buffer + delete[] varbuf; - // return number of chars written - return nn; + // return number of chars written + return nn; } -int -vformatstr(std::string& s, const char* format, va_list pargs) { +int vformatstr(std::string &s, const char *format, va_list pargs) { return vformatstr_impl(s, false, format, pargs); } -int -vformatstr_cat(std::string& s, const char* format, va_list pargs) { +int vformatstr_cat(std::string &s, const char *format, va_list pargs) { return vformatstr_impl(s, true, format, pargs); } -int -formatstr(std::string& s, const char* format, ...) { - va_list args; - va_start(args, format); - int r = vformatstr_impl(s, false, format, args); - va_end(args); - return r; +int formatstr(std::string &s, const char *format, ...) { + va_list args; + va_start(args, format); + int r = vformatstr_impl(s, false, format, args); + va_end(args); + return r; } -int -formatstr_cat(std::string& s, const char* format, ...) { - va_list args; - va_start(args, format); - int r = vformatstr_impl(s, true, format, args); - va_end(args); - return r; +int formatstr_cat(std::string &s, const char *format, ...) { + va_list args; + va_start(args, format); + int r = vformatstr_impl(s, true, format, args); + va_end(args); + return r; } diff --git a/src/stl_string_utils.hh b/src/stl_string_utils.hh index 324b784..9222f52 100644 --- a/src/stl_string_utils.hh +++ b/src/stl_string_utils.hh @@ -21,16 +21,19 @@ #include #ifndef CHECK_PRINTF_FORMAT - #ifdef __GNUC__ - #define CHECK_PRINTF_FORMAT(a,b) __attribute__((__format__(__printf__, a, b))) - #else - #define CHECK_PRINTF_FORMAT(a,b) - #endif +#ifdef __GNUC__ +#define CHECK_PRINTF_FORMAT(a, b) __attribute__((__format__(__printf__, a, b))) +#else +#define CHECK_PRINTF_FORMAT(a, b) +#endif #endif -void trim( std::string & str ); -std::string substring( const std::string & str, size_t left, size_t right = std::string::npos ); -void toLower( std::string & str); +void trim(std::string &str); +std::string substring(const std::string &str, size_t left, + size_t right = std::string::npos); +void toLower(std::string &str); -int formatstr(std::string& s, const char* format, ...) CHECK_PRINTF_FORMAT(2,3); -int formatstr_cat(std::string& s, const char* format, ...) CHECK_PRINTF_FORMAT(2,3); +int formatstr(std::string &s, const char *format, ...) + CHECK_PRINTF_FORMAT(2, 3); +int formatstr_cat(std::string &s, const char *format, ...) + CHECK_PRINTF_FORMAT(2, 3); From 8d1f4616de93aec052059d9d8e20789a01d2c4fa Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Mon, 15 Apr 2024 17:48:50 +0000 Subject: [PATCH 2/3] Cleanup missed lint item by hand --- src/HTTPCommands.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HTTPCommands.cc b/src/HTTPCommands.cc index 29acc06..6579180 100644 --- a/src/HTTPCommands.cc +++ b/src/HTTPCommands.cc @@ -68,7 +68,7 @@ HTTPRequest::~HTTPRequest() {} this->errorCode = "E_CURL_LIB"; \ this->errorMessage = "curl_easy_setopt( " #B " ) failed."; \ /* dprintf( D_ALWAYS, "curl_easy_setopt( %s ) failed (%d): '%s', \ - failing.\n", #B, rv##B, curl_easy_strerror( rv##B ) ); */ \ + failing.\n", #B, rv##B, curl_easy_strerror( rv##B ) ); */ \ return false; \ } \ } From 32ada698a0e7d78a19316bf438e2ed01eb1784bf Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Mon, 15 Apr 2024 17:50:34 +0000 Subject: [PATCH 3/3] Add missing newline to .clang-format --- .clang-format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.clang-format b/.clang-format index 8675285..ec827d5 100644 --- a/.clang-format +++ b/.clang-format @@ -1,4 +1,4 @@ BasedOnStyle: LLVM IndentWidth: 4 UseTab: Always -TabWidth: 4 \ No newline at end of file +TabWidth: 4