diff --git a/CMakeLists.txt b/CMakeLists.txt index a430f6e..91e9b6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,8 +46,8 @@ endif() include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS}) -add_library(XrdS3 SHARED src/S3File.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc) -add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc) +add_library(XrdS3 SHARED src/S3File.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc) +add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc) target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES}) target_link_libraries(XrdHTTPServer -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES}) diff --git a/src/AWSCredential.hh b/src/AWSCredential.hh index daed127..cbdd857 100644 --- a/src/AWSCredential.hh +++ b/src/AWSCredential.hh @@ -1,5 +1,21 @@ -#ifndef AWS_CREDENTIAL_H -#define AWS_CREDENTIAL_H +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ +#pragma once class AWSCredential { @@ -19,5 +35,3 @@ class AWSCredential { const std::string m_secret_key; const std::string m_security_token; }; - -#endif /* AWS_CREDENTIAL_H */ diff --git a/src/AWSv4-impl.cc b/src/AWSv4-impl.cc index 931a3a0..2050330 100644 --- a/src/AWSv4-impl.cc +++ b/src/AWSv4-impl.cc @@ -1,3 +1,21 @@ +/*************************************************************** + * + * Copyright (C) 2023, HTCondor team, UW-Madison + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + /** * Utilities for generating pre-signed URLs. * @@ -82,7 +100,7 @@ convertMessageDigestToLowercaseHex( char * buffer = (char *)malloc( (mdLength * 2) + 1 ); char * ptr = buffer; for (unsigned int i = 0; i < mdLength; ++i, ptr += 2) { - sprintf(ptr, "%02x", messageDigest[i]); + snprintf(ptr, 3, "%02x", messageDigest[i]); } hexEncoded.assign(buffer, mdLength * 2); free(buffer); diff --git a/src/AWSv4-impl.hh b/src/AWSv4-impl.hh index 0c9cad3..f7c0b51 100644 --- a/src/AWSv4-impl.hh +++ b/src/AWSv4-impl.hh @@ -1,5 +1,25 @@ -#ifndef AWSV4_IMPL_H -#define AWSV4_IMPL_H +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#pragma once + +#include +#include namespace AWSv4Impl { @@ -27,5 +47,3 @@ createSignature( const std::string & secretAccessKey, std::string & signature ); } - -#endif /* AWSV4_IMPL_H */ diff --git a/src/HTTPCommands.cc b/src/HTTPCommands.cc index ee1ba3a..4145760 100644 --- a/src/HTTPCommands.cc +++ b/src/HTTPCommands.cc @@ -1,19 +1,41 @@ -#include +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #include -#include -#include #include #include -#include #include - #include +#include +#include #include +#include +#include +#include + #include "HTTPCommands.hh" +#include "logging.hh" #include "stl_string_utils.hh" #include "shortfile.hh" +using namespace XrdHTTPServer; + // // "This function gets called by libcurl as soon as there is data received // that needs to be saved. The size of the data pointed to by ptr is size @@ -78,7 +100,7 @@ bool HTTPRequest::SendHTTPRequest( const std::string & payload ) { if( (protocol != "http") && (protocol != "https") ) { this->errorCode = "E_INVALID_SERVICE_URL"; this->errorMessage = "Service URL not of a known protocol (http[s])."; - // dprintf( D_ALWAYS, "Service URL '%s' not of a known protocol (http[s]).\n", serviceURL.c_str() ); + m_log.Log(LogMask::Warning, "HTTPRequest::SendHTTPRequest", "Host URL '", hostUrl.c_str(), "' not of a known protocol (http[s])."); return false; } @@ -261,6 +283,12 @@ bool HTTPRequest::sendPreparedRequest( return false; } + if (curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1) != CURLE_OK) { + this->errorCode = "E_CURL_LIB"; + this->errorMessage = "curl_easy_setopt( CURLOPT_FOLLOWLOCATION ) failed."; + return false; + } + // // Set security options. // @@ -402,7 +430,7 @@ HTTPUpload::~HTTPUpload() { } bool HTTPUpload::SendRequest( const std::string & payload, off_t offset, size_t size ) { if( offset != 0 || size != 0 ) { std::string range; - formatstr( range, "bytes=%zu-%zu", offset, offset + size - 1 ); + formatstr( range, "bytes=%lld-%lld", static_cast(offset), static_cast(offset + size - 1)); headers["Range"] = range.c_str(); } @@ -417,7 +445,7 @@ HTTPDownload::~HTTPDownload() { } bool HTTPDownload::SendRequest( off_t offset, size_t size ) { if( offset != 0 || size != 0 ) { std::string range; - formatstr( range, "bytes=%zu-%zu", offset, offset + size - 1 ); + formatstr( range, "bytes=%lld-%lld", static_cast(offset), static_cast(offset + size - 1)); headers["Range"] = range.c_str(); this->expectedResponseCode = 206; } diff --git a/src/HTTPCommands.hh b/src/HTTPCommands.hh index 81fafd8..f12b869 100644 --- a/src/HTTPCommands.hh +++ b/src/HTTPCommands.hh @@ -1,17 +1,40 @@ -#ifndef HTTP_COMMANDS_H -#define HTTP_COMMANDS_H - +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#pragma once + +#include +#include + +class XrdSysError; class HTTPRequest { public: HTTPRequest( - const std::string & hostUrl + const std::string & hostUrl, + XrdSysError &log ) : hostUrl(hostUrl), requiresSignature(false), responseCode(0), includeResponseHeader(false), - httpVerb( "POST" ) + httpVerb("POST"), + m_log(log) { // Parse the URL and populate // What to do if the function returns false? @@ -60,15 +83,18 @@ protected: bool includeResponseHeader; std::string httpVerb; + + XrdSysError &m_log; }; class HTTPUpload : public HTTPRequest { public: HTTPUpload( const std::string & h, - const std::string & o + const std::string & o, + XrdSysError &log ) : - HTTPRequest(h), + HTTPRequest(h, log), object(o) { hostUrl = hostUrl + "/" + object; } @@ -85,9 +111,10 @@ class HTTPDownload : public HTTPRequest { public: HTTPDownload( const std::string & h, - const std::string & o + const std::string & o, + XrdSysError &log ) : - HTTPRequest(h), + HTTPRequest(h, log), object(o) { hostUrl = hostUrl + "/" + object; } @@ -103,9 +130,10 @@ class HTTPHead : public HTTPRequest { public: HTTPHead( const std::string & h, - const std::string & o + const std::string & o, + XrdSysError &log ) : - HTTPRequest(h), + HTTPRequest(h, log), object(o) { hostUrl = hostUrl + "/" + object; } @@ -116,5 +144,3 @@ public: protected: std::string object; }; - -#endif /* HTTP_COMMANDS_H */ diff --git a/src/HTTPDirectory.hh b/src/HTTPDirectory.hh index 086bb4d..af2188f 100644 --- a/src/HTTPDirectory.hh +++ b/src/HTTPDirectory.hh @@ -1,8 +1,27 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #pragma once #include "XrdOuc/XrdOucEnv.hh" #include "XrdOss/XrdOss.hh" +class XrdSysError; class HTTPDirectory : public XrdOssDF { public: @@ -20,21 +39,21 @@ public: return -ENOSYS; } - virtual int Readdir(char *buff, int blen) + virtual int Readdir(char *buff, int blen) override { return -ENOSYS; } - virtual int StatRet(struct stat *statStruct) + virtual int StatRet(struct stat *statStruct) override { return -ENOSYS; } - virtual int Close(long long *retsz=0) + virtual int Close(long long *retsz=0) override { return -ENOSYS; } protected: - XrdSysError m_log; + XrdSysError &m_log; }; diff --git a/src/HTTPFile.cc b/src/HTTPFile.cc index 794230f..4b5d701 100644 --- a/src/HTTPFile.cc +++ b/src/HTTPFile.cc @@ -1,26 +1,45 @@ -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdSec/XrdSecEntityAttr.hh" -#include "XrdSfs/XrdSfsInterface.hh" -#include "XrdVersion.hh" +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#include "HTTPCommands.hh" #include "HTTPFileSystem.hh" #include "HTTPFile.hh" +#include "logging.hh" +#include "stl_string_utils.hh" #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include - -#include #include -#include "HTTPCommands.hh" +#include -#include "stl_string_utils.hh" -#include +using namespace XrdHTTPServer; HTTPFileSystem* g_http_oss = nullptr; @@ -69,8 +88,8 @@ parse_path( const std::string & hname, const char * path, std::string & object ) int HTTPFile::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) { - std::string configured_hostname = m_oss->getHTTPHostName(); - std::string configured_hostUrl = m_oss->getHTTPHostUrl(); + auto configured_hostname = m_oss->getHTTPHostName(); + auto configured_hostUrl = m_oss->getHTTPHostUrl(); // // Check the path for validity. @@ -84,7 +103,6 @@ HTTPFile::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) // if you're creating a file on upload, you don't care. this->object = object; - //this->protocol = configured_protocol; this->hostname = configured_hostname; this->hostUrl = configured_hostUrl; @@ -97,12 +115,15 @@ HTTPFile::Read(void *buffer, off_t offset, size_t size) { HTTPDownload download( this->hostUrl, - this->object + this->object, + m_log ); - fprintf( stderr, "D_FULLDEBUG: about to perform download.SendRequest from HTTPFile::Read(): hostname: '%s' object: '%s'\n", hostname.c_str(), object.c_str() ); + m_log.Log(LogMask::Debug, "HTTPFile::Read", "About to perform download from HTTPFile::Read(): hostname / object:", hostname.c_str(), object.c_str()); - if(! download.SendRequest( offset, size ) ) { - fprintf( stderr, "D_FULLDEBUG: failed to send GetObject command: %lu '%s'\n", download.getResponseCode(), download.getResultString().c_str() ); + if (!download.SendRequest(offset, size)) { + std::stringstream ss; + ss << "Failed to send GetObject command: " << download.getResponseCode() << "'" << download.getResultString() << "'"; + m_log.Log(LogMask::Warning, "HTTPFile::Read", ss.str().c_str()); return 0; } @@ -115,19 +136,22 @@ HTTPFile::Read(void *buffer, off_t offset, size_t size) int HTTPFile::Fstat(struct stat *buff) { - fprintf( stderr, "D_FULLDEBUG: In HTTPFile::Fstat: hostname: '%s' object: '%s'\n", hostname.c_str(), object.c_str() ); + m_log.Log(LogMask::Debug, "HTTPFile::Fstat", "About to perform HTTPFile::Fstat(): hostname / object", hostname.c_str(), object.c_str()); HTTPHead head( this->hostUrl, - this->object + this->object, + m_log ); - if(! head.SendRequest()) { + if (!head.SendRequest()) { // SendRequest() returns false for all errors, including ones // where the server properly responded with something other // than code 200. If xrootd wants us to distinguish between // these cases, head.getResponseCode() is initialized to 0, so // we can check. - fprintf( stderr, "D_FULLDEBUG: failed to send HeadObject command: %lu '%s'\n", head.getResponseCode(), head.getResultString().c_str() ); + std::stringstream ss; + ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" << head.getResultString() << "'"; + m_log.Log(LogMask::Warning, "HTTPFile::Fstat", ss.str().c_str()); return -ENOENT; } @@ -189,7 +213,8 @@ HTTPFile::Write(const void *buffer, off_t offset, size_t size) { HTTPUpload upload( this->hostUrl, - this->object + this->object, + m_log ); std::string payload( (char *)buffer, size ); diff --git a/src/HTTPFile.hh b/src/HTTPFile.hh index 6a8e2ed..53c1774 100644 --- a/src/HTTPFile.hh +++ b/src/HTTPFile.hh @@ -1,3 +1,21 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #pragma once #include "XrdOuc/XrdOucEnv.hh" @@ -111,7 +129,7 @@ public: return -ENOSYS; } - virtual int Close(long long *retsz=0); // upstream is abstract definition + virtual int Close(long long *retsz=0) override; // upstream is abstract definition size_t getContentLength() { return content_length; } time_t getLastModified() { return last_modified; } diff --git a/src/HTTPFileSystem.cc b/src/HTTPFileSystem.cc index f579bf8..404b035 100644 --- a/src/HTTPFileSystem.cc +++ b/src/HTTPFileSystem.cc @@ -1,14 +1,35 @@ -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdVersion.hh" -#include "HTTPFileSystem.hh" +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #include "HTTPDirectory.hh" #include "HTTPFile.hh" +#include "HTTPFileSystem.hh" +#include "logging.hh" + +#include +#include +#include +#include #include #include #include +#include #include #include @@ -17,6 +38,8 @@ #include "stl_string_utils.hh" +using namespace XrdHTTPServer; + HTTPFileSystem::HTTPFileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP) : m_env(envP), m_log(lp, "httpserver_") @@ -48,7 +71,9 @@ HTTPFileSystem::handle_required_config( return false; } - // fprintf( stderr, "Setting %s = %s\n", desired_name, source.c_str() ); + std::stringstream ss; + ss << "Setting " << desired_name << "=" << source; + m_log.Log(LogMask::Debug, "Config", ss.str().c_str()); target = source; return true; } @@ -72,6 +97,13 @@ HTTPFileSystem::Config(XrdSysLogger *lp, const char *configfn) Config.Attach(cfgFD); while ((temporary = Config.GetMyFirstWord())) { attribute = temporary; + if (attribute == "httpserver.trace") { + if (!XrdHTTPServer::ConfigLog(Config, m_log)) { + m_log.Emsg("Config", "Failed to configure the log level"); + } + continue; + } + temporary = Config.GetWord(); if(! temporary) { continue; } value = temporary; diff --git a/src/HTTPFileSystem.hh b/src/HTTPFileSystem.hh index 856a95f..2c7e77e 100644 --- a/src/HTTPFileSystem.hh +++ b/src/HTTPFileSystem.hh @@ -1,9 +1,27 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #pragma once -#include "XrdOuc/XrdOucStream.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdVersion.hh" -#include "XrdOss/XrdOss.hh" +#include +#include +#include +#include #include #include diff --git a/src/S3Commands.cc b/src/S3Commands.cc index 0bb1526..9006712 100644 --- a/src/S3Commands.cc +++ b/src/S3Commands.cc @@ -1,19 +1,38 @@ -#include -#include +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#include "AWSv4-impl.hh" +#include "S3Commands.hh" +#include "shortfile.hh" +#include "stl_string_utils.hh" + #include #include +#include + +#include #include #include -#include - #include +#include +#include #include -#include "S3Commands.hh" -#include "AWSv4-impl.hh" -#include "stl_string_utils.hh" -#include "shortfile.hh" - AmazonRequest::~AmazonRequest() { } bool AmazonRequest::SendRequest() { @@ -378,7 +397,7 @@ AmazonS3Upload::~AmazonS3Upload() { } bool AmazonS3Upload::SendRequest( const std::string & payload, off_t offset, size_t size ) { if( offset != 0 || size != 0 ) { std::string range; - formatstr( range, "bytes=%zu-%zu", offset, offset + size - 1 ); + formatstr(range, "bytes=%lld-%lld", static_cast(offset), static_cast(offset + size - 1)); headers["Range"] = range.c_str(); } @@ -393,7 +412,7 @@ AmazonS3Download::~AmazonS3Download() { } bool AmazonS3Download::SendRequest( off_t offset, size_t size ) { if( offset != 0 || size != 0 ) { std::string range; - formatstr( range, "bytes=%zu-%zu", offset, offset + size -1 ); + formatstr(range, "bytes=%lld-%lld",static_cast(offset), static_cast(offset + size - 1)); headers["Range"] = range.c_str(); this->expectedResponseCode = 206; } diff --git a/src/S3Commands.hh b/src/S3Commands.hh index a7385ec..15a7d1c 100644 --- a/src/S3Commands.hh +++ b/src/S3Commands.hh @@ -1,8 +1,27 @@ -#ifndef S3_COMMANDS_H -#define S3_COMMANDS_H +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#pragma once #include "HTTPCommands.hh" +#include + class AmazonRequest : public HTTPRequest { public: AmazonRequest( @@ -12,9 +31,10 @@ public: const std::string & b, const std::string & o, const std::string & style, - int sv = 4 + int sv, + XrdSysError &log ) : - HTTPRequest( s ), + HTTPRequest(s, log), accessKeyFile(akf), secretKeyFile(skf), signatureVersion(sv), @@ -98,9 +118,10 @@ public: const std::string & skf, const std::string & b, const std::string & o, - const std::string & style + const std::string & style, + XrdSysError &log ) : - AmazonRequest(s, akf, skf, b, o, style){} + AmazonRequest(s, akf, skf, b, o, style, 4, log) {} virtual ~AmazonS3Upload(); @@ -119,9 +140,10 @@ public: const std::string & skf, const std::string & b, const std::string & o, - const std::string & style + const std::string & style, + XrdSysError &log ) : - AmazonRequest(s, akf, skf, b, o, style){} + AmazonRequest(s, akf, skf, b, o, style, 4, log){} virtual ~AmazonS3Download(); @@ -137,13 +159,12 @@ public: const std::string & skf, const std::string & b, const std::string & o, - const std::string & style + const std::string & style, + XrdSysError &log ) : - AmazonRequest(s, akf, skf, b, o, style){} + AmazonRequest(s, akf, skf, b, o, style, 4, log){} virtual ~AmazonS3Head(); virtual bool SendRequest(); }; - -#endif /* S3_COMMANDS_H */ diff --git a/src/S3Directory.hh b/src/S3Directory.hh index 2cbd378..f99b9c4 100644 --- a/src/S3Directory.hh +++ b/src/S3Directory.hh @@ -1,8 +1,25 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #pragma once #include "HTTPDirectory.hh" - // Leaving in duplicate definitions for now. It remains // to be seen if we'll need to change these and have specific // behaviors for either HTTP or S3 variants in the future. diff --git a/src/S3File.cc b/src/S3File.cc index 47ad94e..068351d 100644 --- a/src/S3File.cc +++ b/src/S3File.cc @@ -1,27 +1,45 @@ -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdSec/XrdSecEntityAttr.hh" -#include "XrdSfs/XrdSfsInterface.hh" -#include "XrdVersion.hh" +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#include "S3Commands.hh" #include "S3FileSystem.hh" #include "S3File.hh" +#include "logging.hh" #include "stl_string_utils.hh" +#include +#include +#include +#include +#include +#include + #include +#include +#include #include #include #include -#include - -#include - -#include #include -#include "S3Commands.hh" +#include -#include "stl_string_utils.hh" +using namespace XrdHTTPServer; S3FileSystem* g_s3_oss = nullptr; @@ -47,35 +65,31 @@ parse_path( const S3FileSystem & fs, const char * path, std::string & bucket, st auto pathComponents = p.begin(); ++pathComponents; - if( pathComponents == p.end() ) { return -ENOENT; } - if( * pathComponents != configured_s3_service_name ) { + if(pathComponents == p.end()) { return -ENOENT; } + if (* pathComponents != configured_s3_service_name) { return -ENOENT; } ++pathComponents; - if( pathComponents == p.end() ) { return -ENOENT; } - if( * pathComponents != configured_s3_region ) { + if (pathComponents == p.end()) { return -ENOENT; } + if (*pathComponents != configured_s3_region) { return -ENOENT; } ++pathComponents; - if( pathComponents == p.end() ) { return -ENOENT; } + if (pathComponents == p.end()) { return -ENOENT; } bucket = *pathComponents; // Objects names may contain path separators. ++pathComponents; - if( pathComponents == p.end() ) { return -ENOENT; } - + if (pathComponents == p.end()) { return -ENOENT; } std::filesystem::path objectPath = *pathComponents++; - for( ; pathComponents != p.end(); ++pathComponents ) { - objectPath /= (* pathComponents); + for ( ; pathComponents != p.end(); ++pathComponents) { + objectPath /= (*pathComponents); } object = objectPath.string(); - fprintf( stderr, "object = %s\n", object.c_str() ); - - return 0; } @@ -121,12 +135,14 @@ S3File::Read(void *buffer, off_t offset, size_t size) this->s3_secret_key, this->s3_bucket_name, this->s3_object_name, - this->s3_url_style + this->s3_url_style, + m_log ); - if(! download.SendRequest( offset, size ) ) { - fprintf( stderr, "D_FULLDEBUG: failed to send GetObject command: %lu '%s'\n", download.getResponseCode(), download.getResultString().c_str() ); + std::stringstream ss; + ss << "Failed to send GetObject command: " << download.getResponseCode() << "'" << download.getResultString() << "'"; + m_log.Log(LogMask::Warning, "S3File::Read", ss.str().c_str()); return 0; } @@ -145,7 +161,8 @@ S3File::Fstat(struct stat *buff) this->s3_secret_key, this->s3_bucket_name, this->s3_object_name, - this->s3_url_style + this->s3_url_style, + m_log ); if(! head.SendRequest()) { @@ -154,7 +171,9 @@ S3File::Fstat(struct stat *buff) // than code 200. If xrootd wants us to distinguish between // these cases, head.getResponseCode() is initialized to 0, so // we can check. - fprintf( stderr, "D_FULLDEBUG: failed to send HeadObject command: %lu '%s'\n", head.getResponseCode(), head.getResultString().c_str() ); + std::stringstream ss; + ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" << head.getResultString() << "'"; + m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str()); return -ENOENT; } @@ -220,7 +239,8 @@ S3File::Write(const void *buffer, off_t offset, size_t size) this->s3_secret_key, this->s3_bucket_name, this->s3_object_name, - this->s3_url_style + this->s3_url_style, + m_log ); std::string payload( (char *)buffer, size ); diff --git a/src/S3File.hh b/src/S3File.hh index ed8a8d4..65cb53e 100644 --- a/src/S3File.hh +++ b/src/S3File.hh @@ -1,12 +1,31 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #pragma once -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdSec/XrdSecEntityAttr.hh" -#include "XrdVersion.hh" -#include "XrdOss/XrdOss.hh" #include "S3FileSystem.hh" +#include +#include +#include +#include +#include + #include int parse_path( const S3FileSystem & fs, const char * path, std::string & bucket, std::string & object ); @@ -111,7 +130,7 @@ public: return -ENOSYS; } - int Close(long long *retsz=0); + int Close(long long *retsz=0) override; size_t getContentLength() { return content_length; } time_t getLastModified() { return last_modified; } diff --git a/src/S3FileSystem.cc b/src/S3FileSystem.cc index 65e8bca..ea27094 100644 --- a/src/S3FileSystem.cc +++ b/src/S3FileSystem.cc @@ -1,23 +1,40 @@ -#include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucStream.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdVersion.hh" -#include "S3FileSystem.hh" +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #include "S3Directory.hh" #include "S3File.hh" +#include "S3FileSystem.hh" #include "stl_string_utils.hh" +#include +#include +#include +#include + #include -#include #include +#include #include #include #include #include -#include "stl_string_utils.hh" - S3FileSystem::S3FileSystem(XrdSysLogger *lp, const char *configfn, XrdOucEnv *envP) : m_env(envP), m_log(lp, "s3_") diff --git a/src/S3FileSystem.hh b/src/S3FileSystem.hh index 065c611..975189e 100644 --- a/src/S3FileSystem.hh +++ b/src/S3FileSystem.hh @@ -1,9 +1,27 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + #pragma once -#include "XrdOuc/XrdOucStream.hh" -#include "XrdSec/XrdSecEntity.hh" -#include "XrdVersion.hh" -#include "XrdOss/XrdOss.hh" +#include +#include +#include +#include #include #include diff --git a/src/logging.cc b/src/logging.cc new file mode 100644 index 0000000..845e08c --- /dev/null +++ b/src/logging.cc @@ -0,0 +1,73 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#include "logging.hh" + +#include +#include + +#include + +using namespace XrdHTTPServer; + +std::string +XrdHTTPServer::LogMaskToString(int mask) { + if (mask == LogMask::All) {return "all";} + + bool has_entry = false; + std::stringstream ss; + if (mask & LogMask::Debug) { + ss << "debug"; + has_entry = true; + } + if (mask & LogMask::Info) { + ss << (has_entry ? ", " : "") << "info"; + has_entry = true; + } + if (mask & LogMask::Warning) { + ss << (has_entry ? ", " : "") << "warning"; + has_entry = true; + } + if (mask & LogMask::Error) { + ss << (has_entry ? ", " : "") << "error"; + has_entry = true; + } + return ss.str(); +} + +bool +XrdHTTPServer::ConfigLog(XrdOucStream &conf, XrdSysError &log) { + std::string map_filename; + log.setMsgMask(0); + char *val = nullptr; + if (!(val = conf.GetToken())) { + log.Emsg("Config", "httpserver.trace requires an argument. Usage: httpserver.trace [all|error|warning|info|debug|none]"); + return false; + } + do { + if (!strcmp(val, "all")) {log.setMsgMask(log.getMsgMask() | LogMask::All);} + else if (!strcmp(val, "error")) {log.setMsgMask(log.getMsgMask() | LogMask::Error);} + else if (!strcmp(val, "warning")) {log.setMsgMask(log.getMsgMask() | LogMask::Warning);} + else if (!strcmp(val, "info")) {log.setMsgMask(log.getMsgMask() | LogMask::Info);} + else if (!strcmp(val, "debug")) {log.setMsgMask(log.getMsgMask() | LogMask::Debug);} + else if (!strcmp(val, "none")) {log.setMsgMask(0);} + else {log.Emsg("Config", "scitokens.trace encountered an unknown directive:", val); return false;} + } while ((val = conf.GetToken())); + log.Emsg("Config", "Logging levels enabled -", XrdHTTPServer::LogMaskToString(log.getMsgMask()).c_str()); + return true; +} diff --git a/src/logging.hh b/src/logging.hh new file mode 100644 index 0000000..41a5750 --- /dev/null +++ b/src/logging.hh @@ -0,0 +1,43 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#pragma once + +#include + +class XrdOucStream; +class XrdSysError; + +namespace XrdHTTPServer { + +enum LogMask { + Debug = 0x01, + Info = 0x02, + Warning = 0x04, + Error = 0x08, + All = 0xff +}; + +// Given a bitset based on LogMask, return a human-readable string of the set logging levels. +std::string LogMaskToString(int mask); + +// Given an xrootd configuration object that matched on httpserver.trace, parse +// the remainder of the line and configure the logger appropriately. +bool ConfigLog(XrdOucStream &conf, XrdSysError &log); + +} diff --git a/src/shortfile.cc b/src/shortfile.cc index cd15c08..0931ea2 100644 --- a/src/shortfile.cc +++ b/src/shortfile.cc @@ -1,12 +1,31 @@ +/*************************************************************** + * + * Copyright (C) 2023, HTCondor Team, UW-Madison + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#include "shortfile.hh" + +#include + #include #include #include #include #include -#include -#include "shortfile.hh" - ssize_t full_read( int fd, void * ptr, size_t nbytes ) { diff --git a/src/shortfile.hh b/src/shortfile.hh index 91b68b8..34af1e8 100644 --- a/src/shortfile.hh +++ b/src/shortfile.hh @@ -1,6 +1,23 @@ -#ifndef SHORTFILE_H -#define SHORTFILE_H +/*************************************************************** + * + * Copyright (C) 2023, HTCondor Team, UW-Madison + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ -bool readShortFile( const std::string & fileName, std::string & contents ); +#pragma once -#endif /* SHORTFILE_H */ +#include + +bool readShortFile(const std::string &fileName, std::string &contents); diff --git a/src/stl_string_utils.cc b/src/stl_string_utils.cc index 7b5e9ad..4d1d07a 100644 --- a/src/stl_string_utils.cc +++ b/src/stl_string_utils.cc @@ -1,10 +1,29 @@ -#include -#include +/*************************************************************** + * + * Copyright (C) 2024, HTCondor Team, UW-Madison + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ -#include -#include #include "stl_string_utils.hh" +#include +#include +#include + +#include + std::string substring( const std::string & str, size_t left, size_t right ) { if( right == std::string::npos ) { diff --git a/src/stl_string_utils.hh b/src/stl_string_utils.hh index 6b14d10..324b784 100644 --- a/src/stl_string_utils.hh +++ b/src/stl_string_utils.hh @@ -1,5 +1,24 @@ -#ifndef STL_STRING_UTILS_H -#define STL_STRING_UTILS_H +/*************************************************************** + * + * Copyright (C) 2024, HTCondor Team, UW-Madison + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + +#pragma once + +#include #ifndef CHECK_PRINTF_FORMAT #ifdef __GNUC__ @@ -15,5 +34,3 @@ void toLower( std::string & str); int formatstr(std::string& s, const char* format, ...) CHECK_PRINTF_FORMAT(2,3); int formatstr_cat(std::string& s, const char* format, ...) CHECK_PRINTF_FORMAT(2,3); - -#endif /* STL_STRING_UTILS_H */