Simple cleanups of the S3/HTTP backend code

Includes:
- Adding copyright notices to each source file.
- Use XRootD's logging framework and corresponding configuration.
- Re-arrange/cleanup headers
- Fix all compilation warnings identified by Mac OS X's compiler.

CMakeLists.txt

@@ -46,8 +46,8 @@ endif()
 
 include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS})
 
-add_library(XrdS3 SHARED src/S3File.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc)
-add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc)
+add_library(XrdS3 SHARED src/S3File.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
+add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 
 target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES})
 target_link_libraries(XrdHTTPServer -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES})

src/AWSCredential.hh

@@ -1,5 +1,21 @@
-#ifndef AWS_CREDENTIAL_H
-#define AWS_CREDENTIAL_H
+/***************************************************************
+ *
+ * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+#pragma once
 
 class AWSCredential {
 
@@ -19,5 +35,3 @@ class AWSCredential {
         const std::string m_secret_key;
         const std::string m_security_token;
 };
-
-#endif /* AWS_CREDENTIAL_H */

src/AWSv4-impl.cc

@@ -1,3 +1,21 @@
+/***************************************************************
+ *
+ * Copyright (C) 2023, HTCondor team, UW-Madison
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
 /**
  * Utilities for generating pre-signed URLs.
  *
@@ -82,7 +100,7 @@ convertMessageDigestToLowercaseHex(
 	char * buffer = (char *)malloc( (mdLength * 2) + 1 );
 	char * ptr = buffer;
 	for (unsigned int i = 0; i < mdLength; ++i, ptr += 2) {
-		sprintf(ptr, "%02x", messageDigest[i]);
+		snprintf(ptr, 2, "%02x", messageDigest[i]);
 	}
 	hexEncoded.assign(buffer, mdLength * 2);
 	free(buffer);

src/AWSv4-impl.hh

@@ -1,5 +1,25 @@
-#ifndef AWSV4_IMPL_H
-#define AWSV4_IMPL_H
+/***************************************************************
+ *
+ * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+#pragma once
+
+#include <map>
+#include <string>
 
 namespace AWSv4Impl {
 
@@ -27,5 +47,3 @@ createSignature( const std::string & secretAccessKey,
     std::string & signature );
 
 }
-
-#endif /* AWSV4_IMPL_H */

src/HTTPCommands.cc

@@ -1,19 +1,41 @@
-#include <sstream>
+/***************************************************************
+ *
+ * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
 #include <algorithm>
-#include <openssl/hmac.h>
-#include <curl/curl.h>
 #include <cassert>
 #include <cstring>
-#include <memory>
 #include <filesystem>
-
 #include <map>
+#include <memory>
+#include <sstream>
 #include <string>
 
+#include <openssl/hmac.h>
+#include <curl/curl.h>
+#include <XrdSys/XrdSysError.hh>
+
 #include "HTTPCommands.hh"
+#include "logging.hh"
 #include "stl_string_utils.hh"
 #include "shortfile.hh"
 
+using namespace XrdHTTPServer;
+
 //
 // "This function gets called by libcurl as soon as there is data received
 //  that needs to be saved. The size of the data pointed to by ptr is size
@@ -78,7 +100,7 @@ bool HTTPRequest::SendHTTPRequest( const std::string & payload ) {
     if( (protocol != "http") && (protocol != "https") ) {
         this->errorCode = "E_INVALID_SERVICE_URL";
         this->errorMessage = "Service URL not of a known protocol (http[s]).";
-        // dprintf( D_ALWAYS, "Service URL '%s' not of a known protocol (http[s]).\n", serviceURL.c_str() );
+        m_log.Log(LogMask::Warning, "HTTPRequest::SendHTTPRequest", "Host URL '", hostUrl.c_str(), "' not of a known protocol (http[s]).");
         return false;
     }
 
@@ -261,6 +283,12 @@ bool HTTPRequest::sendPreparedRequest(
         return false;
     }
 
+    if (curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1) != CURLE_OK) {
+        this->errorCode = "E_CURL_LIB";
+        this->errorMessage = "curl_easy_setopt( CURLOPT_FOLLOWLOCATION ) failed.";
+        return false;
+    }
+
     //
     // Set security options.
     //
@@ -402,7 +430,7 @@ HTTPUpload::~HTTPUpload() { }
 bool HTTPUpload::SendRequest( const std::string & payload, off_t offset, size_t size ) {
 	if( offset != 0 || size != 0 ) {
 		std::string range;
-		formatstr( range, "bytes=%zu-%zu", offset, offset + size - 1 );
+		formatstr( range, "bytes=%lld-%lld", offset, offset + size - 1 );
 		headers["Range"] = range.c_str();
 	}
 
@@ -417,7 +445,7 @@ HTTPDownload::~HTTPDownload() { }
 bool HTTPDownload::SendRequest( off_t offset, size_t size ) {
 	if( offset != 0 || size != 0 ) {
 		std::string range;
-		formatstr( range, "bytes=%zu-%zu", offset, offset + size - 1 );
+		formatstr( range, "bytes=%lld-%lld", offset, offset + size - 1 );
 		headers["Range"] = range.c_str();
 		this->expectedResponseCode = 206;
 	}

src/HTTPCommands.hh

@@ -1,17 +1,40 @@
-#ifndef HTTP_COMMANDS_H
-#define HTTP_COMMANDS_H
-
+/***************************************************************
+ *
+ * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+#pragma once
+
+#include <map>
+#include <string>
+
+class XrdSysError;
 
 class HTTPRequest {
 public:
     HTTPRequest(
-        const std::string & hostUrl
+        const std::string & hostUrl,
+        XrdSysError &log
     ) :
     hostUrl(hostUrl),
     requiresSignature(false),
     responseCode(0),
     includeResponseHeader(false),
-    httpVerb( "POST" )
+    httpVerb("POST"),
+    m_log(log)
     { 
         // Parse the URL and populate
         // What to do if the function returns false?
@@ -60,15 +83,18 @@ protected:
     bool includeResponseHeader;
 
     std::string httpVerb;
+
+    XrdSysError &m_log;
 };
 
 class HTTPUpload : public HTTPRequest {
 public:
     HTTPUpload(
         const std::string & h,
-        const std::string & o
+        const std::string & o,
+        XrdSysError &log
     ) :
-        HTTPRequest(h),
+        HTTPRequest(h, log),
         object(o)
     { hostUrl = hostUrl + "/" + object; }
 
@@ -85,9 +111,10 @@ class HTTPDownload : public HTTPRequest {
 public:
     HTTPDownload(
         const std::string & h,
-        const std::string & o
+        const std::string & o,
+        XrdSysError &log
     ) :
-        HTTPRequest(h),
+        HTTPRequest(h, log),
         object(o)
     { hostUrl = hostUrl + "/" + object; }
 
@@ -103,9 +130,10 @@ class HTTPHead : public HTTPRequest {
 public:
     HTTPHead(
         const std::string & h,
-        const std::string & o
+        const std::string & o,
+        XrdSysError &log
     ) :
-        HTTPRequest(h),
+        HTTPRequest(h, log),
         object(o)
     { hostUrl = hostUrl + "/" + object; }
 
@@ -116,5 +144,3 @@ public:
 protected:
     std::string object;
 };
-
-#endif /* HTTP_COMMANDS_H */

src/HTTPDirectory.hh

@@ -1,8 +1,27 @@
+/***************************************************************
+ *
+ * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
 #pragma once
 
 #include "XrdOuc/XrdOucEnv.hh"
 #include "XrdOss/XrdOss.hh"
 
+class XrdSysError;
 
 class HTTPDirectory : public XrdOssDF {
 public:
@@ -20,21 +39,21 @@ public:
         return -ENOSYS;
     }
 
-    virtual int Readdir(char *buff, int blen)
+    virtual int Readdir(char *buff, int blen) override
     {
         return -ENOSYS;
     }
 
-    virtual int StatRet(struct stat *statStruct)
+    virtual int StatRet(struct stat *statStruct) override
     {
         return -ENOSYS;
     }
 
-    virtual int Close(long long *retsz=0)
+    virtual int Close(long long *retsz=0) override
     {
         return -ENOSYS;
     }
 
 protected:
-    XrdSysError m_log;
+    XrdSysError &m_log;
 };