Merge pull request #36 from bbockelm/http_auth

Token support for HTTP backend

CMakeLists.txt

@@ -10,6 +10,7 @@ set( CMAKE_BUILD_TYPE Debug )
 
 find_package( Xrootd REQUIRED )
 find_package( CURL REQUIRED )
+find_package( Threads REQUIRED )
 
 include (FindPkgConfig)
 pkg_check_modules(LIBCRYPTO REQUIRED libcrypto)
@@ -56,11 +57,11 @@ endif()
 
 include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS})
 
-add_library(XrdS3 SHARED src/S3File.cc src/S3Directory.cc src/S3AccessInfo.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
-add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
+add_library(XrdS3 SHARED src/S3File.cc src/S3Directory.cc src/S3AccessInfo.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/TokenFile.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
+add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/TokenFile.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 
-target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES} tinyxml2::tinyxml2)
-target_link_libraries(XrdHTTPServer -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES})
+target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES} tinyxml2::tinyxml2 Threads::Threads)
+target_link_libraries(XrdHTTPServer -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES} Threads::Threads)
 
 # The CMake documentation strongly advises against using these macros; instead, the pkg_check_modules
 # is supposed to fill out the full path to ${LIBCRYPTO_LIBRARIES}.  As of cmake 3.26.1, this does not

src/HTTPCommands.cc

@@ -168,12 +168,6 @@ bool HTTPRequest::sendPreparedRequest(const std::string &protocol,
 
 	m_log.Log(XrdHTTPServer::Debug, "SendRequest", "Sending HTTP request",
 			  uri.c_str());
-	CURLcode rv = curl_global_init(CURL_GLOBAL_ALL);
-	if (rv != 0) {
-		this->errorCode = "E_CURL_LIB";
-		this->errorMessage = "curl_global_init() failed.";
-		return false;
-	}
 
 	std::unique_ptr<CURL, decltype(&curl_easy_cleanup)> curl(
 		curl_easy_init(), &curl_easy_cleanup);
@@ -185,7 +179,7 @@ bool HTTPRequest::sendPreparedRequest(const std::string &protocol,
 	}
 
 	char errorBuffer[CURL_ERROR_SIZE];
-	rv = curl_easy_setopt(curl.get(), CURLOPT_ERRORBUFFER, errorBuffer);
+	auto rv = curl_easy_setopt(curl.get(), CURLOPT_ERRORBUFFER, errorBuffer);
 	if (rv != CURLE_OK) {
 		this->errorCode = "E_CURL_LIB";
 		this->errorMessage = "curl_easy_setopt( CURLOPT_ERRORBUFFER ) failed.";
@@ -347,6 +341,24 @@ bool HTTPRequest::sendPreparedRequest(const std::string &protocol,
 		}
 	}
 
+	if (m_token) {
+		const auto iter = headers.find("Authorization");
+		if (iter == headers.end()) {
+			std::string token;
+			if (m_token->Get(token) && !token.empty()) {
+				headers["Authorization"] = "Bearer " + token;
+			} else {
+				errorCode = "E_TOKEN";
+				errorMessage = "failed to load authorization token from file";
+			}
+		}
+	}
+	{
+		const auto iter = headers.find("User-Agent");
+		if (iter == headers.end()) {
+			headers["User-Agent"] = "xrootd-http/devel";
+		}
+	}
 	std::string headerPair;
 	struct curl_slist *header_slist = NULL;
 	for (auto i = headers.begin(); i != headers.end(); ++i) {
@@ -448,6 +460,13 @@ bool HTTPUpload::SendRequest(const std::string &payload, off_t offset,
 	return SendHTTPRequest(payload);
 }
 
+void HTTPRequest::init() {
+	CURLcode rv = curl_global_init(CURL_GLOBAL_ALL);
+	if (rv != 0) {
+		throw std::runtime_error("libcurl failed to initialize");
+	}
+}
+
 // ---------------------------------------------------------------------------
 
 HTTPDownload::~HTTPDownload() {}

src/HTTPCommands.hh

@@ -18,6 +18,8 @@
 
 #pragma once
 
+#include "TokenFile.hh"
+
 #include <map>
 #include <memory>
 #include <string>
@@ -26,9 +28,9 @@ class XrdSysError;
 
 class HTTPRequest {
   public:
-	HTTPRequest(const std::string &hostUrl, XrdSysError &log)
-		: hostUrl(hostUrl), requiresSignature(false), responseCode(0),
-		  includeResponseHeader(false), httpVerb("POST"), m_log(log) {
+	HTTPRequest(const std::string &hostUrl, XrdSysError &log,
+				const TokenFile *token)
+		: hostUrl(hostUrl), m_log(log), m_token(token) {
 		// Parse the URL and populate
 		// What to do if the function returns false?
 		// TODO: Figure out best way to deal with this
@@ -57,6 +59,12 @@ class HTTPRequest {
 		size_t sentSoFar;
 	};
 
+	// Initialize libraries for HTTP.
+	//
+	// Should be called at least once per application from a non-threaded
+	// context.
+	static void init();
+
   protected:
 	bool sendPreparedRequest(const std::string &protocol,
 							 const std::string &uri,
@@ -69,7 +77,7 @@ class HTTPRequest {
 	std::string hostUrl;
 	std::string protocol;
 
-	bool requiresSignature;
+	bool requiresSignature{false};
 	struct timespec signatureTime;
 
 	std::string errorMessage;
@@ -78,18 +86,22 @@ class HTTPRequest {
 	std::string resultString;
 	unsigned long responseCode{0};
 	unsigned long expectedResponseCode = 200;
-	bool includeResponseHeader;
+	bool includeResponseHeader{false};
 
-	std::string httpVerb;
+	std::string httpVerb{"POST"};
 	std::unique_ptr<HTTPRequest::Payload> callback_payload;
 
 	XrdSysError &m_log;
+
+  private:
+	const TokenFile *m_token;
 };
 
 class HTTPUpload : public HTTPRequest {
   public:
-	HTTPUpload(const std::string &h, const std::string &o, XrdSysError &log)
-		: HTTPRequest(h, log), object(o) {
+	HTTPUpload(const std::string &h, const std::string &o, XrdSysError &log,
+			   const TokenFile *token)
+		: HTTPRequest(h, log, token), object(o) {
 		hostUrl = hostUrl + "/" + object;
 	}
 
@@ -105,8 +117,9 @@ class HTTPUpload : public HTTPRequest {
 
 class HTTPDownload : public HTTPRequest {
   public:
-	HTTPDownload(const std::string &h, const std::string &o, XrdSysError &log)
-		: HTTPRequest(h, log), object(o) {
+	HTTPDownload(const std::string &h, const std::string &o, XrdSysError &log,
+				 const TokenFile *token)
+		: HTTPRequest(h, log, token), object(o) {
 		hostUrl = hostUrl + "/" + object;
 	}
 
@@ -120,8 +133,9 @@ class HTTPDownload : public HTTPRequest {
 
 class HTTPHead : public HTTPRequest {
   public:
-	HTTPHead(const std::string &h, const std::string &o, XrdSysError &log)
-		: HTTPRequest(h, log), object(o) {
+	HTTPHead(const std::string &h, const std::string &o, XrdSysError &log,
+			 const TokenFile *token)
+		: HTTPRequest(h, log, token), object(o) {
 		hostUrl = hostUrl + "/" + object;
 	}
 

src/HTTPFile.cc

@@ -111,22 +111,24 @@ int HTTPFile::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) {
 		return rv;
 	}
 
-	// We used to query S3 here to see if the object existed, but of course
-	// if you're creating a file on upload, you don't care.
+	m_object = object;
+	m_hostname = configured_hostname;
+	m_hostUrl = configured_hostUrl;
 
-	this->object = object;
-	this->hostname = configured_hostname;
-	this->hostUrl = configured_hostUrl;
+	if (!Oflag) {
+		struct stat buf;
+		return Fstat(&buf);
+	}
 
 	return 0;
 }
 
 ssize_t HTTPFile::Read(void *buffer, off_t offset, size_t size) {
-	HTTPDownload download(this->hostUrl, this->object, m_log);
+	HTTPDownload download(m_hostUrl, m_object, m_log, m_oss->getToken());
 	m_log.Log(
 		LogMask::Debug, "HTTPFile::Read",
 		"About to perform download from HTTPFile::Read(): hostname / object:",
-		hostname.c_str(), object.c_str());
+		m_hostname.c_str(), m_object.c_str());
 
 	if (!download.SendRequest(offset, size)) {
 		std::stringstream ss;
@@ -142,22 +144,54 @@ ssize_t HTTPFile::Read(void *buffer, off_t offset, size_t size) {
 }
 
 int HTTPFile::Fstat(struct stat *buff) {
+	if (m_stat) {
+		buff->st_mode = 0600 | S_IFREG;
+		buff->st_nlink = 1;
+		buff->st_uid = 1;
+		buff->st_gid = 1;
+		buff->st_size = content_length;
+		buff->st_mtime = last_modified;
+		buff->st_atime = 0;
+		buff->st_ctime = 0;
+		buff->st_dev = 0;
+		buff->st_ino = 0;
+		return 0;
+	}
+
 	m_log.Log(LogMask::Debug, "HTTPFile::Fstat",
-			  "About to perform HTTPFile::Fstat():", hostUrl.c_str(),
-			  object.c_str());
-	HTTPHead head(hostUrl, object, m_log);
+			  "About to perform HTTPFile::Fstat():", m_hostUrl.c_str(),
+			  m_object.c_str());
+	HTTPHead head(m_hostUrl, m_object, m_log, m_oss->getToken());
 
 	if (!head.SendRequest()) {
 		// SendRequest() returns false for all errors, including ones
 		// where the server properly responded with something other
 		// than code 200.  If xrootd wants us to distinguish between
 		// these cases, head.getResponseCode() is initialized to 0, so
 		// we can check.
-		std::stringstream ss;
-		ss << "Failed to send HeadObject command: " << head.getResponseCode()
-		   << "'" << head.getResultString() << "'";
-		m_log.Log(LogMask::Warning, "HTTPFile::Fstat", ss.str().c_str());
-		return -ENOENT;
+		auto httpCode = head.getResponseCode();
+		if (httpCode) {
+			std::stringstream ss;
+			ss << "HEAD command failed: " << head.getResponseCode() << ": "
+			   << head.getResultString();
+			m_log.Log(LogMask::Warning, "HTTPFile::Fstat", ss.str().c_str());
+			switch (httpCode) {
+			case 404:
+				return -ENOENT;
+			case 500:
+				return -EIO;
+			case 403:
+				return -EPERM;
+			default:
+				return -EIO;
+			}
+		} else {
+			std::stringstream ss;
+			ss << "Failed to send HEAD command: " << head.getErrorCode() << ": "
+			   << head.getErrorMessage();
+			m_log.Log(LogMask::Warning, "HTTPFile::Fstat", ss.str().c_str());
+			return -EIO;
+		}
 	}
 
 	std::string headers = head.getResultString();
@@ -197,22 +231,25 @@ int HTTPFile::Fstat(struct stat *buff) {
 		current_newline = next_newline;
 	}
 
-	buff->st_mode = 0600 | S_IFREG;
-	buff->st_nlink = 1;
-	buff->st_uid = 1;
-	buff->st_gid = 1;
-	buff->st_size = this->content_length;
-	buff->st_mtime = this->last_modified;
-	buff->st_atime = 0;
-	buff->st_ctime = 0;
-	buff->st_dev = 0;
-	buff->st_ino = 0;
+	if (buff) {
+		buff->st_mode = 0600 | S_IFREG;
+		buff->st_nlink = 1;
+		buff->st_uid = 1;
+		buff->st_gid = 1;
+		buff->st_size = this->content_length;
+		buff->st_mtime = this->last_modified;
+		buff->st_atime = 0;
+		buff->st_ctime = 0;
+		buff->st_dev = 0;
+		buff->st_ino = 0;
+	}
+	m_stat = true;
 
 	return 0;
 }
 
 ssize_t HTTPFile::Write(const void *buffer, off_t offset, size_t size) {
-	HTTPUpload upload(this->hostUrl, this->object, m_log);
+	HTTPUpload upload(m_hostUrl, m_object, m_log, m_oss->getToken());
 
 	std::string payload((char *)buffer, size);
 	if (!upload.SendRequest(payload, offset, size)) {
@@ -256,6 +293,7 @@ XrdOss *XrdOssGetStorageSystem2(XrdOss *native_oss, XrdSysLogger *Logger,
 	envP->Export("XRDXROOTD_NOPOSC", "1");
 
 	try {
+		HTTPRequest::init();
 		g_http_oss = new HTTPFileSystem(Logger, config_fn, envP);
 		return g_http_oss;
 	} catch (std::runtime_error &re) {

src/HTTPFile.hh

@@ -95,12 +95,14 @@ class HTTPFile : public XrdOssDF {
 	time_t getLastModified() { return last_modified; }
 
   private:
+	bool m_stat{false};
+
 	XrdSysError &m_log;
 	HTTPFileSystem *m_oss;
 
-	std::string hostname;
-	std::string hostUrl;
-	std::string object;
+	std::string m_hostname;
+	std::string m_hostUrl;
+	std::string m_object;
 
 	size_t content_length;
 	time_t last_modified;

src/HTTPFileSystem.cc

@@ -42,7 +42,7 @@ using namespace XrdHTTPServer;
 
 HTTPFileSystem::HTTPFileSystem(XrdSysLogger *lp, const char *configfn,
 							   XrdOucEnv *envP)
-	: m_env(envP), m_log(lp, "httpserver_") {
+	: m_env(envP), m_log(lp, "httpserver_"), m_token("", &m_log) {
 	m_log.Say("------ Initializing the HTTP filesystem plugin.");
 	if (!Config(lp, configfn)) {
 		throw std::runtime_error("Failed to configure HTTP filesystem plugin.");
@@ -86,6 +86,7 @@ bool HTTPFileSystem::Config(XrdSysLogger *lp, const char *configfn) {
 	char *temporary;
 	std::string value;
 	std::string attribute;
+	std::string token_file;
 	Config.Attach(cfgFD);
 	while ((temporary = Config.GetMyFirstWord())) {
 		attribute = temporary;
@@ -109,7 +110,9 @@ bool HTTPFileSystem::Config(XrdSysLogger *lp, const char *configfn) {
 			!handle_required_config(attribute, "httpserver.url_base", value,
 									m_url_base) ||
 			!handle_required_config(attribute, "httpserver.storage_prefix",
-									value, m_storage_prefix)) {
+									value, m_storage_prefix) ||
+			!handle_required_config(attribute, "httpserver.token_file", value,
+									token_file)) {
 			Config.Close();
 			return false;
 		}
@@ -128,6 +131,10 @@ bool HTTPFileSystem::Config(XrdSysLogger *lp, const char *configfn) {
 		}
 	}
 
+	if (!token_file.empty()) {
+		m_token = std::move(TokenFile(token_file, &m_log));
+	}
+
 	int retc = Config.LastError();
 	if (retc) {
 		m_log.Emsg("Config", -retc, "read config file", configfn);
@@ -161,14 +168,7 @@ int HTTPFileSystem::Stat(const char *path, struct stat *buff, int opts,
 		m_log.Emsg("Stat", "Failed to open path:", path);
 	}
 	// Assume that HTTPFile::FStat() doesn't write to buff unless it succeeds.
-	rv = httpFile.Fstat(buff);
-	if (rv != 0) {
-		formatstr(error, "File %s not found.", path);
-		m_log.Emsg("Stat", error.c_str());
-		return -ENOENT;
-	}
-
-	return 0;
+	return httpFile.Fstat(buff);
 }
 
 int HTTPFileSystem::Create(const char *tid, const char *path, mode_t mode,