S3 Multibucket

CMakeLists.txt

@@ -46,7 +46,7 @@ endif()
 
 include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS})
 
-add_library(XrdS3 SHARED src/S3File.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
+add_library(XrdS3 SHARED src/S3File.cc src/S3AccessInfo.cc src/S3FileSystem.cc src/AWSv4-impl.cc src/S3Commands.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 add_library(XrdHTTPServer SHARED src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 
 target_link_libraries(XrdS3 -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${CURL_LIBRARIES} ${LIBCRYPTO_LIBRARIES})

README.md

@@ -73,6 +73,8 @@ ofs.osslib libXrdHTTPServer.so
 # Upon last testing, the plugin did not yet work in async mode
 xrootd.async off
 
+
+
 # Configure the upstream HTTP server that XRootD is to treat as a filesystem
 httpserver.host_name <hostname of HTTP server>
 httpserver.host_url <host url>
@@ -127,13 +129,35 @@ ofs.osslib libXrdS3.so
 # Upon last testing, the plugin did not yet work in async mode
 xrootd.async off
 
-# Configure the upstream HTTP server that XRootD is to treat as a filesystem
-s3.service_name     <s3 service name, eg s3.amazonaws.com>
-s3.region           <s3 region, eg us-east-1>
-s3.service_url      <s3 service url, eg https://s3.us-east-1.amazonaws.com>
-# For buckets requiring an access/secret key:
-# s3.access_key_file  </path/to/access_key>
-# s3.secret_key_file  </path/to/secret_key>
+#example url
+#https://<origin url>/my-magic-path/bar/foo
+# these must be in this order to allow parsing of multiple entries
+# To export a bucket requiring an access/private key:
+s3.begin
+s3.path_name        my-magic-path
+s3.bucket_name      hubzero-private-rich
+s3.service_name     s3.amazonaws.com
+s3.region           us-east-1
+s3.access_key_file  /xrootd-dev/access-key
+s3.secret_key_file  /xrootd-dev/secret-key
+s3.service_url      https://s3.us-east-1.amazonaws.com
+s3.end
+
+# To export an unauthenticated (public) bucket, remove
+# the key-related directives
+s3.begin
+s3.path_name        my-other-magic-path
+s3.bucket_name      hubzero-private-rich-2
+s3.service_name     s3.amazonaws.com
+s3.region           us-east-1
+s3.service_url      https://s3.us-east-1.amazonaws.com
+s3.end
+
+# Specify the path style for URL queries at the endpoint. Valid
+# options are `path` and `virtual`, where path corresponds to URLs
+# like `https://my-service-url.com/bucket/object` and virtual
+# corresponds to URLs like `https://bucket.my-service-url.com/object`
+s3.url_style        virtual
 ```
 
 

src/S3AccessInfo.cc

@@ -0,0 +1,51 @@
+//
+// Created by Rich Wellner on 2/29/24.
+//
+
+#include "S3AccessInfo.hh"
+
+const std::string &S3AccessInfo::getS3BucketName() const {
+  return s3_bucket_name;
+}
+
+void S3AccessInfo::setS3BucketName(const std::string &s3BucketName) {
+  s3_bucket_name = s3BucketName;
+}
+
+const std::string &S3AccessInfo::getS3ServiceName() const {
+  return s3_service_name;
+}
+
+void S3AccessInfo::setS3ServiceName(const std::string &s3ServiceName) {
+  s3_service_name = s3ServiceName;
+}
+
+const std::string &S3AccessInfo::getS3Region() const { return s3_region; }
+
+void S3AccessInfo::setS3Region(const std::string &s3Region) {
+  s3_region = s3Region;
+}
+
+const std::string &S3AccessInfo::getS3ServiceUrl() const {
+  return s3_service_url;
+}
+
+void S3AccessInfo::setS3ServiceUrl(const std::string &s3ServiceUrl) {
+  s3_service_url = s3ServiceUrl;
+}
+
+const std::string &S3AccessInfo::getS3AccessKeyFile() const {
+  return s3_access_key_file;
+}
+
+void S3AccessInfo::setS3AccessKeyFile(const std::string &s3AccessKeyFile) {
+  s3_access_key_file = s3AccessKeyFile;
+}
+
+const std::string &S3AccessInfo::getS3SecretKeyFile() const {
+  return s3_secret_key_file;
+}
+
+void S3AccessInfo::setS3SecretKeyFile(const std::string &s3SecretKeyFile) {
+  s3_secret_key_file = s3SecretKeyFile;
+}

src/S3AccessInfo.hh

@@ -0,0 +1,46 @@
+//
+// Created by Rich Wellner on 2/29/24.
+//
+
+#ifndef XROOTD_S3_HTTP_S3ACCESSINFO_HH
+#define XROOTD_S3_HTTP_S3ACCESSINFO_HH
+
+
+#include <string>
+
+class S3AccessInfo {
+public:
+    const std::string &getS3BucketName() const;
+
+    void setS3BucketName(const std::string &s3BucketName);
+
+    const std::string &getS3ServiceName() const;
+
+    void setS3ServiceName(const std::string &s3ServiceName);
+
+    const std::string &getS3Region() const;
+
+    void setS3Region(const std::string &s3Region);
+
+    const std::string &getS3ServiceUrl() const;
+
+    void setS3ServiceUrl(const std::string &s3ServiceUrl);
+
+    const std::string &getS3AccessKeyFile() const;
+
+    void setS3AccessKeyFile(const std::string &s3AccessKeyFile);
+
+    const std::string &getS3SecretKeyFile() const;
+
+    void setS3SecretKeyFile(const std::string &s3SecretKeyFile);
+
+private:
+    std::string s3_bucket_name;
+    std::string s3_service_name;
+    std::string s3_region;
+    std::string s3_service_url;
+    std::string s3_access_key_file;
+    std::string s3_secret_key_file;
+};
+
+#endif //XROOTD_S3_HTTP_S3ACCESSINFO_HH

src/S3Commands.cc

@@ -53,13 +53,10 @@
     return AWSv4Impl::canonicalizeQueryString( query_parameters );
 }
 
 // Takes in the configured `s3.service_url` and uses the bucket/object requested to generate
 // the virtual host URL, as well as the canonical URI (which is the path to the object).
-bool AmazonRequest::parseURL(	const std::string & url,
-				const std::string & bucket,
-				const std::string & object,
-				std::string & host,
-				std::string & path ) {
+bool AmazonRequest::parseURL( const std::string & url,
+                              std::string & path ) {
     auto i = url.find( "://" );
     if( i == std::string::npos ) { return false; }
     //protocol = substring( url, 0, i );

src/S3Commands.hh

@@ -46,7 +46,7 @@ public:
         // Start off by parsing the hostUrl, which we use in conjunction with the bucket to fill in the host (for setting host header).
         // For example, if the incoming hostUrl (which we get from config) is "https://my-url.com:443", the bucket is "my-bucket", and
         // the object is "my-object", then the host will be "my-bucket.my-url.com:443" and the canonicalURI will be "/my-object".
-        if (! parseURL(hostUrl, b, o, host, canonicalURI)) {
+        if (! parseURL(hostUrl, canonicalURI)) {
             errorCode = "E_INVALID_SERVICE_URL";
             errorMessage = "Failed to parse host and canonicalURI from service URL.";
         }
@@ -75,9 +75,9 @@ public:
         return &secretKeyFile; }
 
     bool parseURL(	const std::string & url,
-            const std::string & bucket,
-            const std::string & object,
-            std::string & host,
+            //const std::string & bucket,
+            //const std::string & object,
+            //std::string & host,
             std::string & path );
 
     virtual bool SendRequest();

src/S3File.cc

@@ -45,7 +45,7 @@
 
 XrdVERSIONINFO(XrdOssGetFileSystem, S3);
 
 S3File::S3File(XrdSysError &log, S3FileSystem *oss) :
     m_log(log),
     m_oss(oss),
     content_length(0),
@@ -54,70 +54,75 @@
 
 
 int
-parse_path( const S3FileSystem & fs, const char * path, std::string & bucket, std::string & object ) {
-    const std::string & configured_s3_service_name = fs.getS3ServiceName();
-    const std::string & configured_s3_region = fs.getS3Region();
-
+parse_path( const S3FileSystem & fs, const char * fullPath, std::string & exposedPath, std::string & object ) {
     //
     // Check the path for validity.
     //
-    std::filesystem::path p(path);
+    std::filesystem::path p(fullPath);
     auto pathComponents = p.begin();
 
-    ++pathComponents;
-    if(pathComponents == p.end()) { return -ENOENT; }
-    if (* pathComponents != configured_s3_service_name) {
-        return -ENOENT;
-    }
-
-    ++pathComponents;
-    if (pathComponents == p.end()) { return -ENOENT; }
-    if (*pathComponents != configured_s3_region) {
-        return -ENOENT;
+    // Iterate through components of the fullPath until we either find a match
+    // or we've reached the end of the path.
+    std::filesystem::path currentPath = *pathComponents;
+    while (pathComponents != p.end()) {
+        if (fs.exposedPathExists(currentPath.string())) {
+            exposedPath = currentPath.string();
+            break;
+        }
+        ++pathComponents;
+        if (pathComponents != p.end()) {
+            currentPath /= *pathComponents;
+        } else {
+            return -ENOENT;
+        }
     }
 
-    ++pathComponents;
-    if (pathComponents == p.end()) { return -ENOENT; }
-    bucket = *pathComponents;
-
     // Objects names may contain path separators.
     ++pathComponents;
-    if (pathComponents == p.end()) { return -ENOENT; }
+    if( pathComponents == p.end() ) { return -ENOENT; }
 
     std::filesystem::path objectPath = *pathComponents++;
-    for ( ; pathComponents != p.end(); ++pathComponents) {
-        objectPath /= (*pathComponents);
+    for( ; pathComponents != p.end(); ++pathComponents ) {
+        objectPath /= (* pathComponents);
     }
     object = objectPath.string();
 
+    fprintf( stderr, "object = %s\n", object.c_str() );
+
     return 0;
 }
 
 
 int
 S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
 {
-    std::string configured_s3_region = m_oss->getS3Region();
-
-    //
-    // Check the path for validity.
-    //
-    std::string bucket, object;
-    int rv = parse_path( * m_oss, path, bucket, object );
+    std::string exposedPath, object;
+    int rv = parse_path( * m_oss, path, exposedPath, object );
     if( rv != 0 ) { return rv; }
+    if(!m_oss->exposedPathExists(exposedPath)) return -ENOENT;
 
+    std::string configured_s3_region = m_oss->getS3Region(exposedPath);
+    std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath);
+    std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath);
+    std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath);
+    std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath);
 
-    std::string configured_s3_service_url = m_oss->getS3ServiceURL();
-    std::string configured_s3_access_key = m_oss->getS3AccessKeyFile();
-    std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile();
+    // We used to query S3 here to see if the object existed, but of course
+    // if you're creating a file on upload, you don't care.
+
+    this->s3_object_name = object;
+    this->s3_bucket_name = configured_s3_bucket_name;
+    this->s3_service_url = configured_s3_service_url;
+    this->s3_access_key = configured_s3_access_key;
+    this->s3_secret_key = configured_s3_secret_key;
     std::string configured_s3_url_style = m_oss->getS3URLStyle();
 
 
     // We used to query S3 here to see if the object existed, but of course
     // if you're creating a file on upload, you don't care.
 
     this->s3_object_name = object;
-    this->s3_bucket_name = bucket;
+    this->s3_bucket_name = configured_s3_bucket_name;
     this->s3_service_url = configured_s3_service_url;
     this->s3_access_key = configured_s3_access_key;
     this->s3_secret_key = configured_s3_secret_key;
@@ -178,7 +183,7 @@
     }
 
 
     std::string headers = head.getResultString();
 
     std::string line;
     size_t current_newline = 0;
@@ -186,7 +191,7 @@
     size_t last_character = headers.size();
     while( current_newline != std::string::npos && current_newline != last_character - 1 ) {
         next_newline = headers.find( "\r\n", current_newline + 2);
-        std::string line = substring( headers, current_newline + 2, next_newline );
+        line = substring( headers, current_newline + 2, next_newline );
 
         size_t colon = line.find(":");
         if( colon != std::string::npos && colon != line.size() ) {

src/S3File.hh

@@ -28,7 +28,7 @@
 
 #include <memory>
 
-int parse_path( const S3FileSystem & fs, const char * path, std::string & bucket, std::string & object );
+int parse_path( const S3FileSystem & fs, const char * path, std::string & exposedPath, std::string & object );
 
 class S3File : public XrdOssDF {
 public: