Skip to content

S3 Multibucket

S3 Multibucket #35

Triggered via pull request April 2, 2024 14:00
@jhiemstrawiscjhiemstrawisc
synchronize #23
Status Success
Total duration 24s
Artifacts

linter.yml

on: pull_request_target
Run linters
15s
Run linters
Fit to window
Zoom out
Zoom in

Annotations

1 error and 9 warnings
Run linters
This action does not have permission to push to forks. You may want to run it only on `push` events.
Run linters: src/S3Commands.cc#L56
[lint] reported by reviewdog 🐶 Raw Output: src/S3Commands.cc:56:-// Takes in the configured `s3.service_url` and uses the bucket/object requested to generate src/S3Commands.cc:57:-// the virtual host URL, as well as the canonical URI (which is the path to the object). src/S3Commands.cc:58:-bool AmazonRequest::parseURL( const std::string & url, src/S3Commands.cc:59:- std::string & path ) { src/S3Commands.cc:60:- auto i = url.find( "://" ); src/S3Commands.cc:61:- if( i == std::string::npos ) { return false; } src/S3Commands.cc:62:- //protocol = substring( url, 0, i ); src/S3Commands.cc:63:- src/S3Commands.cc:64:- auto j = url.find( "/", i + 3 ); src/S3Commands.cc:65:- if( j == std::string::npos ) { src/S3Commands.cc:66:- if (style == "path") { src/S3Commands.cc:67:- host = substring( url, i + 3 ); src/S3Commands.cc:68:- } else { src/S3Commands.cc:69:- host = bucket + "." + substring( url, i + 3 ); src/S3Commands.cc:70:- } src/S3Commands.cc:71:- src/S3Commands.cc:72:- path = "/" + object; src/S3Commands.cc:73:- return true; src/S3Commands.cc:57:+// Takes in the configured `s3.service_url` and uses the bucket/object requested src/S3Commands.cc:58:+// to generate the virtual host URL, as well as the canonical URI (which is the src/S3Commands.cc:59:+// path to the object). src/S3Commands.cc:60:+bool AmazonRequest::parseURL(const std::string &url, std::string &path) { src/S3Commands.cc:61:+ auto i = url.find("://"); src/S3Commands.cc:62:+ if (i == std::string::npos) { src/S3Commands.cc:63:+ return false; src/S3Commands.cc:64:+ } src/S3Commands.cc:65:+ // protocol = substring( url, 0, i ); src/S3Commands.cc:66:+ src/S3Commands.cc:67:+ auto j = url.find("/", i + 3); src/S3Commands.cc:68:+ if (j == std::string::npos) { src/S3Commands.cc:69:+ if (style == "path") { src/S3Commands.cc:70:+ host = substring(url, i + 3); src/S3Commands.cc:71:+ } else { src/S3Commands.cc:72:+ host = bucket + "." + substring(url, i + 3);
Run linters: src/S3File.cc#L48
[lint] reported by reviewdog 🐶 Raw Output: src/S3File.cc:48:-S3File::S3File(XrdSysError &log, S3FileSystem *oss) : src/S3File.cc:49:- m_log(log), src/S3File.cc:50:- m_oss(oss), src/S3File.cc:51:- content_length(0), src/S3File.cc:52:- last_modified(0) src/S3File.cc:53:-{} src/S3File.cc:54:- src/S3File.cc:55:- src/S3File.cc:56:-int src/S3File.cc:57:-parse_path( const S3FileSystem & fs, const char * fullPath, std::string & exposedPath, std::string & object ) { src/S3File.cc:58:- // src/S3File.cc:59:- // Check the path for validity. src/S3File.cc:60:- // src/S3File.cc:61:- std::filesystem::path p(fullPath); src/S3File.cc:62:- auto pathComponents = p.begin(); src/S3File.cc:63:- src/S3File.cc:64:- // Iterate through components of the fullPath until we either find a match src/S3File.cc:65:- // or we've reached the end of the path. src/S3File.cc:66:- std::filesystem::path currentPath = *pathComponents; src/S3File.cc:67:- while (pathComponents != p.end()) { src/S3File.cc:68:- if (fs.exposedPathExists(currentPath.string())) { src/S3File.cc:69:- exposedPath = currentPath.string(); src/S3File.cc:70:- break; src/S3File.cc:71:- } src/S3File.cc:72:- ++pathComponents; src/S3File.cc:73:- if (pathComponents != p.end()) { src/S3File.cc:74:- currentPath /= *pathComponents; src/S3File.cc:75:- } else { src/S3File.cc:76:- return -ENOENT; src/S3File.cc:77:- } src/S3File.cc:48:+S3File::S3File(XrdSysError &log, S3FileSystem *oss) src/S3File.cc:49:+ : m_log(log), m_oss(oss), content_length(0), last_modified(0) {} src/S3File.cc:50:+ src/S3File.cc:51:+int parse_path(const S3FileSystem &fs, const char *fullPath, src/S3File.cc:52:+ std::string &exposedPath, std::string &object) { src/S3File.cc:53:+ // src/S3File.cc:54:+ // Check the path for validity. src/S3File.cc:55:+ // src/S3File.cc:56:+ std::filesystem::path p(fullPath); src/S3File.cc:57:+ auto pathComponents = p.begin(); src/S3File.cc:58:+ src/S3File.cc:59:+ // Iterate through components of the fullPath until we either find a match src/S3File.cc:60:+ // or we've reached the end of the path. src/S3File.cc:61:+ std::filesystem::path currentPath = *pathComponents; src/S3File.cc:62:+ while (pathComponents != p.end()) { src/S3File.cc:63:+ if (fs.exposedPathExists(currentPath.string())) { src/S3File.cc:64:+ exposedPath = currentPath.string(); src/S3File.cc:65:+ break;
Run linters: src/S3File.cc#L96
[lint] reported by reviewdog 🐶 Raw Output: src/S3File.cc:96:-int src/S3File.cc:97:-S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env) src/S3File.cc:98:-{ src/S3File.cc:99:- std::string exposedPath, object; src/S3File.cc:100:- int rv = parse_path( * m_oss, path, exposedPath, object ); src/S3File.cc:101:- if( rv != 0 ) { return rv; } src/S3File.cc:102:- if(!m_oss->exposedPathExists(exposedPath)) return -ENOENT; src/S3File.cc:103:- src/S3File.cc:104:- std::string configured_s3_region = m_oss->getS3Region(exposedPath); src/S3File.cc:105:- std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath); src/S3File.cc:106:- std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath); src/S3File.cc:107:- std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath); src/S3File.cc:108:- std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath); src/S3File.cc:109:- src/S3File.cc:110:- // We used to query S3 here to see if the object existed, but of course src/S3File.cc:111:- // if you're creating a file on upload, you don't care. src/S3File.cc:112:- src/S3File.cc:113:- this->s3_object_name = object; src/S3File.cc:114:- this->s3_bucket_name = configured_s3_bucket_name; src/S3File.cc:115:- this->s3_service_url = configured_s3_service_url; src/S3File.cc:116:- this->s3_access_key = configured_s3_access_key; src/S3File.cc:117:- this->s3_secret_key = configured_s3_secret_key; src/S3File.cc:118:- std::string configured_s3_url_style = m_oss->getS3URLStyle(); src/S3File.cc:119:- src/S3File.cc:120:- src/S3File.cc:121:- // We used to query S3 here to see if the object existed, but of course src/S3File.cc:122:- // if you're creating a file on upload, you don't care. src/S3File.cc:123:- src/S3File.cc:124:- this->s3_object_name = object; src/S3File.cc:125:- this->s3_bucket_name = configured_s3_bucket_name; src/S3File.cc:126:- this->s3_service_url = configured_s3_service_url; src/S3File.cc:127:- this->s3_access_key = configured_s3_access_key; src/S3File.cc:128:- this->s3_secret_key = configured_s3_secret_key; src/S3File.cc:129:- this->s3_url_style = configured_s3_url_style; src/S3File.cc:130:- return 0; src/S3File.cc:89:+ return 0;
Run linters: src/S3File.cc#L186
[lint] reported by reviewdog 🐶 Raw Output: src/S3File.cc:186:- std::string headers = head.getResultString(); src/S3File.cc:187:- src/S3File.cc:188:- std::string line; src/S3File.cc:189:- size_t current_newline = 0; src/S3File.cc:190:- size_t next_newline = std::string::npos; src/S3File.cc:191:- size_t last_character = headers.size(); src/S3File.cc:192:- while( current_newline != std::string::npos && current_newline != last_character - 1 ) { src/S3File.cc:193:- next_newline = headers.find( "\r\n", current_newline + 2); src/S3File.cc:194:- line = substring( headers, current_newline + 2, next_newline ); src/S3File.cc:195:- src/S3File.cc:196:- size_t colon = line.find(":"); src/S3File.cc:197:- if( colon != std::string::npos && colon != line.size() ) { src/S3File.cc:198:- std::string attr = substring( line, 0, colon ); src/S3File.cc:199:- std::string value = substring( line, colon + 1 ); src/S3File.cc:200:- trim(value); src/S3File.cc:201:- toLower(attr); src/S3File.cc:202:- src/S3File.cc:203:- if( attr == "content-length" ) { src/S3File.cc:204:- this->content_length = std::stol(value); src/S3File.cc:205:- } else if( attr == "last-modified" ) { src/S3File.cc:206:- struct tm t; src/S3File.cc:207:- char * eos = strptime( value.c_str(), src/S3File.cc:208:- "%a, %d %b %Y %T %Z", src/S3File.cc:209:- & t ); src/S3File.cc:210:- if( eos == & value.c_str()[value.size()] ) { src/S3File.cc:211:- time_t epoch = timegm(& t); src/S3File.cc:212:- if( epoch != -1 ) { src/S3File.cc:213:- this->last_modified = epoch; src/S3File.cc:214:- } src/S3File.cc:215:- } src/S3File.cc:216:- } src/S3File.cc:147:+int S3File::Fstat(struct stat *buff) { src/S3File.cc:148:+ AmazonS3Head head(this->s3_service_url, this->s3_access_key, src/S3File.cc:149:+ this->s3_secret_key, this->s3_bucket_name, src/S3File.cc:150:+ this->s3_object_name, this->s3_url_style, m_log); src/S3File.cc:151:+ src/S3File.cc:152:+ if (!head.SendRequest()) { src/S3File.cc:153:+ // SendRequest() returns false for all errors, including ones src/S3File.cc:154:+ // where the server properly responded with something other src/S3File.cc:155:+ // than code 200. If xrootd wants us to distinguish between src/S3File.cc:156:+ // these cases, head.getResponseCode() is initialized to 0, so src/S3File.cc:157:+ // we can check. src/S3File.cc:158:+ std::stringstream ss; src/S3File.cc:159:+ ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'" src/S3File.cc:160:+ << head.getResultString() << "'"; src/S3File.cc:161:+ m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str()); src/S3File.cc:162:+ return -ENOENT; src/S3File.cc:163:+ } src/S3File.cc:164:+ src/S3File.cc:165:+ std::string headers = head.getResultString(); src/S3File.cc:166:+ src/S3File.cc:167:+ std::string line; src/S3File.cc:168:+ size_t current_newline = 0; src/S3File.cc:169:+ size_t next_newline = std::string::npos; src/S3File.cc:170:+ size_t last_character = headers.size(); src/S3File.cc:171:+ while (current_newline != std::string::npos && src/S3File.cc:172:+ current_newline != last_character - 1) { src/S3File.cc:173:+ next_newline = headers.find("\r\n", current_newline + 2); src/S3File.cc:174:+ line = substring(headers, current_newline + 2, next_newline); src/S3File.cc:175:+ src/S3File.cc:176:+ size_t colon = line.find(":"); src/S3File.cc:177:+ if (colon != std::string::npos && colon != line.size()) { src/S3File.cc:178:+ std::string attr = substring(line, 0, colon); src/S3File.cc:179:+ std::string value = substring(line, colon + 1); src/S3File.cc:180:+ trim(value); src/S3File.cc:181:+ toLower(attr); src/S3File.cc:182:+ src/S3File.cc:183:+ if (attr == "content-length") { src/S3
Run linters: src/S3FileSystem.cc#L65
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:65:- } src/S3FileSystem.cc:66:- return true; src/S3FileSystem.cc:67:-} src/S3FileSystem.cc:68:- src/S3FileSystem.cc:69:- src/S3FileSystem.cc:70:-bool src/S3FileSystem.cc:71:-S3FileSystem::Config(XrdSysLogger *lp, const char *configfn) src/S3FileSystem.cc:72:-{ src/S3FileSystem.cc:73:- XrdOucEnv myEnv; src/S3FileSystem.cc:74:- XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> "); src/S3FileSystem.cc:75:- src/S3FileSystem.cc:76:- int cfgFD = open(configfn, O_RDONLY, 0); src/S3FileSystem.cc:77:- if (cfgFD < 0) { src/S3FileSystem.cc:78:- m_log.Emsg("Config", errno, "open config file", configfn); src/S3FileSystem.cc:85:+ } src/S3FileSystem.cc:86:+ if (newAccessInfo->getS3Region().empty()) { src/S3FileSystem.cc:87:+ m_log.Emsg("Config", "s3.region not specified");
Run linters: src/S3FileSystem.cc#L81
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:81:- src/S3FileSystem.cc:82:- char * temporary; src/S3FileSystem.cc:83:- std::string value; src/S3FileSystem.cc:84:- std::string attribute; src/S3FileSystem.cc:85:- Config.Attach(cfgFD); src/S3FileSystem.cc:86:- S3AccessInfo *newAccessInfo = new S3AccessInfo(); src/S3FileSystem.cc:87:- std::string exposedPath; src/S3FileSystem.cc:88:- while ((temporary = Config.GetMyFirstWord())) { src/S3FileSystem.cc:89:- attribute = temporary; src/S3FileSystem.cc:90:- temporary = Config.GetWord(); src/S3FileSystem.cc:91:- if(attribute == "s3.end") { src/S3FileSystem.cc:92:- s3_access_map[exposedPath] = newAccessInfo; src/S3FileSystem.cc:93:- if(newAccessInfo->getS3ServiceName().empty()) { src/S3FileSystem.cc:94:- m_log.Emsg("Config", "s3.service_name not specified"); src/S3FileSystem.cc:95:- return false; src/S3FileSystem.cc:96:- } src/S3FileSystem.cc:97:- if(newAccessInfo->getS3Region().empty()) { src/S3FileSystem.cc:98:- m_log.Emsg("Config", "s3.region not specified"); src/S3FileSystem.cc:99:- return false; src/S3FileSystem.cc:100:- } src/S3FileSystem.cc:101:- newAccessInfo = new S3AccessInfo(); src/S3FileSystem.cc:102:- exposedPath = ""; src/S3FileSystem.cc:103:- continue; src/S3FileSystem.cc:104:- } src/S3FileSystem.cc:105:- if(! temporary) { continue; } src/S3FileSystem.cc:106:- value = temporary; src/S3FileSystem.cc:107:- src/S3FileSystem.cc:108:- if(!handle_required_config("s3.path_name",value) ) { Config.Close(); return false; } src/S3FileSystem.cc:109:- if(!handle_required_config("s3.bucket_name",value) ) { Config.Close(); return false; } src/S3FileSystem.cc:110:- if(!handle_required_config("s3.service_name",value) ) { Config.Close(); return false; } src/S3FileSystem.cc:111:- if(!handle_required_config("s3.region", value ) ) { Config.Close(); return false; } src/S3FileSystem.cc:112:- if(!handle_required_config("s3.service_url", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:113:- if(!handle_required_config("s3.access_key_file", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:114:- if(!handle_required_config("s3.secret_key_file", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:115:- if(!handle_required_config("s3.url_style", value) ) { Config.Close(); return false; } src/S3FileSystem.cc:116:- src/S3FileSystem.cc:117:- if (attribute == "s3.path_name") { src/S3FileSystem.cc:118:- // Normalize paths so that they all start with / src/S3FileSystem.cc:119:- if (value[0] != '/') { src/S3FileSystem.cc:120:- exposedPath = "/" + value; src/S3FileSystem.cc:121:- } else { src/S3FileSystem.cc:122:- exposedPath = value; src/S3FileSystem.cc:123:- } src/S3FileSystem.cc:124:- } src/S3FileSystem.cc:125:- else if(attribute == "s3.bucket_name") newAccessInfo->setS3BucketName(value); src/S3FileSystem.cc:126:- else if(attribute == "s3.service_name") newAccessInfo->setS3ServiceName(value); src/S3FileSystem.cc:127:- else if(attribute == "s3.region") newAccessInfo->setS3Region(value); src/S3FileSystem.cc:128:- else if(attribute == "s3.access_key_file") newAccessInfo->setS3AccessKeyFile(value); src/S3FileSystem.cc:129:- else if(attribute == "s3.secret_key_file") newAccessInfo->setS3SecretKeyFile(value); src/S3FileSystem.cc:130:- else if(attribute == "s3.service_url") newAccessInfo->setS3ServiceUrl(value); src/S3FileSystem.cc:131:- else if(attribute == "s3.url_style") this->s3_url_style = value; src/S3FileSystem.cc:132:- src/S3FileSystem.cc:94:+ if (!temporary) { src/S3FileSystem.cc:95:+ continue;
Run linters: src/S3FileSystem.cc#L135
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:135:- if( this->s3_url_style.empty() ) { src/S3FileSystem.cc:136:- m_log.Emsg("Config", "s3.url_style not specified"); src/S3FileSystem.cc:137:- return false; src/S3FileSystem.cc:138:- } else { src/S3FileSystem.cc:139:- // We want this to be case-insensitive. src/S3FileSystem.cc:140:- toLower( this->s3_url_style ); src/S3FileSystem.cc:99:+ if (!handle_required_config("s3.path_name", value)) { src/S3FileSystem.cc:100:+ Config.Close(); src/S3FileSystem.cc:101:+ return false;
Run linters: src/S3FileSystem.cc#L199
[lint] reported by reviewdog 🐶 Raw Output: src/S3FileSystem.cc:199:-int src/S3FileSystem.cc:200:-S3FileSystem::Create( const char *tid, const char *path, mode_t mode, src/S3FileSystem.cc:201:- XrdOucEnv &env, int opts ) src/S3FileSystem.cc:202:-{ src/S3FileSystem.cc:203:- // Is path valid? src/S3FileSystem.cc:204:- std::string exposedPath, object; src/S3FileSystem.cc:205:- int rv = parse_path( * this, path, exposedPath, object ); src/S3FileSystem.cc:206:- if( rv != 0 ) { return rv; } src/S3FileSystem.cc:207:- src/S3FileSystem.cc:208:- // src/S3FileSystem.cc:209:- // We could instead invoke the upload mchinery directly to create a src/S3FileSystem.cc:210:- // 0-byte file, but it seems smarter to remove a round-trip (in src/S3FileSystem.cc:211:- // S3File::Open(), checking if the file exists) than to add one src/S3FileSystem.cc:212:- // (here, creating the file if it doesn't exist). src/S3FileSystem.cc:213:- // src/S3FileSystem.cc:214:- src/S3FileSystem.cc:215:- return 0; src/S3FileSystem.cc:211:+int S3FileSystem::Create(const char *tid, const char *path, mode_t mode, src/S3FileSystem.cc:212:+ XrdOucEnv &env, int opts) { src/S3FileSystem.cc:213:+ // Is path valid? src/S3FileSystem.cc:214:+ std::string exposedPath, object; src/S3FileSystem.cc:215:+ int rv = parse_path(*this, path, exposedPath, object); src/S3FileSystem.cc:216:+ if (rv != 0) { src/S3FileSystem.cc:217:+ return rv; src/S3FileSystem.cc:218:+ } src/S3FileSystem.cc:219:+ src/S3FileSystem.cc:220:+ // src/S3FileSystem.cc:221:+ // We could instead invoke the upload mchinery directly to create a src/S3FileSystem.cc:222:+ // 0-byte file, but it seems smarter to remove a round-trip (in src/S3FileSystem.cc:223:+ // S3File::Open(), checking if the file exists) than to add one src/S3FileSystem.cc:224:+ // (here, creating the file if it doesn't exist). src/S3FileSystem.cc:225:+ // src/S3FileSystem.cc:226:+ src/S3FileSystem.cc:227:+ return 0;
Run linters
Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: actions/checkout@v3, wearerequired/lint-action@v2. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/.