S3 Multibucket #35
Triggered via pull request
April 2, 2024 14:00
jhiemstrawisc
synchronize
#23
Status
Success
Total duration
24s
Artifacts
–
Annotations
1 error and 9 warnings
Run linters
This action does not have permission to push to forks. You may want to run it only on `push` events.
|
Run linters:
src/S3Commands.cc#L56
[lint] reported by reviewdog 🐶
Raw Output:
src/S3Commands.cc:56:-// Takes in the configured `s3.service_url` and uses the bucket/object requested to generate
src/S3Commands.cc:57:-// the virtual host URL, as well as the canonical URI (which is the path to the object).
src/S3Commands.cc:58:-bool AmazonRequest::parseURL( const std::string & url,
src/S3Commands.cc:59:- std::string & path ) {
src/S3Commands.cc:60:- auto i = url.find( "://" );
src/S3Commands.cc:61:- if( i == std::string::npos ) { return false; }
src/S3Commands.cc:62:- //protocol = substring( url, 0, i );
src/S3Commands.cc:63:-
src/S3Commands.cc:64:- auto j = url.find( "/", i + 3 );
src/S3Commands.cc:65:- if( j == std::string::npos ) {
src/S3Commands.cc:66:- if (style == "path") {
src/S3Commands.cc:67:- host = substring( url, i + 3 );
src/S3Commands.cc:68:- } else {
src/S3Commands.cc:69:- host = bucket + "." + substring( url, i + 3 );
src/S3Commands.cc:70:- }
src/S3Commands.cc:71:-
src/S3Commands.cc:72:- path = "/" + object;
src/S3Commands.cc:73:- return true;
src/S3Commands.cc:57:+// Takes in the configured `s3.service_url` and uses the bucket/object requested
src/S3Commands.cc:58:+// to generate the virtual host URL, as well as the canonical URI (which is the
src/S3Commands.cc:59:+// path to the object).
src/S3Commands.cc:60:+bool AmazonRequest::parseURL(const std::string &url, std::string &path) {
src/S3Commands.cc:61:+ auto i = url.find("://");
src/S3Commands.cc:62:+ if (i == std::string::npos) {
src/S3Commands.cc:63:+ return false;
src/S3Commands.cc:64:+ }
src/S3Commands.cc:65:+ // protocol = substring( url, 0, i );
src/S3Commands.cc:66:+
src/S3Commands.cc:67:+ auto j = url.find("/", i + 3);
src/S3Commands.cc:68:+ if (j == std::string::npos) {
src/S3Commands.cc:69:+ if (style == "path") {
src/S3Commands.cc:70:+ host = substring(url, i + 3);
src/S3Commands.cc:71:+ } else {
src/S3Commands.cc:72:+ host = bucket + "." + substring(url, i + 3);
|
Run linters:
src/S3File.cc#L48
[lint] reported by reviewdog 🐶
Raw Output:
src/S3File.cc:48:-S3File::S3File(XrdSysError &log, S3FileSystem *oss) :
src/S3File.cc:49:- m_log(log),
src/S3File.cc:50:- m_oss(oss),
src/S3File.cc:51:- content_length(0),
src/S3File.cc:52:- last_modified(0)
src/S3File.cc:53:-{}
src/S3File.cc:54:-
src/S3File.cc:55:-
src/S3File.cc:56:-int
src/S3File.cc:57:-parse_path( const S3FileSystem & fs, const char * fullPath, std::string & exposedPath, std::string & object ) {
src/S3File.cc:58:- //
src/S3File.cc:59:- // Check the path for validity.
src/S3File.cc:60:- //
src/S3File.cc:61:- std::filesystem::path p(fullPath);
src/S3File.cc:62:- auto pathComponents = p.begin();
src/S3File.cc:63:-
src/S3File.cc:64:- // Iterate through components of the fullPath until we either find a match
src/S3File.cc:65:- // or we've reached the end of the path.
src/S3File.cc:66:- std::filesystem::path currentPath = *pathComponents;
src/S3File.cc:67:- while (pathComponents != p.end()) {
src/S3File.cc:68:- if (fs.exposedPathExists(currentPath.string())) {
src/S3File.cc:69:- exposedPath = currentPath.string();
src/S3File.cc:70:- break;
src/S3File.cc:71:- }
src/S3File.cc:72:- ++pathComponents;
src/S3File.cc:73:- if (pathComponents != p.end()) {
src/S3File.cc:74:- currentPath /= *pathComponents;
src/S3File.cc:75:- } else {
src/S3File.cc:76:- return -ENOENT;
src/S3File.cc:77:- }
src/S3File.cc:48:+S3File::S3File(XrdSysError &log, S3FileSystem *oss)
src/S3File.cc:49:+ : m_log(log), m_oss(oss), content_length(0), last_modified(0) {}
src/S3File.cc:50:+
src/S3File.cc:51:+int parse_path(const S3FileSystem &fs, const char *fullPath,
src/S3File.cc:52:+ std::string &exposedPath, std::string &object) {
src/S3File.cc:53:+ //
src/S3File.cc:54:+ // Check the path for validity.
src/S3File.cc:55:+ //
src/S3File.cc:56:+ std::filesystem::path p(fullPath);
src/S3File.cc:57:+ auto pathComponents = p.begin();
src/S3File.cc:58:+
src/S3File.cc:59:+ // Iterate through components of the fullPath until we either find a match
src/S3File.cc:60:+ // or we've reached the end of the path.
src/S3File.cc:61:+ std::filesystem::path currentPath = *pathComponents;
src/S3File.cc:62:+ while (pathComponents != p.end()) {
src/S3File.cc:63:+ if (fs.exposedPathExists(currentPath.string())) {
src/S3File.cc:64:+ exposedPath = currentPath.string();
src/S3File.cc:65:+ break;
|
Run linters:
src/S3File.cc#L96
[lint] reported by reviewdog 🐶
Raw Output:
src/S3File.cc:96:-int
src/S3File.cc:97:-S3File::Open(const char *path, int Oflag, mode_t Mode, XrdOucEnv &env)
src/S3File.cc:98:-{
src/S3File.cc:99:- std::string exposedPath, object;
src/S3File.cc:100:- int rv = parse_path( * m_oss, path, exposedPath, object );
src/S3File.cc:101:- if( rv != 0 ) { return rv; }
src/S3File.cc:102:- if(!m_oss->exposedPathExists(exposedPath)) return -ENOENT;
src/S3File.cc:103:-
src/S3File.cc:104:- std::string configured_s3_region = m_oss->getS3Region(exposedPath);
src/S3File.cc:105:- std::string configured_s3_service_url = m_oss->getS3ServiceURL(exposedPath);
src/S3File.cc:106:- std::string configured_s3_access_key = m_oss->getS3AccessKeyFile(exposedPath);
src/S3File.cc:107:- std::string configured_s3_secret_key = m_oss->getS3SecretKeyFile(exposedPath);
src/S3File.cc:108:- std::string configured_s3_bucket_name = m_oss->getS3BucketName(exposedPath);
src/S3File.cc:109:-
src/S3File.cc:110:- // We used to query S3 here to see if the object existed, but of course
src/S3File.cc:111:- // if you're creating a file on upload, you don't care.
src/S3File.cc:112:-
src/S3File.cc:113:- this->s3_object_name = object;
src/S3File.cc:114:- this->s3_bucket_name = configured_s3_bucket_name;
src/S3File.cc:115:- this->s3_service_url = configured_s3_service_url;
src/S3File.cc:116:- this->s3_access_key = configured_s3_access_key;
src/S3File.cc:117:- this->s3_secret_key = configured_s3_secret_key;
src/S3File.cc:118:- std::string configured_s3_url_style = m_oss->getS3URLStyle();
src/S3File.cc:119:-
src/S3File.cc:120:-
src/S3File.cc:121:- // We used to query S3 here to see if the object existed, but of course
src/S3File.cc:122:- // if you're creating a file on upload, you don't care.
src/S3File.cc:123:-
src/S3File.cc:124:- this->s3_object_name = object;
src/S3File.cc:125:- this->s3_bucket_name = configured_s3_bucket_name;
src/S3File.cc:126:- this->s3_service_url = configured_s3_service_url;
src/S3File.cc:127:- this->s3_access_key = configured_s3_access_key;
src/S3File.cc:128:- this->s3_secret_key = configured_s3_secret_key;
src/S3File.cc:129:- this->s3_url_style = configured_s3_url_style;
src/S3File.cc:130:- return 0;
src/S3File.cc:89:+ return 0;
|
Run linters:
src/S3File.cc#L186
[lint] reported by reviewdog 🐶
Raw Output:
src/S3File.cc:186:- std::string headers = head.getResultString();
src/S3File.cc:187:-
src/S3File.cc:188:- std::string line;
src/S3File.cc:189:- size_t current_newline = 0;
src/S3File.cc:190:- size_t next_newline = std::string::npos;
src/S3File.cc:191:- size_t last_character = headers.size();
src/S3File.cc:192:- while( current_newline != std::string::npos && current_newline != last_character - 1 ) {
src/S3File.cc:193:- next_newline = headers.find( "\r\n", current_newline + 2);
src/S3File.cc:194:- line = substring( headers, current_newline + 2, next_newline );
src/S3File.cc:195:-
src/S3File.cc:196:- size_t colon = line.find(":");
src/S3File.cc:197:- if( colon != std::string::npos && colon != line.size() ) {
src/S3File.cc:198:- std::string attr = substring( line, 0, colon );
src/S3File.cc:199:- std::string value = substring( line, colon + 1 );
src/S3File.cc:200:- trim(value);
src/S3File.cc:201:- toLower(attr);
src/S3File.cc:202:-
src/S3File.cc:203:- if( attr == "content-length" ) {
src/S3File.cc:204:- this->content_length = std::stol(value);
src/S3File.cc:205:- } else if( attr == "last-modified" ) {
src/S3File.cc:206:- struct tm t;
src/S3File.cc:207:- char * eos = strptime( value.c_str(),
src/S3File.cc:208:- "%a, %d %b %Y %T %Z",
src/S3File.cc:209:- & t );
src/S3File.cc:210:- if( eos == & value.c_str()[value.size()] ) {
src/S3File.cc:211:- time_t epoch = timegm(& t);
src/S3File.cc:212:- if( epoch != -1 ) {
src/S3File.cc:213:- this->last_modified = epoch;
src/S3File.cc:214:- }
src/S3File.cc:215:- }
src/S3File.cc:216:- }
src/S3File.cc:147:+int S3File::Fstat(struct stat *buff) {
src/S3File.cc:148:+ AmazonS3Head head(this->s3_service_url, this->s3_access_key,
src/S3File.cc:149:+ this->s3_secret_key, this->s3_bucket_name,
src/S3File.cc:150:+ this->s3_object_name, this->s3_url_style, m_log);
src/S3File.cc:151:+
src/S3File.cc:152:+ if (!head.SendRequest()) {
src/S3File.cc:153:+ // SendRequest() returns false for all errors, including ones
src/S3File.cc:154:+ // where the server properly responded with something other
src/S3File.cc:155:+ // than code 200. If xrootd wants us to distinguish between
src/S3File.cc:156:+ // these cases, head.getResponseCode() is initialized to 0, so
src/S3File.cc:157:+ // we can check.
src/S3File.cc:158:+ std::stringstream ss;
src/S3File.cc:159:+ ss << "Failed to send HeadObject command: " << head.getResponseCode() << "'"
src/S3File.cc:160:+ << head.getResultString() << "'";
src/S3File.cc:161:+ m_log.Log(LogMask::Warning, "S3File::Fstat", ss.str().c_str());
src/S3File.cc:162:+ return -ENOENT;
src/S3File.cc:163:+ }
src/S3File.cc:164:+
src/S3File.cc:165:+ std::string headers = head.getResultString();
src/S3File.cc:166:+
src/S3File.cc:167:+ std::string line;
src/S3File.cc:168:+ size_t current_newline = 0;
src/S3File.cc:169:+ size_t next_newline = std::string::npos;
src/S3File.cc:170:+ size_t last_character = headers.size();
src/S3File.cc:171:+ while (current_newline != std::string::npos &&
src/S3File.cc:172:+ current_newline != last_character - 1) {
src/S3File.cc:173:+ next_newline = headers.find("\r\n", current_newline + 2);
src/S3File.cc:174:+ line = substring(headers, current_newline + 2, next_newline);
src/S3File.cc:175:+
src/S3File.cc:176:+ size_t colon = line.find(":");
src/S3File.cc:177:+ if (colon != std::string::npos && colon != line.size()) {
src/S3File.cc:178:+ std::string attr = substring(line, 0, colon);
src/S3File.cc:179:+ std::string value = substring(line, colon + 1);
src/S3File.cc:180:+ trim(value);
src/S3File.cc:181:+ toLower(attr);
src/S3File.cc:182:+
src/S3File.cc:183:+ if (attr == "content-length") {
src/S3
|
Run linters:
src/S3FileSystem.cc#L65
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:65:- }
src/S3FileSystem.cc:66:- return true;
src/S3FileSystem.cc:67:-}
src/S3FileSystem.cc:68:-
src/S3FileSystem.cc:69:-
src/S3FileSystem.cc:70:-bool
src/S3FileSystem.cc:71:-S3FileSystem::Config(XrdSysLogger *lp, const char *configfn)
src/S3FileSystem.cc:72:-{
src/S3FileSystem.cc:73:- XrdOucEnv myEnv;
src/S3FileSystem.cc:74:- XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), &myEnv, "=====> ");
src/S3FileSystem.cc:75:-
src/S3FileSystem.cc:76:- int cfgFD = open(configfn, O_RDONLY, 0);
src/S3FileSystem.cc:77:- if (cfgFD < 0) {
src/S3FileSystem.cc:78:- m_log.Emsg("Config", errno, "open config file", configfn);
src/S3FileSystem.cc:85:+ }
src/S3FileSystem.cc:86:+ if (newAccessInfo->getS3Region().empty()) {
src/S3FileSystem.cc:87:+ m_log.Emsg("Config", "s3.region not specified");
|
Run linters:
src/S3FileSystem.cc#L81
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:81:-
src/S3FileSystem.cc:82:- char * temporary;
src/S3FileSystem.cc:83:- std::string value;
src/S3FileSystem.cc:84:- std::string attribute;
src/S3FileSystem.cc:85:- Config.Attach(cfgFD);
src/S3FileSystem.cc:86:- S3AccessInfo *newAccessInfo = new S3AccessInfo();
src/S3FileSystem.cc:87:- std::string exposedPath;
src/S3FileSystem.cc:88:- while ((temporary = Config.GetMyFirstWord())) {
src/S3FileSystem.cc:89:- attribute = temporary;
src/S3FileSystem.cc:90:- temporary = Config.GetWord();
src/S3FileSystem.cc:91:- if(attribute == "s3.end") {
src/S3FileSystem.cc:92:- s3_access_map[exposedPath] = newAccessInfo;
src/S3FileSystem.cc:93:- if(newAccessInfo->getS3ServiceName().empty()) {
src/S3FileSystem.cc:94:- m_log.Emsg("Config", "s3.service_name not specified");
src/S3FileSystem.cc:95:- return false;
src/S3FileSystem.cc:96:- }
src/S3FileSystem.cc:97:- if(newAccessInfo->getS3Region().empty()) {
src/S3FileSystem.cc:98:- m_log.Emsg("Config", "s3.region not specified");
src/S3FileSystem.cc:99:- return false;
src/S3FileSystem.cc:100:- }
src/S3FileSystem.cc:101:- newAccessInfo = new S3AccessInfo();
src/S3FileSystem.cc:102:- exposedPath = "";
src/S3FileSystem.cc:103:- continue;
src/S3FileSystem.cc:104:- }
src/S3FileSystem.cc:105:- if(! temporary) { continue; }
src/S3FileSystem.cc:106:- value = temporary;
src/S3FileSystem.cc:107:-
src/S3FileSystem.cc:108:- if(!handle_required_config("s3.path_name",value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:109:- if(!handle_required_config("s3.bucket_name",value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:110:- if(!handle_required_config("s3.service_name",value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:111:- if(!handle_required_config("s3.region", value ) ) { Config.Close(); return false; }
src/S3FileSystem.cc:112:- if(!handle_required_config("s3.service_url", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:113:- if(!handle_required_config("s3.access_key_file", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:114:- if(!handle_required_config("s3.secret_key_file", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:115:- if(!handle_required_config("s3.url_style", value) ) { Config.Close(); return false; }
src/S3FileSystem.cc:116:-
src/S3FileSystem.cc:117:- if (attribute == "s3.path_name") {
src/S3FileSystem.cc:118:- // Normalize paths so that they all start with /
src/S3FileSystem.cc:119:- if (value[0] != '/') {
src/S3FileSystem.cc:120:- exposedPath = "/" + value;
src/S3FileSystem.cc:121:- } else {
src/S3FileSystem.cc:122:- exposedPath = value;
src/S3FileSystem.cc:123:- }
src/S3FileSystem.cc:124:- }
src/S3FileSystem.cc:125:- else if(attribute == "s3.bucket_name") newAccessInfo->setS3BucketName(value);
src/S3FileSystem.cc:126:- else if(attribute == "s3.service_name") newAccessInfo->setS3ServiceName(value);
src/S3FileSystem.cc:127:- else if(attribute == "s3.region") newAccessInfo->setS3Region(value);
src/S3FileSystem.cc:128:- else if(attribute == "s3.access_key_file") newAccessInfo->setS3AccessKeyFile(value);
src/S3FileSystem.cc:129:- else if(attribute == "s3.secret_key_file") newAccessInfo->setS3SecretKeyFile(value);
src/S3FileSystem.cc:130:- else if(attribute == "s3.service_url") newAccessInfo->setS3ServiceUrl(value);
src/S3FileSystem.cc:131:- else if(attribute == "s3.url_style") this->s3_url_style = value;
src/S3FileSystem.cc:132:-
src/S3FileSystem.cc:94:+ if (!temporary) {
src/S3FileSystem.cc:95:+ continue;
|
Run linters:
src/S3FileSystem.cc#L135
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:135:- if( this->s3_url_style.empty() ) {
src/S3FileSystem.cc:136:- m_log.Emsg("Config", "s3.url_style not specified");
src/S3FileSystem.cc:137:- return false;
src/S3FileSystem.cc:138:- } else {
src/S3FileSystem.cc:139:- // We want this to be case-insensitive.
src/S3FileSystem.cc:140:- toLower( this->s3_url_style );
src/S3FileSystem.cc:99:+ if (!handle_required_config("s3.path_name", value)) {
src/S3FileSystem.cc:100:+ Config.Close();
src/S3FileSystem.cc:101:+ return false;
|
Run linters:
src/S3FileSystem.cc#L199
[lint] reported by reviewdog 🐶
Raw Output:
src/S3FileSystem.cc:199:-int
src/S3FileSystem.cc:200:-S3FileSystem::Create( const char *tid, const char *path, mode_t mode,
src/S3FileSystem.cc:201:- XrdOucEnv &env, int opts )
src/S3FileSystem.cc:202:-{
src/S3FileSystem.cc:203:- // Is path valid?
src/S3FileSystem.cc:204:- std::string exposedPath, object;
src/S3FileSystem.cc:205:- int rv = parse_path( * this, path, exposedPath, object );
src/S3FileSystem.cc:206:- if( rv != 0 ) { return rv; }
src/S3FileSystem.cc:207:-
src/S3FileSystem.cc:208:- //
src/S3FileSystem.cc:209:- // We could instead invoke the upload mchinery directly to create a
src/S3FileSystem.cc:210:- // 0-byte file, but it seems smarter to remove a round-trip (in
src/S3FileSystem.cc:211:- // S3File::Open(), checking if the file exists) than to add one
src/S3FileSystem.cc:212:- // (here, creating the file if it doesn't exist).
src/S3FileSystem.cc:213:- //
src/S3FileSystem.cc:214:-
src/S3FileSystem.cc:215:- return 0;
src/S3FileSystem.cc:211:+int S3FileSystem::Create(const char *tid, const char *path, mode_t mode,
src/S3FileSystem.cc:212:+ XrdOucEnv &env, int opts) {
src/S3FileSystem.cc:213:+ // Is path valid?
src/S3FileSystem.cc:214:+ std::string exposedPath, object;
src/S3FileSystem.cc:215:+ int rv = parse_path(*this, path, exposedPath, object);
src/S3FileSystem.cc:216:+ if (rv != 0) {
src/S3FileSystem.cc:217:+ return rv;
src/S3FileSystem.cc:218:+ }
src/S3FileSystem.cc:219:+
src/S3FileSystem.cc:220:+ //
src/S3FileSystem.cc:221:+ // We could instead invoke the upload mchinery directly to create a
src/S3FileSystem.cc:222:+ // 0-byte file, but it seems smarter to remove a round-trip (in
src/S3FileSystem.cc:223:+ // S3File::Open(), checking if the file exists) than to add one
src/S3FileSystem.cc:224:+ // (here, creating the file if it doesn't exist).
src/S3FileSystem.cc:225:+ //
src/S3FileSystem.cc:226:+
src/S3FileSystem.cc:227:+ return 0;
|
Run linters
Node.js 16 actions are deprecated. Please update the following actions to use Node.js 20: actions/checkout@v3, wearerequired/lint-action@v2. For more information see: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/.
|