diff --git a/src/admin/python/lsst/qserv/admin/replicationInterface.py b/src/admin/python/lsst/qserv/admin/replicationInterface.py index 23f89258da..0ac590b784 100644 --- a/src/admin/python/lsst/qserv/admin/replicationInterface.py +++ b/src/admin/python/lsst/qserv/admin/replicationInterface.py @@ -201,7 +201,7 @@ def __init__( self.repl_ctrl = urlparse(repl_ctrl_uri) self.auth_key = auth_key self.admin_auth_key = admin_auth_key - self.repl_api_version = 27 + self.repl_api_version = 28 _log.debug(f"ReplicationInterface %s", self.repl_ctrl) def version(self) -> str: diff --git a/src/ccontrol/UserQueryQservManager.cc b/src/ccontrol/UserQueryQservManager.cc index 491060a8c7..20e5285949 100644 --- a/src/ccontrol/UserQueryQservManager.cc +++ b/src/ccontrol/UserQueryQservManager.cc @@ -41,7 +41,7 @@ #include "sql/SqlBulkInsert.h" #include "sql/SqlConnection.h" #include "sql/SqlConnectionFactory.h" -#include "util/StringHelper.h" +#include "util/String.h" #include "wconfig/WorkerConfig.h" using namespace std; @@ -80,7 +80,7 @@ void UserQueryQservManager::submit() { if (_value.size() > 2) { string const space = " "; string const quotesRemoved = _value.substr(1, _value.size() - 2); - for (auto&& str : util::StringHelper::splitString(quotesRemoved, space)) { + for (auto&& str : util::String::split(quotesRemoved, space)) { // This is just in case if the splitter won't recognise consequtive spaces. if (str.empty() || (str == space)) continue; if (command.empty()) { diff --git a/src/czar/Czar.cc b/src/czar/Czar.cc index 45db5d69af..8232692a33 100644 --- a/src/czar/Czar.cc +++ b/src/czar/Czar.cc @@ -56,7 +56,7 @@ #include "util/common.h" #include "util/FileMonitor.h" #include "util/IterableFormatter.h" -#include "util/StringHelper.h" +#include "util/String.h" #include "xrdreq/QueryManagementAction.h" #include "XrdSsi/XrdSsiProvider.hh" @@ -124,9 +124,9 @@ Czar::Czar(string const& configPath, string const& czarName) int qPoolSize = _czarConfig->getQdispPoolSize(); int maxPriority = std::max(0, _czarConfig->getQdispMaxPriority()); string vectRunSizesStr = _czarConfig->getQdispVectRunSizes(); - vector vectRunSizes = util::StringHelper::getIntVectFromStr(vectRunSizesStr, ":", 1); + vector vectRunSizes = util::String::parseToVectInt(vectRunSizesStr, ":", 1); string vectMinRunningSizesStr = _czarConfig->getQdispVectMinRunningSizes(); - vector vectMinRunningSizes = util::StringHelper::getIntVectFromStr(vectMinRunningSizesStr, ":", 0); + vector vectMinRunningSizes = util::String::parseToVectInt(vectMinRunningSizesStr, ":", 0); LOGS(_log, LOG_LVL_INFO, "INFO qdisp config qPoolSize=" << qPoolSize << " maxPriority=" << maxPriority << " vectRunSizes=" << vectRunSizesStr << " -> " << util::prettyCharList(vectRunSizes) diff --git a/src/http/MetaModule.cc b/src/http/MetaModule.cc index 166942b989..8355f0ea82 100644 --- a/src/http/MetaModule.cc +++ b/src/http/MetaModule.cc @@ -37,7 +37,7 @@ string const adminAuthKey; namespace lsst::qserv::http { -unsigned int const MetaModule::version = 27; +unsigned int const MetaModule::version = 28; void MetaModule::process(string const& context, nlohmann::json const& info, qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp, string const& subModuleName) { diff --git a/src/replica/CMakeLists.txt b/src/replica/CMakeLists.txt index 8cf88f791b..d5e09b70a4 100644 --- a/src/replica/CMakeLists.txt +++ b/src/replica/CMakeLists.txt @@ -76,6 +76,7 @@ target_sources(replica PRIVATE GetReplicasQservMgtRequest.cc GetDbStatusQservMgtRequest.cc GetConfigQservMgtRequest.cc + GetResultFilesQservMgtRequest.cc GetStatusQservMgtRequest.cc HealthMonitorTask.cc HttpAsyncReqApp.cc @@ -297,7 +298,6 @@ replica_tests( testChunkLocker testChunkNumber testChunkedTable - testCommonStr testConnectionParams testCsv testFileIngestApp diff --git a/src/replica/Common.cc b/src/replica/Common.cc index 3719398abb..828b943474 100644 --- a/src/replica/Common.cc +++ b/src/replica/Common.cc @@ -23,10 +23,7 @@ #include "replica/Common.h" // System headers -#include #include -#include -#include // Third party headers #include "boost/uuid/uuid.hpp" @@ -350,19 +347,6 @@ DirectorIndexRequestParams::DirectorIndexRequestParams(ProtocolRequestDirectorIn hasTransactions(request.has_transactions()), transactionId(request.transaction_id()) {} -vector strsplit(string const& str, char delimiter) { - vector words; - if (!str.empty()) { - string word; - istringstream ss(str); - while (std::getline(ss, word, delimiter)) { - remove(word.begin(), word.end(), delimiter); - if (!word.empty()) words.push_back(word); - } - } - return words; -} - string tableNameBuilder(string const& databaseName, string const& tableName, string const& suffix) { size_t const tableNameLimit = 64; string const name = databaseName + "__" + tableName + suffix; diff --git a/src/replica/Common.h b/src/replica/Common.h index 0951f3ed2f..9e8e326c35 100644 --- a/src/replica/Common.h +++ b/src/replica/Common.h @@ -351,17 +351,6 @@ class Query { std::string mutexName; }; -/** - * @brief Split the input string into words separated by the delimiter. - * - * @param str The input string. - * @param delimiter The delimiter character. - * @return std::vector A collection of words found in the string. The words - * are guaranteed not to have delimiters. The collection is guaranteed not to have - * empty strings. - */ -std::vector strsplit(std::string const& str, char delimiter = ' '); - /** * @brief Generate the name of a metadata table at czar for the specified data table. * @param databaseName The name of a database where the data table is residing. diff --git a/src/replica/CreateReplicaJob.cc b/src/replica/CreateReplicaJob.cc index 4d25418736..84503bddbd 100644 --- a/src/replica/CreateReplicaJob.cc +++ b/src/replica/CreateReplicaJob.cc @@ -33,7 +33,7 @@ #include "replica/QservMgtServices.h" #include "replica/ServiceProvider.h" #include "replica/StopRequest.h" -#include "util/IterableFormatter.h" +#include "util/String.h" // LSST headers #include "lsst/log/Log.h" @@ -312,7 +312,7 @@ void CreateReplicaJob::_qservAddReplica(replica::Lock const& lock, unsigned int AddReplicaQservMgtRequest::CallbackType const& onFinish) { LOGS(_log, LOG_LVL_DEBUG, context() << __func__ << " ** START ** Qserv notification on ADD replica:" - << ", chunk=" << chunk << ", databases=" << util::printable(databases) + << ", chunk=" << chunk << ", databases=" << util::String::toString(databases) << " worker=" << worker); auto self = shared_from_this(); @@ -322,8 +322,9 @@ void CreateReplicaJob::_qservAddReplica(replica::Lock const& lock, unsigned int LOGS(_log, LOG_LVL_DEBUG, self->context() << __func__ << " ** FINISH ** Qserv notification on ADD replica:" << " chunk=" << request->chunk() - << ", databases=" << util::printable(request->databases()) << ", worker=" - << request->worker() << ", state=" << request->state2string()); + << ", databases=" << util::String::toString(request->databases()) + << ", worker=" << request->worker() + << ", state=" << request->state2string()); if (onFinish) onFinish(request); }, id()); diff --git a/src/replica/DeleteReplicaJob.cc b/src/replica/DeleteReplicaJob.cc index b51462b039..9e722119b0 100644 --- a/src/replica/DeleteReplicaJob.cc +++ b/src/replica/DeleteReplicaJob.cc @@ -33,7 +33,7 @@ #include "replica/QservMgtServices.h" #include "replica/ServiceProvider.h" #include "replica/StopRequest.h" -#include "util/IterableFormatter.h" +#include "util/String.h" // LSST headers #include "lsst/log/Log.h" @@ -293,7 +293,7 @@ void DeleteReplicaJob::_qservRemoveReplica(replica::Lock const& lock, unsigned i RemoveReplicaQservMgtRequest::CallbackType const& onFinish) { LOGS(_log, LOG_LVL_DEBUG, context() << __func__ << " ** START ** Qserv notification on REMOVE replica:" - << " chunk=" << chunk << ", databases=" << util::printable(databases) + << " chunk=" << chunk << ", databases=" << util::String::toString(databases) << ", worker=" << worker << ", force=" << (force ? "true" : "false")); auto self = shared_from_this(); @@ -303,7 +303,7 @@ void DeleteReplicaJob::_qservRemoveReplica(replica::Lock const& lock, unsigned i LOGS(_log, LOG_LVL_DEBUG, self->context() << __func__ << " ** FINISH ** Qserv notification on REMOVE replica:" << " chunk=" << request->chunk() - << ", databases=" << util::printable(request->databases()) + << ", databases=" << util::String::toString(request->databases()) << ", worker=" << request->worker() << ", force=" << (request->force() ? "true" : "false") << ", state=" << request->state2string()); diff --git a/src/replica/GetResultFilesQservMgtRequest.cc b/src/replica/GetResultFilesQservMgtRequest.cc new file mode 100644 index 0000000000..8a8ad0af9f --- /dev/null +++ b/src/replica/GetResultFilesQservMgtRequest.cc @@ -0,0 +1,71 @@ +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +// Class header +#include "replica/GetResultFilesQservMgtRequest.h" + +// Qserv headers +#include "util/String.h" + +// LSST headers +#include "lsst/log/Log.h" + +using namespace std; + +namespace { + +LOG_LOGGER _log = LOG_GET("lsst.qserv.replica.GetResultFilesQservMgtRequest"); + +} // namespace + +namespace lsst::qserv::replica { + +shared_ptr GetResultFilesQservMgtRequest::create( + shared_ptr const& serviceProvider, string const& worker, + vector const& queryIds, unsigned int maxFiles, + GetResultFilesQservMgtRequest::CallbackType const& onFinish) { + return shared_ptr( + new GetResultFilesQservMgtRequest(serviceProvider, worker, queryIds, maxFiles, onFinish)); +} + +GetResultFilesQservMgtRequest::GetResultFilesQservMgtRequest( + shared_ptr const& serviceProvider, string const& worker, + vector const& queryIds, unsigned int maxFiles, + GetResultFilesQservMgtRequest::CallbackType const& onFinish) + : QservMgtRequest(serviceProvider, "QSERV_GET_RESULT_FILES", worker), + _queryIds(queryIds), + _maxFiles(maxFiles), + _onFinish(onFinish) {} + +void GetResultFilesQservMgtRequest::createHttpReqImpl(replica::Lock const& lock) { + string const service = "/files"; + string query; + query += "?query_ids=" + util::String::toString(_queryIds); + query += "&max_files=" + to_string(_maxFiles); + createHttpReq(lock, service, query); +} + +void GetResultFilesQservMgtRequest::notify(replica::Lock const& lock) { + LOGS(_log, LOG_LVL_TRACE, context() << __func__); + notifyDefaultImpl(lock, _onFinish); +} + +} // namespace lsst::qserv::replica diff --git a/src/replica/GetResultFilesQservMgtRequest.h b/src/replica/GetResultFilesQservMgtRequest.h new file mode 100644 index 0000000000..c1131a4b67 --- /dev/null +++ b/src/replica/GetResultFilesQservMgtRequest.h @@ -0,0 +1,98 @@ +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ +#ifndef LSST_QSERV_REPLICA_GETRESULTFILESQSERVMGTREQUEST_H +#define LSST_QSERV_REPLICA_GETRESULTFILESQSERVMGTREQUEST_H + +// System headers +#include +#include +#include + +// Qserv headers +#include "global/intTypes.h" +#include "replica/QservMgtRequest.h" + +namespace lsst::qserv::replica { +class ServiceProvider; +} // namespace lsst::qserv::replica + +// This header declarations +namespace lsst::qserv::replica { + +/** + * Class GetResultFilesQservMgtRequest is a request for obtaining info + * on the partial result files from the Qserv worker. + */ +class GetResultFilesQservMgtRequest : public QservMgtRequest { +public: + typedef std::shared_ptr Ptr; + + /// The function type for notifications on the completion of the request + typedef std::function CallbackType; + + GetResultFilesQservMgtRequest() = delete; + GetResultFilesQservMgtRequest(GetResultFilesQservMgtRequest const&) = delete; + GetResultFilesQservMgtRequest& operator=(GetResultFilesQservMgtRequest const&) = delete; + + virtual ~GetResultFilesQservMgtRequest() final = default; + + /** + * Static factory method is needed to prevent issues with the lifespan + * and memory management of instances created otherwise (as values or via + * low-level pointers). + * @param serviceProvider A reference to a provider of services for accessing + * Configuration, saving the request's persistent state to the database. + * @param worker The name of a worker to send the request to. + * @param queryIds The optional selector for queries. If empty then all queries will + * be considered. + * @param maxFiles The optional limit for maximum number of files to be reported. + * If 0 then no limit is set. + * @param onFinish (optional) callback function to be called upon request completion. + * @return A pointer to the created object. + */ + static std::shared_ptr create( + std::shared_ptr const& serviceProvider, std::string const& worker, + std::vector const& queryIds = std::vector(), unsigned int maxFiles = 0, + CallbackType const& onFinish = nullptr); + +protected: + /// @see QservMgtRequest::createHttpReqImpl() + virtual void createHttpReqImpl(replica::Lock const& lock) final; + + /// @see QservMgtRequest::notify() + virtual void notify(replica::Lock const& lock) final; + +private: + /// @see GetResultFilesQservMgtRequest::create() + GetResultFilesQservMgtRequest(std::shared_ptr const& serviceProvider, + std::string const& worker, std::vector const& queryIds, + unsigned int maxFiles, CallbackType const& onFinish); + + // Input parameters + + std::vector const _queryIds; + unsigned int const _maxFiles; + CallbackType _onFinish; ///< This callback is reset after finishing the request. +}; + +} // namespace lsst::qserv::replica + +#endif // LSST_QSERV_REPLICA_GETRESULTFILESQSERVMGTREQUEST_H diff --git a/src/replica/GetStatusQservMgtRequest.cc b/src/replica/GetStatusQservMgtRequest.cc index 4a8287d866..a47a28a002 100644 --- a/src/replica/GetStatusQservMgtRequest.cc +++ b/src/replica/GetStatusQservMgtRequest.cc @@ -22,10 +22,8 @@ // Class header #include "replica/GetStatusQservMgtRequest.h" -// System headers -#include -#include -#include +// Qserv headers +#include "util/String.h" // LSST headers #include "lsst/log/Log.h" @@ -45,18 +43,10 @@ string taskSelectorToHttpQuery(wbase::TaskSelector const& taskSelector) { query += "?include_tasks=" + string(taskSelector.includeTasks ? "1" : "0"); query += "&max_tasks=" + to_string(taskSelector.maxTasks); if (!taskSelector.queryIds.empty()) { - ostringstream ss; - copy(taskSelector.queryIds.begin(), taskSelector.queryIds.end() - 1, - ostream_iterator(ss, ",")); - ss << taskSelector.queryIds.back(); - query += "&query_ids=" + ss.str(); + query += "&query_ids=" + util::String::toString(taskSelector.queryIds); } if (!taskSelector.taskStates.empty()) { - ostringstream ss; - copy(taskSelector.taskStates.begin(), taskSelector.taskStates.end() - 1, - ostream_iterator(ss, ",")); - ss << taskSelector.taskStates.back(); - query += "&task_states=" + ss.str(); + query += "&task_states=" + util::String::toString(taskSelector.taskStates); } return query; } diff --git a/src/replica/HttpAsyncReqApp.cc b/src/replica/HttpAsyncReqApp.cc index 507b0c5e77..7cbf50a2df 100644 --- a/src/replica/HttpAsyncReqApp.cc +++ b/src/replica/HttpAsyncReqApp.cc @@ -23,13 +23,10 @@ #include "replica/HttpAsyncReqApp.h" // System headers -#include #include #include -#include -#include #include -#include +#include // Third-party headers #include "boost/asio.hpp" @@ -38,6 +35,7 @@ // Qserv headers #include "http/AsyncReq.h" #include "http/Method.h" +#include "util/String.h" using namespace std; using json = nlohmann::json; @@ -54,11 +52,6 @@ bool const injectDatabaseOptions = false; bool const boostProtobufVersionCheck = false; bool const enableServiceProvider = false; -string vector2str(vector const& v) { - ostringstream oss; - copy(v.cbegin(), v.cend(), ostream_iterator(oss, " ")); - return oss.str(); -} } // namespace namespace lsst::qserv::replica { @@ -71,37 +64,38 @@ HttpAsyncReqApp::HttpAsyncReqApp(int argc, char* argv[]) : Application(argc, argv, ::description, ::injectDatabaseOptions, ::boostProtobufVersionCheck, ::enableServiceProvider) { parser().required("url", "The URL to read data from.", _url) - .option("method", "The HTTP method. Allowed values: " + ::vector2str(http::allowedMethods), + .option("method", + "The HTTP method. Allowed values: " + util::String::toString(http::allowedMethods), _method, http::allowedMethods) .option("header", "The HTTP header to be sent with a request. Note this test application allows" " only one header. The format of the header is '[:]'.", - _header) - .option("data", "The data to be sent in the body of a request.", _data) - .option("max-response-data-size", + _header); + parser().option("data", "The data to be sent in the body of a request.", _data); + parser().option("max-response-data-size", "The maximum size (bytes) of the response body. If a value of the parameter is set" " to 0 then the default limit of 8M imposed by the Boost.Beast library will be assumed.", - _maxResponseBodySize) - .option("expiration-ival-sec", + _maxResponseBodySize); + parser().option("expiration-ival-sec", "A timeout to wait before the completion of a request. The expiration timeout includes" " all phases of the request's execution, including establishing a connection" " to the server, sending the request and waiting for the server's response." " If a value of the parameter is set to 0 then no expiration timeout will be" " assumed for the request.", - _expirationIvalSec) - .option("file", + _expirationIvalSec); + parser().option("file", "A path to an output file where the response body received from a remote source will" " be written. This option is ignored if the flag --body is not specified.", - _file) - .flag("result2json", + _file); + parser().flag("result2json", "If specified the flag will cause the application to interpret the response body as" " a JSON object.", - _result2json) - .flag("verbose", + _result2json); + parser().flag("verbose", "The flag that allows printing the completion status and the response header" " info onto the standard output stream.", - _verbose) - .flag("body", + _verbose); + parser().flag("body", "The flag that allows printing the complete response body. If the --file= option" " is specified then the body will be written into that files. Otherwise it will be" " printed onto the standard output stream.", diff --git a/src/replica/HttpProcessor.cc b/src/replica/HttpProcessor.cc index 2d5699c233..7817374f17 100644 --- a/src/replica/HttpProcessor.cc +++ b/src/replica/HttpProcessor.cc @@ -257,6 +257,12 @@ void HttpProcessor::registerServices() { self->_processorConfig, req, resp, "WORKER-DB"); }); + httpServer()->addHandler("GET", "/replication/qserv/worker/files/:worker", + [self](qhttp::Request::Ptr const req, qhttp::Response::Ptr const resp) { + HttpQservMonitorModule::process(self->controller(), self->name(), + self->_processorConfig, req, resp, + "WORKER-FILES"); + }); httpServer()->addHandler("GET", "/replication/qserv/master/status", [self](qhttp::Request::Ptr const req, qhttp::Response::Ptr const resp) { HttpQservMonitorModule::process(self->controller(), self->name(), diff --git a/src/replica/HttpQservMonitorModule.cc b/src/replica/HttpQservMonitorModule.cc index 439ecf81f4..63029e2f42 100644 --- a/src/replica/HttpQservMonitorModule.cc +++ b/src/replica/HttpQservMonitorModule.cc @@ -35,6 +35,7 @@ #include "css/CssError.h" #include "global/intTypes.h" #include "http/Exceptions.h" +#include "replica/Common.h" #include "replica/DatabaseMySQL.h" #include "replica/DatabaseMySQLTypes.h" #include "replica/DatabaseMySQLUtils.h" @@ -45,6 +46,7 @@ #include "replica/QservMgtServices.h" #include "replica/QservStatusJob.h" #include "replica/ServiceProvider.h" +#include "util/String.h" #include "wbase/TaskState.h" // LSST headers @@ -131,6 +133,15 @@ void HttpQservMonitorModule::process(Controller::Ptr const& controller, string c module.execute(subModuleName, authType); } +void HttpQservMonitorModule::_throwIfNotSucceeded(string const& func, + shared_ptr const& request) { + if (request->extendedState() == QservMgtRequest::ExtendedState::SUCCESS) return; + string const msg = "request id: " + request->id() + " of type: " + request->type() + + " sent to worker: " + request->worker() + + " failed, error: " + QservMgtRequest::state2string(request->extendedState()); + throw http::Error(func, msg); +} + HttpQservMonitorModule::HttpQservMonitorModule(Controller::Ptr const& controller, string const& taskName, HttpProcessorConfig const& processorConfig, qhttp::Request::Ptr const& req, @@ -146,6 +157,8 @@ json HttpQservMonitorModule::executeImpl(string const& subModuleName) { return _workerConfig(); else if (subModuleName == "WORKER-DB") return _workerDb(); + else if (subModuleName == "WORKER-FILES") + return _workerFiles(); else if (subModuleName == "CZAR") return _czar(); else if (subModuleName == "CZAR-CONFIG") @@ -215,7 +228,6 @@ json HttpQservMonitorModule::_worker() { string const noParentJobId; GetStatusQservMgtRequest::CallbackType const onFinish = nullptr; - auto const request = controller()->serviceProvider()->qservMgtServices()->status( worker, noParentJobId, taskSelector, onFinish, timeoutSec); request->wait(); @@ -246,19 +258,12 @@ json HttpQservMonitorModule::_workerConfig() { string const noParentJobId; GetConfigQservMgtRequest::CallbackType const onFinish = nullptr; - auto const request = controller()->serviceProvider()->qservMgtServices()->config(worker, noParentJobId, onFinish, timeoutSec); request->wait(); + _throwIfNotSucceeded(__func__, request); - if (request->extendedState() != QservMgtRequest::ExtendedState::SUCCESS) { - string const msg = "database operation failed, error: " + - QservMgtRequest::state2string(request->extendedState()); - throw http::Error(__func__, msg); - } - json result = json::object(); - result["config"] = request->info(); - return result; + return json::object({{"config", request->info()}}); } json HttpQservMonitorModule::_workerDb() { @@ -273,19 +278,36 @@ json HttpQservMonitorModule::_workerDb() { string const noParentJobId; GetDbStatusQservMgtRequest::CallbackType const onFinish = nullptr; - auto const request = controller()->serviceProvider()->qservMgtServices()->databaseStatus( worker, noParentJobId, onFinish, timeoutSec); request->wait(); + _throwIfNotSucceeded(__func__, request); - if (request->extendedState() != QservMgtRequest::ExtendedState::SUCCESS) { - string const msg = "database operation failed, error: " + - QservMgtRequest::state2string(request->extendedState()); - throw http::Error(__func__, msg); - } - json result = json::object(); - result["status"] = request->info(); - return result; + return json::object({{"status", request->info()}}); +} + +json HttpQservMonitorModule::_workerFiles() { + debug(__func__); + checkApiVersion(__func__, 28); + + auto const worker = params().at("worker"); + auto const queryIds = query().optionalVectorUInt64("query_ids"); + auto const maxFiles = query().optionalUInt("max_files", 0); + unsigned int const timeoutSec = query().optionalUInt("timeout_sec", workerResponseTimeoutSec()); + + debug(__func__, "worker=" + worker); + debug(__func__, "query_ids=" + util::String::toString(queryIds)); + debug(__func__, "max_files=" + to_string(maxFiles)); + debug(__func__, "timeout_sec=" + to_string(timeoutSec)); + + string const noParentJobId; + GetResultFilesQservMgtRequest::CallbackType const onFinish = nullptr; + auto const request = controller()->serviceProvider()->qservMgtServices()->resultFiles( + worker, noParentJobId, queryIds, maxFiles, onFinish, timeoutSec); + request->wait(); + _throwIfNotSucceeded(__func__, request); + + return json::object({{"status", request->info()}}); } json HttpQservMonitorModule::_czar() { @@ -368,8 +390,8 @@ wbase::TaskSelector HttpQservMonitorModule::_translateTaskSelector(string const& } selector.maxTasks = query().optionalUInt("max_tasks", 0); debug(func, "include_tasks=" + replica::bool2str(selector.includeTasks)); - debug(func, "queryIds.size()=" + to_string(selector.queryIds.size())); - debug(func, "taskStates.size()=" + to_string(selector.taskStates.size())); + debug(func, "query_ids=" + util::String::toString(selector.queryIds)); + debug(func, "task_states=" + util::String::toString(selector.taskStates)); debug(func, "max_tasks=" + to_string(selector.maxTasks)); return selector; } @@ -417,7 +439,6 @@ json HttpQservMonitorModule::_activeQueries() { checkApiVersion(__func__, 25); unsigned int const timeoutSec = query().optionalUInt("timeout_sec", workerResponseTimeoutSec()); - debug(__func__, "timeout_sec=" + to_string(timeoutSec)); // Check which queries and in which schedulers are being executed @@ -463,7 +484,6 @@ json HttpQservMonitorModule::_activeQueriesProgress() { QueryId const selectQueryId = query().optionalUInt64("query_id", 0); unsigned int const selectLastSeconds = query().optionalUInt("last_seconds", 0); - debug(__func__, "query_id=" + to_string(selectQueryId)); debug(__func__, "last_seconds=" + to_string(selectLastSeconds)); @@ -472,7 +492,6 @@ json HttpQservMonitorModule::_activeQueriesProgress() { QueryGenerator const g(conn); string const command = "query_info " + to_string(selectQueryId) + " " + to_string(selectLastSeconds); string const query = g.call(g.QSERV_MANAGER(command)); - debug(__func__, "query=" + query); // Result set processor populates the JSON object and returns the completion @@ -586,12 +605,9 @@ json HttpQservMonitorModule::_userQuery() { auto const queryId = stoull(params().at("id")); bool const includeMessages = query().optionalUInt("include_messages", 0) != 0; - debug(__func__, "id=" + to_string(queryId)); debug(__func__, "include_messages=" + bool2str(includeMessages)); - json result; - // Connect to the master database // Manage the new connection via the RAII-style handler to ensure the transaction // is automatically rolled-back in case of exceptions. @@ -599,6 +615,7 @@ json HttpQservMonitorModule::_userQuery() { ConnectionHandler const h(Connection::open(Configuration::qservCzarDbParams("qservMeta"))); QueryGenerator const g(h.conn); + json result; h.conn->executeInOwnTransaction([&](auto conn) { unsigned int const limit4past = 0; result["queries_past"] = diff --git a/src/replica/HttpQservMonitorModule.h b/src/replica/HttpQservMonitorModule.h index 292d42c3f6..d474066e18 100644 --- a/src/replica/HttpQservMonitorModule.h +++ b/src/replica/HttpQservMonitorModule.h @@ -40,6 +40,10 @@ namespace lsst::qserv::wbase { struct TaskSelector; } // namespace lsst::qserv::wbase +namespace lsst::qserv::replica { +class QservMgtRequest; +} // namespace lsst::qserv::replica + // This header declarations namespace lsst::qserv::replica { @@ -58,6 +62,7 @@ class HttpQservMonitorModule : public HttpModule { * WORKER - get the status info of a specific worker * WORKER-CONFIG - get configuration parameters of a specific worker * WORKER-DB - get the database status of a specific worker + * WORKER-FILES - get acollection of partial result files from a worker * CZAR - get the status info of Czar * CZAR-CONFIG - get configuration parameters of Czar * CZAR-DB - get the database status of Czar @@ -84,6 +89,15 @@ class HttpQservMonitorModule : public HttpModule { nlohmann::json executeImpl(std::string const& subModuleName) final; private: + /** + * The helper method for check the completion status of a request to ensure it succeded. + * @param func The calling context (for error reporting). + * @param request A request to be evaluated. + * @throw http::Error If the request didn't succeed. + */ + static void _throwIfNotSucceeded(std::string const& func, + std::shared_ptr const& request); + HttpQservMonitorModule(Controller::Ptr const& controller, std::string const& taskName, HttpProcessorConfig const& processorConfig, qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp); @@ -113,6 +127,12 @@ class HttpQservMonitorModule : public HttpModule { */ nlohmann::json _workerDb(); + /** + * Process a request for extracting info on the partial query result files + * from select Qserv worker. + */ + nlohmann::json _workerFiles(); + /** * Process a request for extracting various status info of Czar. */ diff --git a/src/replica/Mutex.cc b/src/replica/Mutex.cc index ce756b46bb..a4d9f463d2 100644 --- a/src/replica/Mutex.cc +++ b/src/replica/Mutex.cc @@ -28,7 +28,7 @@ // Qserv headers #include "lsst/log/Log.h" -#include "util/IterableFormatter.h" +#include "util/String.h" using namespace std; @@ -45,14 +45,14 @@ void Lock::_lock() { if (!_context.empty()) { LOGS(_log, LOG_LVL_TRACE, _context << " LOCK[" << _mutex.id() << "]:1 " - << " LOCKED: " << util::printable(Mutex::lockedId(), "", "", " ")); + << " LOCKED: " << util::String::toString(Mutex::lockedId())); } assert(!_mutex.lockedByCaller()); _mutex.lock(); if (!_context.empty()) { LOGS(_log, LOG_LVL_TRACE, _context << " LOCK[" << _mutex.id() << "]:2 " - << " LOCKED: " << util::printable(Mutex::lockedId(), "", "", " ")); + << " LOCKED: " << util::String::toString(Mutex::lockedId())); } } @@ -60,7 +60,7 @@ void Lock::_unlock() { if (!_context.empty()) { LOGS(_log, LOG_LVL_TRACE, _context << " LOCK[" << _mutex.id() << "]:3 " - << " LOCKED: " << util::printable(Mutex::lockedId(), "", "", " ")); + << " LOCKED: " << util::String::toString(Mutex::lockedId())); } _mutex.unlock(); } diff --git a/src/replica/QservMgtServices.cc b/src/replica/QservMgtServices.cc index 4b863a23e0..4a8b9a93a0 100644 --- a/src/replica/QservMgtServices.cc +++ b/src/replica/QservMgtServices.cc @@ -152,6 +152,19 @@ GetConfigQservMgtRequest::Ptr QservMgtServices::config(string const& worker, str return request; } +GetResultFilesQservMgtRequest::Ptr QservMgtServices::resultFiles( + string const& worker, string const& jobId, vector const& queryIds, unsigned int maxFiles, + GetResultFilesQservMgtRequest::CallbackType const& onFinish, unsigned int requestExpirationIvalSec) { + auto const request = GetResultFilesQservMgtRequest::create( + serviceProvider(), worker, queryIds, maxFiles, + [self = shared_from_this()](QservMgtRequest::Ptr const& request) { + self->_finish(request->id()); + }); + _register(__func__, request, onFinish); + request->start(jobId, requestExpirationIvalSec); + return request; +} + void QservMgtServices::_finish(string const& id) { string const context = "QservMgtServices::" + string(__func__) + "[" + id + "] "; LOGS(_log, LOG_LVL_TRACE, context); diff --git a/src/replica/QservMgtServices.h b/src/replica/QservMgtServices.h index 9366aeda1c..0481d0aa7f 100644 --- a/src/replica/QservMgtServices.h +++ b/src/replica/QservMgtServices.h @@ -27,10 +27,12 @@ #include // Qserv headers +#include "global/intTypes.h" #include "replica/AddReplicaQservMgtRequest.h" #include "replica/GetReplicasQservMgtRequest.h" #include "replica/GetDbStatusQservMgtRequest.h" #include "replica/GetConfigQservMgtRequest.h" +#include "replica/GetResultFilesQservMgtRequest.h" #include "replica/GetStatusQservMgtRequest.h" #include "replica/RemoveReplicaQservMgtRequest.h" #include "replica/SetReplicasQservMgtRequest.h" @@ -294,6 +296,28 @@ class QservMgtServices : public std::enable_shared_from_this { GetConfigQservMgtRequest::CallbackType const& onFinish = nullptr, unsigned int requestExpirationIvalSec = 0); + /** + * Request info on the partial result files of a Qserv worker + * @param worker The name of a worker. + * @param jobId An optional identifier of a job specifying a context in which + * a request will be executed. + * @param queryIds The optional selector for queries. If empty then all queries will + * be considered. + * @param maxFiles The optional limit for maximum number of files to be reported. + * If 0 then no limit is set. + * @param onFinish A callback function to be called upon request completion. + * @param requestExpirationIvalSec The maximum amount of time to wait before + * completion of the request. If a value of the parameter is set to 0 then no + * limit will be enforced. + * @return A pointer to the request object if the request was made. Return + * nullptr otherwise. + */ + GetResultFilesQservMgtRequest::Ptr resultFiles( + std::string const& worker, std::string const& jobId = "", + std::vector const& queryIds = std::vector(), unsigned int maxFiles = 0, + GetResultFilesQservMgtRequest::CallbackType const& onFinish = nullptr, + unsigned int requestExpirationIvalSec = 0); + private: /** * @param serviceProvider Is required for accessing configuration parameters. diff --git a/src/replica/SetReplicasQservMgtRequest.cc b/src/replica/SetReplicasQservMgtRequest.cc index 3a6c10e97a..160270a7bd 100644 --- a/src/replica/SetReplicasQservMgtRequest.cc +++ b/src/replica/SetReplicasQservMgtRequest.cc @@ -24,7 +24,6 @@ // System headers #include -#include #include // Qserv headers @@ -32,7 +31,7 @@ #include "replica/Common.h" #include "replica/Configuration.h" #include "replica/ServiceProvider.h" -#include "util/IterableFormatter.h" +#include "util/String.h" // LSST headers #include "lsst/log/Log.h" @@ -77,9 +76,7 @@ QservReplicaCollection const& SetReplicasQservMgtRequest::replicas() const { list> SetReplicasQservMgtRequest::extendedPersistentState() const { list> result; result.emplace_back("num_replicas", to_string(newReplicas().size())); - ostringstream ss; - ss << util::printable(_databases, "", "", ","); - result.emplace_back("databases", ss.str()); + result.emplace_back("databases", util::String::toString(_databases)); result.emplace_back("force", replica::bool2str(force())); return result; } diff --git a/src/replica/SqlRequest.cc b/src/replica/SqlRequest.cc index 77ea446ff5..96ff871889 100644 --- a/src/replica/SqlRequest.cc +++ b/src/replica/SqlRequest.cc @@ -25,8 +25,6 @@ // System headers #include #include -#include -#include // Third party headers #include "boost/date_time/posix_time/posix_time.hpp" @@ -36,7 +34,7 @@ #include "replica/DatabaseServices.h" #include "replica/Messenger.h" #include "replica/ServiceProvider.h" -#include "util/IterableFormatter.h" +#include "util/String.h" #include "util/TimeUtils.h" // LSST headers @@ -101,9 +99,7 @@ list> SqlRequest::extendedPersistentState() const { result.emplace_back("partition_by_column", requestBody.partition_by_column()); result.emplace_back("transaction_id", to_string(requestBody.transaction_id())); result.emplace_back("num_columns", to_string(requestBody.columns_size())); - ostringstream tablesStream; - tablesStream << util::printable(requestBody.tables(), "", "", " "); - result.emplace_back("tables", tablesStream.str()); + result.emplace_back("tables", util::String::toString(requestBody.tables())); result.emplace_back("batch_mode", bool2str(requestBody.batch_mode())); return result; } diff --git a/src/replica/XrdCmsgetVnId.cc b/src/replica/XrdCmsgetVnId.cc index 7fe1e3c84f..8467e6a1bb 100644 --- a/src/replica/XrdCmsgetVnId.cc +++ b/src/replica/XrdCmsgetVnId.cc @@ -29,6 +29,7 @@ #include "replica/Configuration.h" #include "replica/DatabaseMySQL.h" #include "replica/DatabaseMySQLUtils.h" +#include "util/String.h" // XrootD headers #include "XrdCms/XrdCmsVnId.hh" @@ -38,6 +39,7 @@ using namespace std; using namespace lsst::qserv::replica; using namespace lsst::qserv::replica::database::mysql; +namespace util = lsst::qserv::util; /** * @brief Read a value of the VNID from the Qserv worker database that's @@ -58,7 +60,8 @@ extern "C" string XrdCmsgetVnId(XrdCmsgetVnIdArgs) { string const context = string(__func__) + ": "; string vnId; try { - vector args = lsst::qserv::replica::strsplit(parms); + bool const greedy = true; + vector args = util::String::split(parms, " ", greedy); if (args.size() != 3) { eDest.Say(context.data(), "illegal number of parameters for the plugin. ", "Exactly 3 parameters are required: " diff --git a/src/replica/testCommonStr.cc b/src/replica/testCommonStr.cc deleted file mode 100644 index 7437e11cad..0000000000 --- a/src/replica/testCommonStr.cc +++ /dev/null @@ -1,88 +0,0 @@ -/* - * LSST Data Management System - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ -/** - * @brief Test data types shared by all classes of the module. - */ - -// System headers -#include -#include - -// LSST headers -#include "lsst/log/Log.h" - -// Qserv headers -#include "replica/Common.h" - -// Boost unit test header -#define BOOST_TEST_MODULE CommonStr -#include - -using namespace std; -namespace test = boost::test_tools; -using namespace lsst::qserv::replica; - -BOOST_AUTO_TEST_SUITE(Suite) - -BOOST_AUTO_TEST_CASE(CommonStrTest) { - LOGS_INFO("CommonStrTest test begins"); - - vector words; - BOOST_REQUIRE_NO_THROW({ - words = strsplit(""); - BOOST_CHECK(words.empty()); - }); - BOOST_REQUIRE_NO_THROW({ - words = strsplit("", '_'); - BOOST_CHECK(words.empty()); - }); - BOOST_REQUIRE_NO_THROW({ - words = strsplit(" a bc def "); - BOOST_CHECK_EQUAL(words.size(), 3U); - BOOST_CHECK_EQUAL(words[0], "a"); - BOOST_CHECK_EQUAL(words[1], "bc"); - BOOST_CHECK_EQUAL(words[2], "def"); - }); - BOOST_REQUIRE_NO_THROW({ - words = strsplit("a bc def"); - BOOST_CHECK_EQUAL(words.size(), 3U); - BOOST_CHECK_EQUAL(words[0], "a"); - BOOST_CHECK_EQUAL(words[1], "bc"); - BOOST_CHECK_EQUAL(words[2], "def"); - }); - BOOST_REQUIRE_NO_THROW({ - words = strsplit("a_bc_def", '_'); - BOOST_CHECK_EQUAL(words.size(), 3U); - BOOST_CHECK_EQUAL(words[0], "a"); - BOOST_CHECK_EQUAL(words[1], "bc"); - BOOST_CHECK_EQUAL(words[2], "def"); - }); - BOOST_REQUIRE_NO_THROW({ - words = strsplit("_a_bc_def_", '_'); - BOOST_CHECK_EQUAL(words.size(), 3U); - BOOST_CHECK_EQUAL(words[0], "a"); - BOOST_CHECK_EQUAL(words[1], "bc"); - BOOST_CHECK_EQUAL(words[2], "def"); - }); - LOGS_INFO("CommonStrTest test ends"); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 393de0e73f..02c3b5896a 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -21,7 +21,7 @@ target_sources(util PRIVATE Mutex.cc SemaMgr.cc StringHash.cc - StringHelper.cc + String.cc Substitution.cc TablePrinter.cc ThreadPool.cc @@ -58,5 +58,6 @@ util_tests( testHistogram testMultiError testMutex + testString testTablePrinter ) diff --git a/src/util/String.cc b/src/util/String.cc new file mode 100644 index 0000000000..cb1555a27a --- /dev/null +++ b/src/util/String.cc @@ -0,0 +1,110 @@ +// -*- LSST-C++ -*- +/* + * LSST Data Management System + * Copyright 2020 LSST. + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +// Class header +#include "util/String.h" + +// System headers +#include +#include + +// LSST headers +#include "lsst/log/Log.h" + +using namespace std; + +#define CONTEXT_(func) ("String::" + string(func) + " ") + +namespace { + +LOG_LOGGER _log = LOG_GET("lsst.qserv.util.String"); + +template +vector getNumericVectFromStr(string const& func, vector const& strings, + function const& parseNumber, bool throwOnError, + T defaultVal) { + vector result; + for (string const& str : strings) { + try { + size_t sz = 0; + T const val = parseNumber(str, sz); + if (sz != str.length()) { + LOGS(_log, LOG_LVL_WARN, + CONTEXT_(func) << "unused characters when converting '" << str << "' to " << val); + } + result.push_back(val); + } catch (invalid_argument const& ex) { + string const msg = CONTEXT_(func) + "unable to parse '" + str + "', ex: " + string(ex.what()); + LOGS(_log, LOG_LVL_ERROR, msg); + if (throwOnError) throw invalid_argument(msg); + result.push_back(defaultVal); + } + } + return result; +} +} // namespace + +namespace lsst::qserv::util { + +vector String::split(string const& original, string const& delimiter, bool greedy) { + // Apply trivial optimizations. Note that the specified "greedy" behavior + // must be preserved during the optimisations. + vector result; + if (original.empty()) { + if (!greedy) result.push_back(original); + return result; + } + if (delimiter.empty()) { + if (!original.empty() || !greedy) result.push_back(original); + return result; + } + string str(original); + size_t pos; + bool loop = true; + while (loop) { + pos = str.find(delimiter); + if (pos == string::npos) { + loop = false; + } + auto const candidate = str.substr(0, pos); + if (!candidate.empty() || !greedy) result.push_back(candidate); + str = str.substr(pos + delimiter.length()); + } + return result; +} + +vector String::parseToVectInt(string const& str, string const& delimiter, bool throwOnError, + int defaultVal, bool greedy) { + auto const parseNumber = [](string const& str, size_t& sz) -> int { return stoi(str, &sz); }; + return ::getNumericVectFromStr(__func__, split(str, delimiter, greedy), parseNumber, throwOnError, + defaultVal); +} + +vector String::parseToVectUInt64(string const& str, string const& delimiter, bool throwOnError, + uint64_t defaultVal, bool greedy) { + auto const parseNumber = [](string const& str, size_t& sz) -> uint64_t { return stoull(str, &sz); }; + return ::getNumericVectFromStr(__func__, split(str, delimiter, greedy), parseNumber, + throwOnError, defaultVal); +} + +} // namespace lsst::qserv::util diff --git a/src/util/String.h b/src/util/String.h new file mode 100644 index 0000000000..b04e0cd694 --- /dev/null +++ b/src/util/String.h @@ -0,0 +1,104 @@ +// -*- LSST-C++ -*- +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +#ifndef LSST_QSERV_UTIL_STRING_H +#define LSST_QSERV_UTIL_STRING_H + +// System headers +#include +#include +#include +#include + +namespace lsst::qserv::util { + +/// Functions to help with string processing. +class String { +public: + /** + * Split the input string into substrings using the specified delimiter. + * @param str The input string to be parsed. + * @param delimiter A delimiter. + * @param greedy The optional flag that if 'true' would eliminate empty strings from + * the result. Otherwise the empty strings found between the delimiter will + * be preserved in the result. + * @note The result filtering requested by a value of the parameter 'greedy' + * also applies to a secenario when the input string is empty. In particular, + * in the 'greedy' mode the output collection will be empty. Otherwise, the collection + * will have exactly one element - the empty string. + * @return A collection of strings resulting from splitting the input string into + * sub-strings using the delimiter. The delimiter won't be included into the substrings. + */ + static std::vector split(std::string const& str, std::string const& delimiter, + bool greedy = false); + + /** + * Parse the input string into a collection of numbers (int). + * @param str The input string to be parsed. + * @param delimiter A delimiter. + * @param throwOnError The optional flag telling the function what to do when conversions + * from substrings to numbers fail. If the flag is set to 'true' then an exception + * will be thrown. Otherwise, the default value will be placed into the output collection. + * @param defaultVal The optional default value to be injected where applies. + * @param greedy The optional flag that if 'true' would eliminate empty substrings from + * parsing into the the numeric result. Otherwise the behavior of the methjod will be driven + * by a value of the parameter 'throwOnError'. + * @return A collection of numbers found in the input string. + * @throw std::invalid_argument If substrings found within the input string can't be + * interpreted as numbers of the given type. + * @see function split + */ + static std::vector parseToVectInt(std::string const& str, std::string const& delimiter, + bool throwOnError = true, int defaultVal = 0, bool greedy = false); + + /** + * Parse the input string into a collection of numbers (std::uint64_t). + * @see function parseToVectInt + */ + static std::vector parseToVectUInt64(std::string const& str, std::string const& delimiter, + bool throwOnError = true, + std::uint64_t defaultVal = 0ULL, bool greedy = false); + + /** + * Pack an iterable collection into a string. + * @param coll The input collection. + * @param delimiter An (optional) delimiter between elements. + * @param openingBracket An (optional) opening bracket. + * @param closingBracket An (optional) closing bracket. + * @return The string representation of the collection. + */ + template + static std::string toString(COLLECTION const& coll, std::string const& delimiter = ",", + std::string const& openingBracket = "", + std::string const& closingBracket = "") { + std::ostringstream ss; + for (auto itr = coll.begin(); itr != coll.end(); ++itr) { + if (coll.begin() != itr) ss << delimiter; + ss << openingBracket << *itr << closingBracket; + } + return ss.str(); + } +}; + +} // namespace lsst::qserv::util + +#endif // LSST_QSERV_UTIL_STRING_H diff --git a/src/util/StringHelper.cc b/src/util/StringHelper.cc deleted file mode 100644 index 93158da9bb..0000000000 --- a/src/util/StringHelper.cc +++ /dev/null @@ -1,84 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2020 LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -// Class header -#include "StringHelper.h" - -// System headers -#include - -// LSST headers -#include "lsst/log/Log.h" - -using namespace std; - -namespace { - -LOG_LOGGER _log = LOG_GET("lsst.qserv.util.StringToVector"); - -} - -namespace lsst::qserv::util { - -vector StringHelper::splitString(string const& original, string const& separator) { - vector result; - string str(original); - size_t pos; - bool loop = true; - while (loop) { - pos = str.find(separator); - if (pos == string::npos) { - loop = false; - } - result.push_back(str.substr(0, pos)); - str = str.substr(pos + separator.length()); - } - return result; -} - -vector StringHelper::getIntVectFromStr(string const& str, string const& separator, bool throwOnError, - int defaultVal) { - auto vectString = splitString(str, separator); - vector result; - for (auto iStr : vectString) { - try { - size_t sz = 0; - int val = stoi(iStr, &sz); - if (sz != iStr.length()) { - LOGS(_log, LOG_LVL_WARN, "unused characters when converting " << iStr << " to " << val); - } - result.push_back(val); - } catch (invalid_argument const& e) { - string msg("getIntsFromString invalid argument in str=" + str + " iStr=" + iStr); - LOGS(_log, LOG_LVL_ERROR, msg); - if (throwOnError) { - throw invalid_argument(msg); - } else { - result.push_back(defaultVal); - } - } - } - return result; -} - -} // namespace lsst::qserv::util diff --git a/src/util/StringHelper.h b/src/util/StringHelper.h deleted file mode 100644 index 6b843d00ff..0000000000 --- a/src/util/StringHelper.h +++ /dev/null @@ -1,55 +0,0 @@ -// -*- LSST-C++ -*- -/* - * LSST Data Management System - * Copyright 2020 LSST. - * - * This product includes software developed by the - * LSST Project (http://www.lsst.org/). - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the LSST License Statement and - * the GNU General Public License along with this program. If not, - * see . - */ - -#ifndef LSST_QSERV_UTIL_STRINGHELPER_H -#define LSST_QSERV_UTIL_STRINGHELPER_H - -// System headers -#include -#include - -// Qserv headers -#include "util/Command.h" - -namespace lsst::qserv::util { - -/// Functions to help with string processing. -class StringHelper { -public: - /// @return a vector of strings resulting from splitting 'str' into separate strings - /// using 'separator' as the delimiter. - /// TODO: If other return types are needed, make a template version. - static std::vector splitString(std::string const& str, std::string const& separator); - - /// @return a vector of int resulting from splitting 'str' into separate strings - /// and converting those strings into integers. - /// Throws invalid_argument if throwOnError is true and one of the strings fails conversion. - /// If throwOnError is false, the default value will be used for that entry and nothing will be thrown. - /// TODO: If other return types are needed, make a template version. - static std::vector getIntVectFromStr(std::string const& str, std::string const& separator, - bool throwOnError = true, int defaultVal = 0); -}; - -} // namespace lsst::qserv::util - -#endif // LSST_QSERV_UTIL_STRINGHELPER_H diff --git a/src/util/testCommon.cc b/src/util/testCommon.cc index 223ee3d337..52ba97f7f0 100644 --- a/src/util/testCommon.cc +++ b/src/util/testCommon.cc @@ -29,6 +29,7 @@ // System headers #include +#include #include #include @@ -37,10 +38,8 @@ // LSST headers #include "lsst/log/Log.h" -#include "StringHelper.h" // Qserv headers #include "util/common.h" -#include "util/IterableFormatter.h" // Boost unit test header #define BOOST_TEST_MODULE common @@ -98,98 +97,4 @@ BOOST_AUTO_TEST_CASE(prettyPrint) { BOOST_CHECK(strBuf3.compare(expectedList3) == 0); } -BOOST_AUTO_TEST_CASE(stringToVector) { - { - auto vect = util::StringHelper::splitString("testing123,qsa4$3,hjdw q,,7321,ml;oujh", ","); - LOGS_ERROR("vect=" << util::printable(vect)); - BOOST_CHECK(vect.size() == 6); - BOOST_CHECK(vect[0] == "testing123"); - BOOST_CHECK(vect[1] == "qsa4$3"); - BOOST_CHECK(vect[2] == "hjdw q"); - BOOST_CHECK(vect[3] == ""); - BOOST_CHECK(vect[4] == "7321"); - BOOST_CHECK(vect[5] == "ml;oujh"); - } - { - auto vect = util::StringHelper::splitString("testing123::q:sa4$3:::hjdw q::::7321::ml;oujh", "::"); - BOOST_CHECK(vect.size() == 6); - BOOST_CHECK(vect[0] == "testing123"); - BOOST_CHECK(vect[1] == "q:sa4$3"); - BOOST_CHECK(vect[2] == ":hjdw q"); - BOOST_CHECK(vect[3] == ""); - BOOST_CHECK(vect[4] == "7321"); - BOOST_CHECK(vect[5] == "ml;oujh"); - } - { - auto vect = util::StringHelper::splitString(":testing123:qsa4$3:hjdw q::7321:ml;oujh:", ":"); - BOOST_CHECK(vect.size() == 8); - BOOST_CHECK(vect[0] == ""); - BOOST_CHECK(vect[1] == "testing123"); - BOOST_CHECK(vect[2] == "qsa4$3"); - BOOST_CHECK(vect[3] == "hjdw q"); - BOOST_CHECK(vect[4] == ""); - BOOST_CHECK(vect[5] == "7321"); - BOOST_CHECK(vect[6] == "ml;oujh"); - BOOST_CHECK(vect[7] == ""); - } - { - auto vect = util::StringHelper::splitString("qsa4$3", ":"); - BOOST_CHECK(vect.size() == 1); - BOOST_CHECK(vect[0] == "qsa4$3"); - } - { - auto vect = util::StringHelper::splitString("", ":"); - BOOST_CHECK(vect.size() == 1); - BOOST_CHECK(vect[0] == ""); - } - - { - auto vect = util::StringHelper::getIntVectFromStr("987:23:0:1:-123", ":"); - unsigned int j = 0; - BOOST_CHECK(vect[j++] == 987); - BOOST_CHECK(vect[j++] == 23); - BOOST_CHECK(vect[j++] == 0); - BOOST_CHECK(vect[j++] == 1); - BOOST_CHECK(vect[j++] == -123); - BOOST_CHECK(vect.size() == j); - } - { - bool caught = false; - try { - auto vect = util::StringHelper::getIntVectFromStr("987:23:x:1:-123", ":"); - } catch (std::invalid_argument const& e) { - caught = true; - } - BOOST_CHECK(caught); - } - { - auto vect = util::StringHelper::getIntVectFromStr("987:23:x8owlq:1:-123:", ":", false, 99); - unsigned int j = 0; - BOOST_CHECK(vect[j++] == 987); - BOOST_CHECK(vect[j++] == 23); - BOOST_CHECK(vect[j++] == 99); - BOOST_CHECK(vect[j++] == 1); - BOOST_CHECK(vect[j++] == -123); - BOOST_CHECK(vect[j++] == 99); - BOOST_CHECK(vect.size() == j); - } -} -/* - test::output_test_stream output; - util::MultiError multiError; - - std::string expected_err_msg = util::MultiError::HEADER_MSG + - "\t[1] Stupid error message"; - - int errCode = 1; - std::string errMsg = "Stupid error message"; - util::Error error(errCode, errMsg); - multiError.push_back(error); - - output << multiError; - std::cout << multiError; - BOOST_REQUIRE(output.is_equal(expected_err_msg)); -} -*/ - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/util/testString.cc b/src/util/testString.cc new file mode 100644 index 0000000000..a549c65c6a --- /dev/null +++ b/src/util/testString.cc @@ -0,0 +1,265 @@ +// -*- LSST-C++ -*- +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +// System headers +#include +#include +#include + +// LSST headers +#include "lsst/log/Log.h" + +// Qserv headers +#include "util/String.h" + +// Boost unit test header +#define BOOST_TEST_MODULE String +#include + +namespace test = boost::test_tools; +namespace util = lsst::qserv::util; + +BOOST_AUTO_TEST_SUITE(Suite) + +BOOST_AUTO_TEST_CASE(SplitStringTest) { + LOGS_INFO("SplitStringTest begins"); + { + std::string const emptyStr; + std::string const delimiter = " "; + auto const vect = util::String::split(emptyStr, delimiter); + LOGS_ERROR("vect=" << util::String::toString(vect, delimiter, "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], emptyStr); + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + std::string const emptyStr; + std::string const delimiter = " "; + bool const greedy = true; + auto const vect = util::String::split(emptyStr, delimiter, greedy); + LOGS_ERROR("vect=" << util::String::toString(vect, delimiter, "'", "'")); + BOOST_CHECK_EQUAL(vect.size(), 0UL); + } + { + std::string const str = " a b cd e f "; + std::string const emptyDelimiter; + auto const vect = util::String::split(str, emptyDelimiter); + LOGS_ERROR("vect=" << util::String::toString(vect, emptyDelimiter, "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], str); + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + auto const vect = util::String::split(" a b cd e f ", " "); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], ""); + BOOST_CHECK_EQUAL(vect[j++], "a"); + BOOST_CHECK_EQUAL(vect[j++], "b"); + BOOST_CHECK_EQUAL(vect[j++], ""); + BOOST_CHECK_EQUAL(vect[j++], "cd"); + BOOST_CHECK_EQUAL(vect[j++], ""); + BOOST_CHECK_EQUAL(vect[j++], ""); + BOOST_CHECK_EQUAL(vect[j++], "e"); + BOOST_CHECK_EQUAL(vect[j++], "f"); + BOOST_CHECK_EQUAL(vect[j++], ""); + BOOST_CHECK_EQUAL(vect[j++], ""); + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + bool const greedy = true; + auto const vect = util::String::split(" a b cd e f ", " ", greedy); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], "a"); + BOOST_CHECK_EQUAL(vect[j++], "b"); + BOOST_CHECK_EQUAL(vect[j++], "cd"); + BOOST_CHECK_EQUAL(vect[j++], "e"); + BOOST_CHECK_EQUAL(vect[j++], "f"); + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + auto const vect = util::String::split("testing123,qsa4$3,hjdw q,,7321,ml;oujh", ","); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + BOOST_CHECK_EQUAL(vect.size(), 6U); + BOOST_CHECK_EQUAL(vect[0], "testing123"); + BOOST_CHECK_EQUAL(vect[1], "qsa4$3"); + BOOST_CHECK_EQUAL(vect[2], "hjdw q"); + BOOST_CHECK_EQUAL(vect[3], ""); + BOOST_CHECK_EQUAL(vect[4], "7321"); + BOOST_CHECK_EQUAL(vect[5], "ml;oujh"); + } + { + auto const vect = util::String::split("testing123::q:sa4$3:::hjdw q::::7321::ml;oujh", "::"); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + BOOST_CHECK_EQUAL(vect.size(), 6U); + BOOST_CHECK_EQUAL(vect[0], "testing123"); + BOOST_CHECK_EQUAL(vect[1], "q:sa4$3"); + BOOST_CHECK_EQUAL(vect[2], ":hjdw q"); + BOOST_CHECK_EQUAL(vect[3], ""); + BOOST_CHECK_EQUAL(vect[4], "7321"); + BOOST_CHECK_EQUAL(vect[5], "ml;oujh"); + } + { + auto const vect = util::String::split(":testing123:qsa4$3:hjdw q::7321:ml;oujh:", ":"); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + BOOST_CHECK_EQUAL(vect.size(), 8U); + BOOST_CHECK_EQUAL(vect[0], ""); + BOOST_CHECK_EQUAL(vect[1], "testing123"); + BOOST_CHECK_EQUAL(vect[2], "qsa4$3"); + BOOST_CHECK_EQUAL(vect[3], "hjdw q"); + BOOST_CHECK_EQUAL(vect[4], ""); + BOOST_CHECK_EQUAL(vect[5], "7321"); + BOOST_CHECK_EQUAL(vect[6], "ml;oujh"); + BOOST_CHECK_EQUAL(vect[7], ""); + } + { + auto const vect = util::String::split("qsa4$3", ":"); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + BOOST_CHECK_EQUAL(vect.size(), 1U); + BOOST_CHECK_EQUAL(vect[0], "qsa4$3"); + } + { + auto const vect = util::String::split("", ":"); + BOOST_CHECK_EQUAL(vect.size(), 1U); + BOOST_CHECK_EQUAL(vect[0], ""); + } +} + +BOOST_AUTO_TEST_CASE(GetVecFromStrTest) { + LOGS_INFO("GetVecFromStrTest begins"); + std::string const str11 = "987:23:0:1:-123"; + std::string const str12 = "987:23:x:1:-123"; + { + auto const vect = util::String::parseToVectInt(str11, ":"); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], 987); + BOOST_CHECK_EQUAL(vect[j++], 23); + BOOST_CHECK_EQUAL(vect[j++], 0); + BOOST_CHECK_EQUAL(vect[j++], 1); + BOOST_CHECK_EQUAL(vect[j++], -123); + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + bool caught = false; + try { + auto vect = util::String::parseToVectInt(str12, ":"); + } catch (std::invalid_argument const& e) { + caught = true; + } + BOOST_CHECK(caught); + } + std::string const str2 = ":987:23:x8owlq:1:-123:"; + { + int const defaultVal = 99; + auto const vect = util::String::parseToVectInt(str2, ":", false, defaultVal); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // The empty string in the non-greedy mode + BOOST_CHECK_EQUAL(vect[j++], 987); + BOOST_CHECK_EQUAL(vect[j++], 23); + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // Couldn't parse "x8owlq" as a number + BOOST_CHECK_EQUAL(vect[j++], 1); + BOOST_CHECK_EQUAL(vect[j++], -123); + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // The empty string in the non-greedy mode + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + int const defaultVal = 99; + bool const greedy = true; + auto const vect = util::String::parseToVectInt(str2, ":", false, defaultVal, greedy); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], 987); + BOOST_CHECK_EQUAL(vect[j++], 23); + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // Couldn't parse "x8owlq" as a number + BOOST_CHECK_EQUAL(vect[j++], 1); + BOOST_CHECK_EQUAL(vect[j++], -123); + BOOST_CHECK_EQUAL(vect.size(), j); + } + std::string const str3 = ":123456789123123:23:x8owlq::1:-123:"; + { + auto const defaultVal = std::numeric_limits::max(); + bool const greedy = true; + auto const vect = util::String::parseToVectUInt64(str3, ":", false, defaultVal, greedy); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], 123456789123123ULL); + BOOST_CHECK_EQUAL(vect[j++], 23ULL); + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // Couldn't parse "x8owlq" as a number + BOOST_CHECK_EQUAL(vect[j++], 1ULL); + BOOST_CHECK_EQUAL(vect[j++], -123LL); + BOOST_CHECK_EQUAL(vect.size(), j); + } + { + auto const defaultVal = std::numeric_limits::max(); + auto const vect = util::String::parseToVectUInt64(str3, ":", false, defaultVal); + LOGS_ERROR("vect=" << util::String::toString(vect, " ", "'", "'")); + size_t j = 0; + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // The empty string in the non-greedy mode + BOOST_CHECK_EQUAL(vect[j++], 123456789123123ULL); + BOOST_CHECK_EQUAL(vect[j++], 23ULL); + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // Couldn't parse "x8owlq" as a number + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // The empty string in the non-greedy mode + BOOST_CHECK_EQUAL(vect[j++], 1ULL); + BOOST_CHECK_EQUAL(vect[j++], -123LL); + BOOST_CHECK_EQUAL(vect[j++], defaultVal); // The empty string in the non-greedy mode + BOOST_CHECK_EQUAL(vect.size(), j); + } +} + +BOOST_AUTO_TEST_CASE(ToStringTest) { + LOGS_INFO("ToStringTest test begins"); + + // These values match thevdefault values of the corresponding parameters of + // the utility function. + std::string sep = ","; + std::string openBrkt = ""; + std::string closeBrkt = ""; + + std::vector const empty; + BOOST_CHECK_EQUAL(util::String::toString(empty), ""); + BOOST_CHECK_EQUAL(util::String::toString(empty, " "), ""); + BOOST_CHECK_EQUAL(util::String::toString(empty, sep, openBrkt, closeBrkt), ""); + + std::vector const one = {1}; + BOOST_CHECK_EQUAL(util::String::toString(one), "1"); + BOOST_CHECK_EQUAL(util::String::toString(one, " "), "1"); + BOOST_CHECK_EQUAL(util::String::toString(one, "", openBrkt, closeBrkt), "1"); + + std::vector const integers = {1, 2, 3, 4, 5}; + BOOST_CHECK_EQUAL(util::String::toString(integers), "1,2,3,4,5"); + BOOST_CHECK_EQUAL(util::String::toString(integers, sep), "1,2,3,4,5"); + BOOST_CHECK_EQUAL(util::String::toString(integers, " ", openBrkt, closeBrkt), "1 2 3 4 5"); + BOOST_CHECK_EQUAL(util::String::toString(integers, "", openBrkt, closeBrkt), "12345"); + + std::vector const strings = {"a", "b", "c", "d", "e"}; + BOOST_CHECK_EQUAL(util::String::toString(strings), "a,b,c,d,e"); + BOOST_CHECK_EQUAL(util::String::toString(strings, sep), "a,b,c,d,e"); + BOOST_CHECK_EQUAL(util::String::toString(strings, " ", openBrkt, closeBrkt), "a b c d e"); + BOOST_CHECK_EQUAL(util::String::toString(strings, "", openBrkt, closeBrkt), "abcde"); + BOOST_CHECK_EQUAL(util::String::toString(strings, sep, "[", "]"), "[a],[b],[c],[d],[e]"); + BOOST_CHECK_EQUAL(util::String::toString(strings, " ", "[", "]"), "[a] [b] [c] [d] [e]"); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wbase/FileChannelShared.cc b/src/wbase/FileChannelShared.cc index df0d617131..11ec73af0a 100644 --- a/src/wbase/FileChannelShared.cc +++ b/src/wbase/FileChannelShared.cc @@ -24,6 +24,7 @@ // System headers #include +#include #include // Third party headers @@ -37,7 +38,9 @@ #include "wconfig/WorkerConfig.h" #include "wpublish/QueriesAndChunks.h" #include "util/MultiError.h" +#include "util/String.h" #include "util/Timer.h" +#include "util/TimeUtils.h" // LSST headers #include "lsst/log/Log.h" @@ -45,12 +48,42 @@ using namespace std; using namespace nlohmann; namespace fs = boost::filesystem; +namespace util = lsst::qserv::util; namespace wconfig = lsst::qserv::wconfig; namespace { LOG_LOGGER _log = LOG_GET("lsst.qserv.wbase.FileChannelShared"); +string const resultFileExt = ".proto"; + +bool isResultFile(fs::path const& filePath) { + return filePath.has_filename() && filePath.has_extension() && (filePath.extension() == resultFileExt); +} + +/** + * Extract task attributes from the file path. + * The file path is required to have the following format: + * @code + * [/]---[.] + * @code + * @param filePath The file to be evaluated. + * @return nlohmann::json::object Task attributes. + * @throw std::invalid_argument If the file path did not match expectations. + */ +json file2task(fs::path const& filePath) { + vector const taskAttributes = + util::String::parseToVectUInt64(filePath.stem().string(), "-"); + if (taskAttributes.size() != 4) { + throw invalid_argument("FileChannelShared::" + string(__func__) + + " not a valid result file: " + filePath.string()); + } + return json::object({{"query_id", taskAttributes[0]}, + {"job_id", taskAttributes[1]}, + {"chunk_id", taskAttributes[2]}, + {"attemptcount", taskAttributes[3]}}); +} + /** * Iterate over the result files at the results folder and remove those * which satisfy the desired criteria. @@ -67,7 +100,6 @@ LOG_LOGGER _log = LOG_GET("lsst.qserv.wbase.FileChannelShared"); size_t cleanUpResultsImpl(string const& context, fs::path const& dirPath, function fileCanBeRemoved = nullptr) { size_t numFilesRemoved = 0; - string const ext = ".proto"; boost::system::error_code ec; auto itr = fs::directory_iterator(dirPath, ec); if (ec.value() != 0) { @@ -79,7 +111,7 @@ size_t cleanUpResultsImpl(string const& context, fs::path const& dirPath, for (auto&& entry : boost::make_iterator_range(itr, {})) { auto filePath = entry.path(); bool const removeIsCleared = - filePath.has_filename() && filePath.has_extension() && (filePath.extension() == ext) && + ::isResultFile(filePath) && ((fileCanBeRemoved == nullptr) || fileCanBeRemoved(filePath.filename().string())); if (removeIsCleared) { fs::remove_all(filePath, ec); @@ -173,11 +205,10 @@ json FileChannelShared::statusToJson() { result["available_bytes"] = space.available; uintmax_t sizeResultFilesBytes = 0; uintmax_t numResultFiles = 0; - string const ext = ".proto"; auto itr = fs::directory_iterator(dirPath); for (auto&& entry : boost::make_iterator_range(itr, {})) { auto const filePath = entry.path(); - if (filePath.has_filename() && filePath.has_extension() && (filePath.extension() == ext)) { + if (::isResultFile(filePath)) { numResultFiles++; sizeResultFilesBytes += fs::file_size(filePath); } @@ -191,6 +222,57 @@ json FileChannelShared::statusToJson() { return result; } +json FileChannelShared::filesToJson(vector const& queryIds, unsigned int maxFiles) { + string const context = "FileChannelShared::" + string(__func__) + " "; + set queryIdsFilter; + for (auto const queryId : queryIds) { + queryIdsFilter.insert(queryId); + } + auto const config = wconfig::WorkerConfig::instance(); + fs::path const dirPath = config->resultsDirname(); + unsigned int numTotal = 0; + unsigned int numSelected = 0; + json files = json::array(); + lock_guard const lock(_resultsDirCleanupMtx); + try { + auto itr = fs::directory_iterator(dirPath); + for (auto&& entry : boost::make_iterator_range(itr, {})) { + auto const filePath = entry.path(); + if (::isResultFile(filePath)) { + ++numTotal; + + // Skip files not matching the query criteria if the one was requested. + json const jsonTask = ::file2task(filePath); + QueryId const queryId = jsonTask.at("query_id"); + if (!queryIdsFilter.empty() && !queryIdsFilter.contains(queryId)) continue; + + // Stop collecting files after reaching the limit (if any). And keep counting. + ++numSelected; + if ((maxFiles != 0) && (files.size() >= maxFiles)) continue; + + // A separate exception handler to avoid and ignore race conditions if + // the current file gets deleted. In this scenario the file will not be + // reported in the result. + try { + files.push_back(json::object({{"filename", filePath.filename().string()}, + {"size", fs::file_size(filePath)}, + {"ctime", fs::creation_time(filePath)}, + {"mtime", fs::last_write_time(filePath)}, + {"current_time_ms", util::TimeUtils::now()}, + {"task", jsonTask}})); + } catch (exception const& ex) { + LOGS(_log, LOG_LVL_WARN, + context << "failed to get info on files at " << dirPath << ", ex: " << ex.what()); + } + } + } + } catch (exception const& ex) { + LOGS(_log, LOG_LVL_WARN, + context << "failed to iterate over files at " << dirPath << ", ex: " << ex.what()); + } + return json::object({{"files", files}, {"num_selected", numSelected}, {"num_total", numTotal}}); +} + FileChannelShared::Ptr FileChannelShared::create(shared_ptr const& sendChannel, shared_ptr const& transmitMgr, shared_ptr const& taskMsg) { diff --git a/src/wbase/FileChannelShared.h b/src/wbase/FileChannelShared.h index 8b28950962..62fb5e287a 100644 --- a/src/wbase/FileChannelShared.h +++ b/src/wbase/FileChannelShared.h @@ -28,6 +28,7 @@ #include #include #include +#include // Third-party headers #include @@ -98,6 +99,16 @@ class FileChannelShared : public ChannelShared { /// @return Status and statistics on the results folder (capacity, usage, etc.) static nlohmann::json statusToJson(); + /** + * Locate existing result files. + * @param queryIds The optional selector for queries. If the collection is empty + * then all queries will be considered. + * @param maxFiles The optional limit for maximum number of files to be reported. + * If 0 then no limit is set. + * @return A collection of the results files matching the optional filter. + */ + static nlohmann::json filesToJson(std::vector const& queryIds, unsigned int maxFiles); + /// The factory method for the channel class. static Ptr create(std::shared_ptr const& sendChannel, std::shared_ptr const& transmitMgr, diff --git a/src/www/qserv/css/QservWorkerFiles.css b/src/www/qserv/css/QservWorkerFiles.css new file mode 100644 index 0000000000..6d00c1f38f --- /dev/null +++ b/src/www/qserv/css/QservWorkerFiles.css @@ -0,0 +1,24 @@ +#fwk-qserv-files-controls label { + font-weight: bold; +} +table#fwk-qserv-files caption { + caption-side: top; + text-align: right; + padding-top: 0; +} +table#fwk-qserv-files > thead > tr > th.sticky { + position:sticky; + top:80px; + z-index:2; +} +table#fwk-qserv-files tbody th, +table#fwk-qserv-files tbody td { + vertical-align:middle; +} +table#fwk-qserv-files pre { + padding: 0; + margin: 0; +} +table#fwk-qserv-files caption.updating { + background-color: #ffeeba; +} diff --git a/src/www/qserv/css/QservWorkerResultsFilesystem.css b/src/www/qserv/css/QservWorkerResultsFilesystem.css index 8f1a2669b6..37b02f93b0 100644 --- a/src/www/qserv/css/QservWorkerResultsFilesystem.css +++ b/src/www/qserv/css/QservWorkerResultsFilesystem.css @@ -22,3 +22,6 @@ table#fwk-qserv-results-filesystem > thead > tr > th.sticky { top:80px; z-index:2; } +table#fwk-qserv-results-filesystem tbody > tr.display-worker-files:hover { + cursor:pointer; +} \ No newline at end of file diff --git a/src/www/qserv/js/Common.js b/src/www/qserv/js/Common.js index cda8f05054..202fb7c8c9 100644 --- a/src/www/qserv/js/Common.js +++ b/src/www/qserv/js/Common.js @@ -6,7 +6,7 @@ function(sqlFormatter, _) { class Common { - static RestAPIVersion = 27; + static RestAPIVersion = 28; static query2text(query, expanded) { if (expanded) { return sqlFormatter.format(query, Common._sqlFormatterConfig); @@ -37,6 +37,16 @@ function(sqlFormatter, }, '') + ` `; } + static KB = 1000; + static MB = 1000 * 1000; + static GB = 1000 * 1000 * 1000; + static format_data_rate(v) { + if (v == 0) return v + ""; // as string + else if (v < Common.KB * 10) return v.toFixed(0); + else if (v < Common.MB * 10) return (v / Common.KB).toFixed(0) + " KB"; + else if (v < Common.GB * 10) return (v / Common.MB).toFixed(0) + " MB"; + else return (v / Common.GB).toFixed(0) + " GB"; + } } return Common; }); diff --git a/src/www/qserv/js/QservCzarStatistics.js b/src/www/qserv/js/QservCzarStatistics.js index f61022946c..3455f26c6e 100644 --- a/src/www/qserv/js/QservCzarStatistics.js +++ b/src/www/qserv/js/QservCzarStatistics.js @@ -269,7 +269,7 @@ function(CSSLoader, if (runTimeSec > 0) { const perf = data.qdisp_stats[counter] / runTimeSec; if (QservCzarStatistics._totals_data_rate.has(counter)) { - that._set_counter_perf('totals', counter, QservCzarStatistics._format_data_rate(perf), '_sum'); + that._set_counter_perf('totals', counter, Common.format_data_rate(perf), '_sum'); } else { that._set_counter_perf('totals', counter, perf.toFixed(0), '_sum'); } @@ -281,7 +281,7 @@ function(CSSLoader, if (deltaT > 0) { const perf = deltaVal / deltaT; if (QservCzarStatistics._totals_data_rate.has(counter)) { - that._set_counter_perf('totals', counter, QservCzarStatistics._format_data_rate(perf)); + that._set_counter_perf('totals', counter, Common.format_data_rate(perf)); } else { that._set_counter_perf('totals', counter, perf.toFixed(0)); } @@ -366,26 +366,16 @@ function(CSSLoader, }, '') + ` `; } - static _KB = 1000; - static _MB = 1000 * 1000; - static _GB = 1000 * 1000 * 1000; static _format_bucket_limit(v, data_rate=false) { if (isNaN(v)) return v; if (data_rate) { - if (v < QservCzarStatistics._KB) return v + " B/s"; - else if (v < QservCzarStatistics._MB) return (v / QservCzarStatistics._KB).toFixed(0) + " KB/s"; - else if (v < QservCzarStatistics._GB) return (v / QservCzarStatistics._MB).toFixed(0) + " MB/s"; - return (v / QservCzarStatistics._GB).toFixed(0) + " GB/s"; + if (v < Common.KB) return v + " B/s"; + else if (v < Common.MB) return (v / Common.KB).toFixed(0) + " KB/s"; + else if (v < Common.GB) return (v / Common.MB).toFixed(0) + " MB/s"; + return (v / Common.GB).toFixed(0) + " GB/s"; } return v.toLocaleString(); } - static _format_data_rate(v) { - if (v == 0) return v + ""; // as string - else if (v < QservCzarStatistics._KB * 10) return v.toFixed(0); - else if (v < QservCzarStatistics._MB * 10) return (v / QservCzarStatistics._KB).toFixed(0) + " KB"; - else if (v < QservCzarStatistics._GB * 10) return (v / QservCzarStatistics._MB).toFixed(0) + " MB"; - else return (v / QservCzarStatistics._GB).toFixed(0) + " GB"; - } /** * @param {Number} seconds diff --git a/src/www/qserv/js/QservMonitoringDashboard.js b/src/www/qserv/js/QservMonitoringDashboard.js index 8e3e3ff03d..f8de348587 100644 --- a/src/www/qserv/js/QservMonitoringDashboard.js +++ b/src/www/qserv/js/QservMonitoringDashboard.js @@ -56,6 +56,7 @@ require([ 'qserv/QservWorkerTasks', 'qserv/QservWorkerTaskHist', 'qserv/QservWorkerResultsFilesystem', + 'qserv/QservWorkerFiles', 'qserv/QservWorkerConfig', 'qserv/ReplicationController', 'qserv/ReplicationTools', @@ -101,6 +102,7 @@ function(CSSLoader, QservWorkerTasks, QservWorkerTaskHist, QservWorkerResultsFilesystem, + QservWorkerFiles, QservWorkerConfig, ReplicationController, ReplicationTools, @@ -175,6 +177,7 @@ function(CSSLoader, new QservWorkerTasks('Tasks'), new QservWorkerTaskHist('Task Histograms'), new QservWorkerResultsFilesystem('Results Filesystem'), + new QservWorkerFiles('Files'), new QservWorkerConfig('Config') ] }, diff --git a/src/www/qserv/js/QservWorkerFiles.js b/src/www/qserv/js/QservWorkerFiles.js new file mode 100644 index 0000000000..bf8825fcd3 --- /dev/null +++ b/src/www/qserv/js/QservWorkerFiles.js @@ -0,0 +1,302 @@ +define([ + 'webfwk/CSSLoader', + 'webfwk/Fwk', + 'webfwk/FwkApplication', + 'qserv/Common', + 'underscore'], + +function(CSSLoader, + Fwk, + FwkApplication, + Common, + _) { + + CSSLoader.load('qserv/css/QservWorkerFiles.css'); + + class QservWorkerFiles extends FwkApplication { + + constructor(name) { + super(name); + } + fwk_app_on_show() { + console.log('show: ' + this.fwk_app_name); + this.fwk_app_on_update(); + } + fwk_app_on_hide() { + console.log('hide: ' + this.fwk_app_name); + } + fwk_app_on_update() { + if (this.fwk_app_visible) { + this._init(); + if (this._prev_update_sec === undefined) { + this._prev_update_sec = 0; + } + let now_sec = Fwk.now().sec; + if (now_sec - this._prev_update_sec > this._update_interval_sec()) { + this._prev_update_sec = now_sec; + this._init(); + this._load(); + } + } + } + set_worker(worker) { + this._init(); + this._load(worker); + } + _init() { + if (this._initialized === undefined) this._initialized = false; + if (this._initialized) return; + this._initialized = true; + let html = ` +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ ${Common.html_update_ival('update-interval', 10)} +
+
+ + +
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + +
QIDchunkjobattemptfilenamesizes-1created→modified→inspected
Loading...
+
+
`; + let cont = this.fwk_app_container.html(html); + cont.find(".form-control-selector").change(() => { + this._load(); + }); + cont.find("button#reset-files-form").click(() => { + this._set_query(''); + this._set_max_files(200); + this._set_update_interval_sec(10); + this._load(); + }); + } + _form_control(elem_type, id) { + if (this._form_control_obj === undefined) this._form_control_obj = {}; + if (!_.has(this._form_control_obj, id)) { + this._form_control_obj[id] = this.fwk_app_container.find(elem_type + '#' + id); + } + return this._form_control_obj[id]; + } + _update_interval_sec() { return this._form_control('select', 'update-interval').val(); } + _set_update_interval_sec(val) { this._form_control('select', 'update-interval').val(val); } + _query() { return this._form_control('select', 'query').val(); } + _set_query(val) { this._form_control('select', 'query').val(val); } + _set_queries(queries) { + const prev_query = this._query(); + console.log("prev_query", prev_query, "queries", queries); + let html = ''; + for (let i in queries) { + const query = queries[i]; + const selected = (!_.isEmpty(prev_query) && (prev_query == query)); + html += ` +`; + } + this._form_control('select', 'query').html(html); + } + _max_files() { return this._form_control('select', 'max-files').val(); } + _set_max_files(val) { this._form_control('select', 'max-files').val(val); } + _set_num_files(total, selected, displayed) { this._form_control('input', 'num-files').val(displayed + ' / ' + selected + ' / ' + total); } + _worker() { return this._form_control('select', 'worker').val(); } + _set_worker(val) { this._form_control('select', 'worker').val(val); } + _set_workers(workers) { + const prev_worker = this._worker(); + let html = ''; + for (let i in workers) { + const worker = workers[i]; + const selected = (_.isEmpty(prev_worker) && (i === 0)) || + (!_.isEmpty(prev_worker) && (prev_worker === worker)); + html += ` + `; + } + this._form_control('select', 'worker').html(html); + } + + /** + * Table for displaying files that exist at the worker. + */ + _table() { + if (this._table_obj === undefined) { + this._table_obj = this.fwk_app_container.find('table#fwk-qserv-files'); + } + return this._table_obj; + } + _load(worker = undefined) { + if (this._loading === undefined) this._loading = false; + if (this._loading) return; + this._loading = true; + this._table().children('caption').addClass('updating'); + Fwk.web_service_GET( + "/replication/config", + {version: Common.RestAPIVersion}, + (data) => { + let workers = []; + for (let i in data.config.workers) { + workers.push(data.config.workers[i].name); + } + this._set_workers(workers); + if (!_.isUndefined(worker)) this._set_worker(worker); + this._load_files(); + }, + (msg) => { + console.log('request failed', this.fwk_app_name, msg); + this._table().children('caption').html('No Response'); + this._table().children('caption').removeClass('updating'); + this._loading = false; + } + ); + } + _load_files() { + Fwk.web_service_GET( + "/replication/qserv/worker/files/" + this._worker(), + { timeout_sec: 2, version: Common.RestAPIVersion, + query_ids: this._query(), + max_files: this._max_files() + }, + (data) => { + if (data.success) { + this._display(data.status); + Fwk.setLastUpdate(this._table().children('caption')); + } else { + console.log('request failed', this.fwk_app_name, data.error); + this._table().children('caption').html('' + data.error + ''); + } + this._table().children('caption').removeClass('updating'); + this._loading = false; + }, + (msg) => { + console.log('request failed', this.fwk_app_name, msg); + this._table().children('caption').html('No Response'); + this._table().children('caption').removeClass('updating'); + this._loading = false; + } + ); + } + _display(status) { + // Update a collection of queries in the selector. + const query_ids = _.uniq(_.map(status.files, function(file) { return file.task.query_id; })); + query_ids.sort(); + this._set_queries(query_ids); + // Turn the collection of files into a dictionary + const files = _.groupBy(status.files, function(file) { return file.task.query_id; }); + console.log(files); + const queryInspectTitle = "Click to see detailed info (progress, messages, etc.) on the query."; + + // Display files + let html = ''; + for (let queryId in files) { + const queryFiles = _.sortBy(files[queryId], function(file) { return file.task.chunk_id; }); + console.log(queryFiles); + let rowspan = 1; + let htmlFiles = ''; + for (let i in queryFiles) { + const file = queryFiles[i]; + const createTime_msec = file.ctime * 1000; + const modifyTime_msec = file.mtime * 1000; + const snapshotTime_msec = file.current_time_ms; + htmlFiles += ` + + 
${file.task.chunk_id}
+ 
${file.task.job_id}
+ 
${file.task.attemptcount}
+ 
${file.filename}
+ 
${file.size}
+ 
${QservWorkerFiles._io_performance(file.size, file.ctime, file.mtime)}
+ 
${(new Date(createTime_msec)).toISOString()}
+ 
${QservWorkerFiles._timestamps_diff2str(createTime_msec, modifyTime_msec)}
+ 
${QservWorkerFiles._timestamp2hhmmss(modifyTime_msec)}
+ 
${QservWorkerFiles._timestamps_diff2str(modifyTime_msec, snapshotTime_msec)}
+ 
${QservWorkerFiles._timestamp2hhmmss(snapshotTime_msec)}
+`; + rowspan++; + } + html += ` + + 
${queryId}
+ + + +` + htmlFiles; + + } + let tbody = this._table().children('tbody').html(html); + let displayQuery = function(e) { + let button = $(e.currentTarget); + let queryId = button.parent().parent().attr("id"); + Fwk.find("Status", "Query Inspector").set_query_id(queryId); + Fwk.show("Status", "Query Inspector"); + }; + tbody.find("button.inspect-query").click(displayQuery); + this._set_num_files(status.num_total, status.num_selected, status.files.length); + } + static _timestamps_diff2str(begin, end, snapshot) { + return ((end - begin) / 1000.0).toFixed(1); + } + static _timestamp2hhmmss(ts) { + return (new Date(ts)).toISOString().substring(11, 19); + } + static _io_performance(size, begin_time_sec, end_time_sec) { + let bytes_per_sec = size; + let interval = end_time_sec - begin_time_sec; + if (interval > 0) bytes_per_sec = size / interval; + return Common.format_data_rate(bytes_per_sec); + } + } + return QservWorkerFiles; +}); diff --git a/src/www/qserv/js/QservWorkerResultsFilesystem.js b/src/www/qserv/js/QservWorkerResultsFilesystem.js index 560feb8e23..712b422a56 100644 --- a/src/www/qserv/js/QservWorkerResultsFilesystem.js +++ b/src/www/qserv/js/QservWorkerResultsFilesystem.js @@ -139,6 +139,7 @@ function(CSSLoader, * Display MySQL connections */ _display(data) { + const workerFilesInspectTitle = "Click to see files existing on the worker."; let html = ''; for (let worker in data) { if (!data[worker].success) { @@ -160,7 +161,7 @@ function(CSSLoader, (100.0 * (filesystem.capacity_bytes - filesystem.available_bytes) / filesystem.capacity_bytes).toFixed(1) : -1; html += ` - + ${worker} ${filesystem.protocol} ${filesystem.folder} @@ -173,7 +174,13 @@ function(CSSLoader, `; } } - this._table().children('tbody').html(html); + let tbody = this._table().children('tbody').html(html); + let displayWorkerFiles = function(e) { + const worker = $(e.currentTarget).attr("worker"); + Fwk.find("Workers", "Files").set_worker(worker); + Fwk.show("Workers", "Files"); + }; + tbody.find("tr.display-worker-files").click(displayWorkerFiles); } static _GiB = 1024 * 1024 * 1024; static _bytes2gb(bytes) { diff --git a/src/xrdsvc/HttpMonitorModule.cc b/src/xrdsvc/HttpMonitorModule.cc index 65be798124..bbd4c65667 100644 --- a/src/xrdsvc/HttpMonitorModule.cc +++ b/src/xrdsvc/HttpMonitorModule.cc @@ -30,6 +30,7 @@ #include "http/Exceptions.h" #include "http/RequestQuery.h" #include "mysql/MySqlUtils.h" +#include "util/String.h" #include "wbase/FileChannelShared.h" #include "wbase/TaskState.h" #include "wconfig/WorkerConfig.h" @@ -66,6 +67,8 @@ json HttpMonitorModule::executeImpl(string const& subModuleName) { return _mysql(); else if (subModuleName == "STATUS") return _status(); + else if (subModuleName == "FILES") + return _files(); else if (subModuleName == "ECHO") return _echo(); throw invalid_argument(context() + func + " unsupported sub-module"); @@ -114,6 +117,16 @@ json HttpMonitorModule::_status() { return result; } +json HttpMonitorModule::_files() { + debug(__func__); + checkApiVersion(__func__, 28); + auto const queryIds = query().optionalVectorUInt64("query_ids"); + auto const maxFiles = query().optionalUInt("max_files", 0); + debug(__func__, "query_ids=" + util::String::toString(queryIds)); + debug(__func__, "max_files=" + to_string(maxFiles)); + return wbase::FileChannelShared::filesToJson(queryIds, maxFiles); +} + json HttpMonitorModule::_echo() { debug(__func__); checkApiVersion(__func__, 27); diff --git a/src/xrdsvc/HttpMonitorModule.h b/src/xrdsvc/HttpMonitorModule.h index f64c42dee5..01c5c171c5 100644 --- a/src/xrdsvc/HttpMonitorModule.h +++ b/src/xrdsvc/HttpMonitorModule.h @@ -54,6 +54,7 @@ class HttpMonitorModule : public xrdsvc::HttpModule { * 'CONFIG' - get configuration parameters * 'MYSQL' - get the status (running queries) of the worker's MySQL service * 'STATUS' - get the status info (tasks, schedulers, etc.) + * 'FILES' - get info on the partial result files * 'ECHO' - send back the received data * * @throws std::invalid_argument for unknown values of parameter 'subModuleName' @@ -86,6 +87,9 @@ class HttpMonitorModule : public xrdsvc::HttpModule { /// @return The worker status info (tasks, schedulers, etc.). nlohmann::json _status(); + /// @return An info on the partial result files. + nlohmann::json _files(); + /// @return Send back the received data. nlohmann::json _echo(); }; diff --git a/src/xrdsvc/HttpReplicaMgtModule.cc b/src/xrdsvc/HttpReplicaMgtModule.cc index dcfff6ba35..5ffcb03f72 100644 --- a/src/xrdsvc/HttpReplicaMgtModule.cc +++ b/src/xrdsvc/HttpReplicaMgtModule.cc @@ -35,7 +35,7 @@ #include "http/RequestBody.h" #include "http/RequestQuery.h" #include "mysql/MySqlUtils.h" -#include "util/IterableFormatter.h" +#include "util/String.h" #include "wconfig/WorkerConfig.h" #include "wcontrol/Foreman.h" #include "wcontrol/ResourceMonitor.h" @@ -57,12 +57,6 @@ json const extErrorReplicaInUse = json::object({{"in_use", 1}}); string makeResource(string const& database, int chunk) { return "/chk/" + database + "/" + to_string(chunk); } -string vec2str(vector const& v) { - ostringstream ss; - ss << lsst::qserv::util::printable(v, "", "", ","); - return ss.str(); -} - } // namespace namespace lsst::qserv::xrdsvc { @@ -108,7 +102,7 @@ json HttpReplicaMgtModule::_getReplicas() { bool const inUseOnly = query().optionalUInt("in_use_only", 0) != 0; vector const databases = query().requiredVectorStr("databases"); debug(__func__, "in_use_only: " + string(inUseOnly ? "1" : "0")); - debug(__func__, "databases: " + ::vec2str(databases)); + debug(__func__, "databases: " + util::String::toString(databases)); set databaseFilter; for (string const& database : databases) { databaseFilter.insert(database); @@ -123,7 +117,7 @@ json HttpReplicaMgtModule::_setReplicas() { bool const force = body().optional("force", 0) != 0; vector const databases = body().requiredColl("databases"); debug(__func__, "force: " + string(force ? "1" : "0")); - debug(__func__, "databases: " + ::vec2str(databases)); + debug(__func__, "databases: " + util::String::toString(databases)); set databaseFilter; for (string const& database : databases) { databaseFilter.insert(database); @@ -308,7 +302,7 @@ void HttpReplicaMgtModule::_modifyReplica(string const& func, Direction directio bool const force = body().optional("force", 0) != 0; debug(func, "chunk: " + to_string(chunk)); - debug(func, "databases: " + ::vec2str(databases)); + debug(func, "databases: " + util::String::toString(databases)); debug(func, "force: " + string(force ? "1" : "0")); if (databases.empty()) { diff --git a/src/xrdsvc/HttpSvc.cc b/src/xrdsvc/HttpSvc.cc index 3fcd76b2d1..d38c0c8125 100644 --- a/src/xrdsvc/HttpSvc.cc +++ b/src/xrdsvc/HttpSvc.cc @@ -98,6 +98,11 @@ uint16_t HttpSvc::start() { [self](shared_ptr const& req, shared_ptr const& resp) { HttpMonitorModule::process(::serviceName, self->_foreman, req, resp, "STATUS"); }}}); + _httpServerPtr->addHandlers( + {{"GET", "/files", + [self](shared_ptr const& req, shared_ptr const& resp) { + HttpMonitorModule::process(::serviceName, self->_foreman, req, resp, "FILES"); + }}}); _httpServerPtr->addHandlers( {{"POST", "/echo", [self](shared_ptr const& req, shared_ptr const& resp) {