DM-41535: Extended display for the status of partial result files at Qserv workers

src/admin/python/lsst/qserv/admin/replicationInterface.py

@@ -201,7 +201,7 @@ def __init__(
         self.repl_ctrl = urlparse(repl_ctrl_uri)
         self.auth_key = auth_key
         self.admin_auth_key = admin_auth_key
-        self.repl_api_version = 27
+        self.repl_api_version = 28
         _log.debug(f"ReplicationInterface %s", self.repl_ctrl)
 
     def version(self) -> str:

src/ccontrol/UserQueryQservManager.cc

@@ -41,7 +41,7 @@
 #include "sql/SqlBulkInsert.h"
 #include "sql/SqlConnection.h"
 #include "sql/SqlConnectionFactory.h"
-#include "util/StringHelper.h"
+#include "util/String.h"
 #include "wconfig/WorkerConfig.h"
 
 using namespace std;
@@ -80,7 +80,7 @@ void UserQueryQservManager::submit() {
     if (_value.size() > 2) {
         string const space = " ";
         string const quotesRemoved = _value.substr(1, _value.size() - 2);
-        for (auto&& str : util::StringHelper::splitString(quotesRemoved, space)) {
+        for (auto&& str : util::String::split(quotesRemoved, space)) {
             // This is just in case if the splitter won't recognise consequtive spaces.
             if (str.empty() || (str == space)) continue;
             if (command.empty()) {

src/czar/Czar.cc

@@ -56,7 +56,7 @@
 #include "util/common.h"
 #include "util/FileMonitor.h"
 #include "util/IterableFormatter.h"
-#include "util/StringHelper.h"
+#include "util/String.h"
 #include "xrdreq/QueryManagementAction.h"
 #include "XrdSsi/XrdSsiProvider.hh"
 
@@ -124,9 +124,9 @@ Czar::Czar(string const& configPath, string const& czarName)
     int qPoolSize = _czarConfig->getQdispPoolSize();
     int maxPriority = std::max(0, _czarConfig->getQdispMaxPriority());
     string vectRunSizesStr = _czarConfig->getQdispVectRunSizes();
-    vector<int> vectRunSizes = util::StringHelper::getIntVectFromStr(vectRunSizesStr, ":", 1);
+    vector<int> vectRunSizes = util::String::parseToVectInt(vectRunSizesStr, ":", 1);
     string vectMinRunningSizesStr = _czarConfig->getQdispVectMinRunningSizes();
-    vector<int> vectMinRunningSizes = util::StringHelper::getIntVectFromStr(vectMinRunningSizesStr, ":", 0);
+    vector<int> vectMinRunningSizes = util::String::parseToVectInt(vectMinRunningSizesStr, ":", 0);
     LOGS(_log, LOG_LVL_INFO,
          "INFO qdisp config qPoolSize=" << qPoolSize << " maxPriority=" << maxPriority << " vectRunSizes="
                                         << vectRunSizesStr << " -> " << util::prettyCharList(vectRunSizes)

src/http/MetaModule.cc

@@ -37,7 +37,7 @@ string const adminAuthKey;
 
 namespace lsst::qserv::http {
 
-unsigned int const MetaModule::version = 27;
+unsigned int const MetaModule::version = 28;
 
 void MetaModule::process(string const& context, nlohmann::json const& info, qhttp::Request::Ptr const& req,
                          qhttp::Response::Ptr const& resp, string const& subModuleName) {

src/replica/CMakeLists.txt

@@ -76,6 +76,7 @@ target_sources(replica PRIVATE
     GetReplicasQservMgtRequest.cc
     GetDbStatusQservMgtRequest.cc
     GetConfigQservMgtRequest.cc
+    GetResultFilesQservMgtRequest.cc
     GetStatusQservMgtRequest.cc
     HealthMonitorTask.cc
     HttpAsyncReqApp.cc
@@ -297,7 +298,6 @@ replica_tests(
     testChunkLocker
     testChunkNumber
     testChunkedTable
-    testCommonStr
     testConnectionParams
     testCsv
     testFileIngestApp

src/replica/Common.cc

@@ -23,10 +23,7 @@
 #include "replica/Common.h"
 
 // System headers
-#include <algorithm>
 #include <stdexcept>
-#include <sstream>
-#include <type_traits>
 
 // Third party headers
 #include "boost/uuid/uuid.hpp"
@@ -350,19 +347,6 @@ DirectorIndexRequestParams::DirectorIndexRequestParams(ProtocolRequestDirectorIn
           hasTransactions(request.has_transactions()),
           transactionId(request.transaction_id()) {}
 
-vector<string> strsplit(string const& str, char delimiter) {
-    vector<string> words;
-    if (!str.empty()) {
-        string word;
-        istringstream ss(str);
-        while (std::getline(ss, word, delimiter)) {
-            remove(word.begin(), word.end(), delimiter);
-            if (!word.empty()) words.push_back(word);
-        }
-    }
-    return words;
-}
-
 string tableNameBuilder(string const& databaseName, string const& tableName, string const& suffix) {
     size_t const tableNameLimit = 64;
     string const name = databaseName + "__" + tableName + suffix;

src/replica/Common.h

@@ -351,17 +351,6 @@ class Query {
     std::string mutexName;
 };
 
-/**
- * @brief Split the input string into words separated by the delimiter.
- *
- * @param str The input string.
- * @param delimiter The delimiter character.
- * @return std::vector<std::string> A collection of words found in the string. The words
- *   are guaranteed not to have delimiters. The collection is guaranteed not to have
- *   empty strings.
- */
-std::vector<std::string> strsplit(std::string const& str, char delimiter = ' ');
-
 /**
  * @brief Generate the name of a metadata table at czar for the specified data table.
  * @param databaseName The name of a database where the data table is residing.

src/replica/CreateReplicaJob.cc

@@ -33,7 +33,7 @@
 #include "replica/QservMgtServices.h"
 #include "replica/ServiceProvider.h"
 #include "replica/StopRequest.h"
-#include "util/IterableFormatter.h"
+#include "util/String.h"
 
 // LSST headers
 #include "lsst/log/Log.h"
@@ -312,7 +312,7 @@ void CreateReplicaJob::_qservAddReplica(replica::Lock const& lock, unsigned int
                                         AddReplicaQservMgtRequest::CallbackType const& onFinish) {
     LOGS(_log, LOG_LVL_DEBUG,
          context() << __func__ << "  ** START ** Qserv notification on ADD replica:"
-                   << ", chunk=" << chunk << ", databases=" << util::printable(databases)
+                   << ", chunk=" << chunk << ", databases=" << util::String::toString(databases)
                    << "  worker=" << worker);
 
     auto self = shared_from_this();
@@ -322,8 +322,9 @@ void CreateReplicaJob::_qservAddReplica(replica::Lock const& lock, unsigned int
                 LOGS(_log, LOG_LVL_DEBUG,
                      self->context() << __func__ << "  ** FINISH ** Qserv notification on ADD replica:"
                                      << "  chunk=" << request->chunk()
-                                     << ", databases=" << util::printable(request->databases()) << ", worker="
-                                     << request->worker() << ", state=" << request->state2string());
+                                     << ", databases=" << util::String::toString(request->databases())
+                                     << ", worker=" << request->worker()
+                                     << ", state=" << request->state2string());
                 if (onFinish) onFinish(request);
             },
             id());

src/replica/DeleteReplicaJob.cc

@@ -33,7 +33,7 @@
 #include "replica/QservMgtServices.h"
 #include "replica/ServiceProvider.h"
 #include "replica/StopRequest.h"
-#include "util/IterableFormatter.h"
+#include "util/String.h"
 
 // LSST headers
 #include "lsst/log/Log.h"
@@ -293,7 +293,7 @@ void DeleteReplicaJob::_qservRemoveReplica(replica::Lock const& lock, unsigned i
                                            RemoveReplicaQservMgtRequest::CallbackType const& onFinish) {
     LOGS(_log, LOG_LVL_DEBUG,
          context() << __func__ << "  ** START ** Qserv notification on REMOVE replica:"
-                   << "  chunk=" << chunk << ", databases=" << util::printable(databases)
+                   << "  chunk=" << chunk << ", databases=" << util::String::toString(databases)
                    << ", worker=" << worker << ", force=" << (force ? "true" : "false"));
 
     auto self = shared_from_this();
@@ -303,7 +303,7 @@ void DeleteReplicaJob::_qservRemoveReplica(replica::Lock const& lock, unsigned i
                 LOGS(_log, LOG_LVL_DEBUG,
                      self->context() << __func__ << "  ** FINISH ** Qserv notification on REMOVE replica:"
                                      << "  chunk=" << request->chunk()
-                                     << ", databases=" << util::printable(request->databases())
+                                     << ", databases=" << util::String::toString(request->databases())
                                      << ", worker=" << request->worker()
                                      << ", force=" << (request->force() ? "true" : "false")
                                      << ", state=" << request->state2string());

src/replica/GetResultFilesQservMgtRequest.cc

@@ -0,0 +1,71 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+
+// Class header
+#include "replica/GetResultFilesQservMgtRequest.h"
+
+// Qserv headers
+#include "util/String.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+using namespace std;
+
+namespace {
+
+LOG_LOGGER _log = LOG_GET("lsst.qserv.replica.GetResultFilesQservMgtRequest");
+
+}  // namespace
+
+namespace lsst::qserv::replica {
+
+shared_ptr<GetResultFilesQservMgtRequest> GetResultFilesQservMgtRequest::create(
+        shared_ptr<ServiceProvider> const& serviceProvider, string const& worker,
+        vector<QueryId> const& queryIds, unsigned int maxFiles,
+        GetResultFilesQservMgtRequest::CallbackType const& onFinish) {
+    return shared_ptr<GetResultFilesQservMgtRequest>(
+            new GetResultFilesQservMgtRequest(serviceProvider, worker, queryIds, maxFiles, onFinish));
+}
+
+GetResultFilesQservMgtRequest::GetResultFilesQservMgtRequest(
+        shared_ptr<ServiceProvider> const& serviceProvider, string const& worker,
+        vector<QueryId> const& queryIds, unsigned int maxFiles,
+        GetResultFilesQservMgtRequest::CallbackType const& onFinish)
+        : QservMgtRequest(serviceProvider, "QSERV_GET_RESULT_FILES", worker),
+          _queryIds(queryIds),
+          _maxFiles(maxFiles),
+          _onFinish(onFinish) {}
+
+void GetResultFilesQservMgtRequest::createHttpReqImpl(replica::Lock const& lock) {
+    string const service = "/files";
+    string query;
+    query += "?query_ids=" + util::String::toString(_queryIds);
+    query += "&max_files=" + to_string(_maxFiles);
+    createHttpReq(lock, service, query);
+}
+
+void GetResultFilesQservMgtRequest::notify(replica::Lock const& lock) {
+    LOGS(_log, LOG_LVL_TRACE, context() << __func__);
+    notifyDefaultImpl<GetResultFilesQservMgtRequest>(lock, _onFinish);
+}
+
+}  // namespace lsst::qserv::replica

src/replica/GetResultFilesQservMgtRequest.h

@@ -0,0 +1,98 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+#ifndef LSST_QSERV_REPLICA_GETRESULTFILESQSERVMGTREQUEST_H
+#define LSST_QSERV_REPLICA_GETRESULTFILESQSERVMGTREQUEST_H
+
+// System headers
+#include <memory>
+#include <string>
+#include <vector>
+
+// Qserv headers
+#include "global/intTypes.h"
+#include "replica/QservMgtRequest.h"
+
+namespace lsst::qserv::replica {
+class ServiceProvider;
+}  // namespace lsst::qserv::replica
+
+// This header declarations
+namespace lsst::qserv::replica {
+
+/**
+ * Class GetResultFilesQservMgtRequest is a request for obtaining info
+ * on the partial result files from the Qserv worker.
+ */
+class GetResultFilesQservMgtRequest : public QservMgtRequest {
+public:
+    typedef std::shared_ptr<GetResultFilesQservMgtRequest> Ptr;
+
+    /// The function type for notifications on the completion of the request
+    typedef std::function<void(Ptr)> CallbackType;
+
+    GetResultFilesQservMgtRequest() = delete;
+    GetResultFilesQservMgtRequest(GetResultFilesQservMgtRequest const&) = delete;
+    GetResultFilesQservMgtRequest& operator=(GetResultFilesQservMgtRequest const&) = delete;
+
+    virtual ~GetResultFilesQservMgtRequest() final = default;
+
+    /**
+     * Static factory method is needed to prevent issues with the lifespan
+     * and memory management of instances created otherwise (as values or via
+     * low-level pointers).
+     * @param serviceProvider A reference to a provider of services for accessing
+     *   Configuration, saving the request's persistent state to the database.
+     * @param worker The name of a worker to send the request to.
+     * @param queryIds The optional selector for queries. If empty then all queries will
+     *   be considered.
+     * @param maxFiles The optional limit for maximum number of files to be reported.
+     *   If 0 then no limit is set.
+     * @param onFinish (optional) callback function to be called upon request completion.
+     * @return A pointer to the created object.
+     */
+    static std::shared_ptr<GetResultFilesQservMgtRequest> create(
+            std::shared_ptr<ServiceProvider> const& serviceProvider, std::string const& worker,
+            std::vector<QueryId> const& queryIds = std::vector<QueryId>(), unsigned int maxFiles = 0,
+            CallbackType const& onFinish = nullptr);
+
+protected:
+    /// @see QservMgtRequest::createHttpReqImpl()
+    virtual void createHttpReqImpl(replica::Lock const& lock) final;
+
+    /// @see QservMgtRequest::notify()
+    virtual void notify(replica::Lock const& lock) final;
+
+private:
+    /// @see GetResultFilesQservMgtRequest::create()
+    GetResultFilesQservMgtRequest(std::shared_ptr<ServiceProvider> const& serviceProvider,
+                                  std::string const& worker, std::vector<QueryId> const& queryIds,
+                                  unsigned int maxFiles, CallbackType const& onFinish);
+
+    // Input parameters
+
+    std::vector<QueryId> const _queryIds;
+    unsigned int const _maxFiles;
+    CallbackType _onFinish;  ///< This callback is reset after finishing the request.
+};
+
+}  // namespace lsst::qserv::replica
+
+#endif  // LSST_QSERV_REPLICA_GETRESULTFILESQSERVMGTREQUEST_H

src/replica/GetStatusQservMgtRequest.cc

@@ -22,10 +22,8 @@
 // Class header
 #include "replica/GetStatusQservMgtRequest.h"
 
-// System headers
-#include <algorithm>
-#include <iterator>
-#include <sstream>
+// Qserv headers
+#include "util/String.h"
 
 // LSST headers
 #include "lsst/log/Log.h"
@@ -45,18 +43,10 @@ string taskSelectorToHttpQuery(wbase::TaskSelector const& taskSelector) {
     query += "?include_tasks=" + string(taskSelector.includeTasks ? "1" : "0");
     query += "&max_tasks=" + to_string(taskSelector.maxTasks);
     if (!taskSelector.queryIds.empty()) {
-        ostringstream ss;
-        copy(taskSelector.queryIds.begin(), taskSelector.queryIds.end() - 1,
-             ostream_iterator<qserv::QueryId>(ss, ","));
-        ss << taskSelector.queryIds.back();
-        query += "&query_ids=" + ss.str();
+        query += "&query_ids=" + util::String::toString(taskSelector.queryIds);
     }
     if (!taskSelector.taskStates.empty()) {
-        ostringstream ss;
-        copy(taskSelector.taskStates.begin(), taskSelector.taskStates.end() - 1,
-             ostream_iterator<wbase::TaskState>(ss, ","));
-        ss << taskSelector.taskStates.back();
-        query += "&task_states=" + ss.str();
+        query += "&task_states=" + util::String::toString(taskSelector.taskStates);
     }
     return query;
 }