Merge branch 'tickets/DM-40126'

src/admin/python/lsst/qserv/admin/replicationInterface.py

@@ -201,7 +201,7 @@ def __init__(
         self.repl_ctrl = urlparse(repl_ctrl_uri)
         self.auth_key = auth_key
         self.admin_auth_key = admin_auth_key
-        self.repl_api_version = 22
+        self.repl_api_version = 23
         _log.debug(f"ReplicationInterface %s", self.repl_ctrl)
 
     def version(self) -> str:

src/qmeta/QMetaMysql.cc

@@ -47,7 +47,7 @@ using namespace std;
 namespace {
 
 // Current version of QMeta schema
-char const VERSION_STR[] = "8";
+char const VERSION_STR[] = "9";
 
 LOG_LOGGER _log = LOG_GET("lsst.qserv.qmeta.QMetaMysql");
 

src/qmeta/schema/migrate-8-to-9.sql

@@ -0,0 +1,2 @@
+-- This index had significant performance implications for Qserv. 
+DROP INDEX IF EXISTS `QInfo_query_index` ON `QInfo`;

src/qmeta/schema/migrate-None-to-8.sql.jinja → src/qmeta/schema/migrate-None-to-9.sql.jinja

@@ -59,7 +59,6 @@ CREATE TABLE IF NOT EXISTS `QInfo` (
   KEY `QInfo_completed_index` (`completed`),
   KEY `QInfo_returned_index` (`returned`),
   KEY `QInfo_qtype_index` (`qType`),
-  FULLTEXT KEY `QInfo_query_index` (`query`),
   CONSTRAINT `QInfo_cid`
     FOREIGN KEY (`czarId`)
     REFERENCES `QCzar` (`czarId`)
@@ -219,4 +218,5 @@ COMMENT = 'Table of messages generated during queries.';
 -- Version 6 added indexes to optimize queries made by the Qserv Web Dashboard.
 -- Version 7 added final row count to QInfo.
 -- Version 8 replaced INT with BIGINT in the byte and row counter columns of QInfo.
-INSERT INTO `QMetadata` (`metakey`, `value`) VALUES ('version', '8');
+-- Version 9 removed the full-text index on the query text from QInfo.
+INSERT INTO `QMetadata` (`metakey`, `value`) VALUES ('version', '9');

src/replica/DatabaseMySQLGenerator.h

@@ -450,6 +450,11 @@ class QueryGenerator {
         return op2(id(col), val(expr), " REGEXP ");
     }
 
+    /// @return "<quoted-col> LIKE <escaped-quoted-expr>"
+    std::string like(std::string const& col, std::string const& expr) const {
+        return op2(id(col), val(expr), " LIKE ");
+    }
+
     /**
      * Pack pairs of column/variable names and their new values into a string which can be
      * further used to form SQL statements of the following kind:

src/replica/HttpMetaModule.cc

@@ -34,7 +34,7 @@ using json = nlohmann::json;
 
 namespace lsst::qserv::replica {
 
-unsigned int const HttpMetaModule::version = 22;
+unsigned int const HttpMetaModule::version = 23;
 
 void HttpMetaModule::process(ServiceProvider::Ptr const& serviceProvider, string const& context,
                              qhttp::Request::Ptr const& req, qhttp::Response::Ptr const& resp,

src/replica/HttpQservMonitorModule.cc

@@ -281,7 +281,7 @@ json HttpQservMonitorModule::_schedulers2chunks2json(map<string, set<int>> const
 
 json HttpQservMonitorModule::_userQueries() {
     debug(__func__);
-    checkApiVersion(__func__, 12);
+    checkApiVersion(__func__, 23);
 
     auto const config = controller()->serviceProvider()->config();
 
@@ -292,7 +292,7 @@ json HttpQservMonitorModule::_userQueries() {
     unsigned int const timeoutSec = query().optionalUInt("timeout_sec", workerResponseTimeoutSec());
     unsigned int const limit4past = query().optionalUInt("limit4past", 1);
     string const searchPattern = query().optionalString("search_pattern", string());
-    bool const searchBooleanMode = query().optionalUInt("search_boolean_mode", 0) != 0;
+    bool const searchRegexpMode = query().optionalUInt("search_regexp_mode", 0) != 0;
     bool const includeMessages = query().optionalUInt("include_messages", 0) != 0;
 
     debug(__func__, "query_status=" + queryStatus);
@@ -302,7 +302,7 @@ json HttpQservMonitorModule::_userQueries() {
     debug(__func__, "timeout_sec=" + to_string(timeoutSec));
     debug(__func__, "limit4past=" + to_string(limit4past));
     debug(__func__, "search_pattern=" + searchPattern);
-    debug(__func__, "search_boolean_mode=" + bool2str(searchBooleanMode));
+    debug(__func__, "search_regexp_mode=" + bool2str(searchRegexpMode));
     debug(__func__, "include_messages=" + bool2str(includeMessages));
 
     // Check which queries and in which schedulers are being executed
@@ -360,8 +360,11 @@ json HttpQservMonitorModule::_userQueries() {
         g.packCond(constraints, cond);
     }
     if (!searchPattern.empty()) {
-        string const mode = searchBooleanMode ? "BOOLEAN" : "NATURAL LANGUAGE";
-        g.packCond(constraints, g.matchAgainst("query", searchPattern, mode));
+        if (searchRegexpMode) {
+            g.packCond(constraints, g.regexp("query", searchPattern));
+        } else {
+            g.packCond(constraints, g.like("query", "%" + searchPattern + "%"));
+        }
     }
     h.conn->executeInOwnTransaction([&](auto conn) {
         result["queries_past"] = _pastUserQueries(conn, constraints, limit4past, includeMessages);

src/replica/testQueryGenerator.cc

@@ -225,6 +225,7 @@ BOOST_AUTO_TEST_CASE(QueryGeneratorTest) {
             {"NOW()>1234567890", g.gt(Sql::NOW, 1234567890)},
             {"`col`>=123", g.geq("col", 123)},
             {"`col` REGEXP '[0-9]+'", g.regexp("col", "[0-9]+")},
+            {"`col` LIKE '%abc%'", g.like("col", "%abc%")},
             {"NOW()<=UNIX_TIMESTAMP(`time`)", g.op2(Sql::NOW, g.UNIX_TIMESTAMP("time"), "<=")},
             {"NOW()=`time`", g.op2(Sql::NOW, g.id("time"), "=")},
 

src/www/dashboard.html

@@ -4,7 +4,7 @@
 <title>Qserv monitoring dashboard</title>
 <meta charset="UTF-8">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
-<script data-main="qserv/js/QservMonitoringDashboard.js?bust=66" src="https://requirejs.org/docs/release/2.3.6/minified/require.js"></script>
+<script data-main="qserv/js/QservMonitoringDashboard.js?bust=67" src="https://requirejs.org/docs/release/2.3.6/minified/require.js"></script>
 </head>
 <body></body>
 </html>

src/www/qserv/js/Common.js

@@ -3,7 +3,7 @@ define([
 
 function(sqlFormatter) {
     class Common {
-        static RestAPIVersion = 22;
+        static RestAPIVersion = 23;
         static query2text(query, expanded) {
             if (expanded) {
                 return sqlFormatter.format(query, Common._sqlFormatterConfig);

src/www/qserv/js/StatusUserQueries.js

@@ -154,8 +154,8 @@ function(CSSLoader,
       <div class="form-group col-md-1">
         <label for="query-search-mode">Search mode:</label>
         <select id="query-search-mode" class="form-control form-control-selector">
-          <option value="NATURAL" selected>NATURAL</option>
-          <option value="BOOLEAN">BOOLEAN</option>
+          <option value="LIKE" selected>LIKE</option>
+          <option value="REGEXP">REGEXP</option>
         </select>
       </div>
       <div class="form-group col-md-1">
@@ -224,7 +224,7 @@ function(CSSLoader,
                 this._set_min_elapsed("0");
                 this._set_query_type("");
                 this._set_query_search_pattern("");
-                this._set_query_search_mode("NATURAL");
+                this._set_query_search_mode("LIKE");
                 this._set_max_queries("200");
                 this._load();
             });
@@ -307,7 +307,7 @@ function(CSSLoader,
                     min_elapsed_sec: this._get_min_elapsed(),
                     query_type: this._get_query_type(),
                     search_pattern: this._get_query_search_pattern(),
-                    search_boolean_mode: this._get_query_search_mode() == "BOOLEAN" ? 1 : 0,
+                    search_regexp_mode: this._get_query_search_mode() == "REGEXP" ? 1 : 0,
                     limit4past: this._get_max_queries(),
                     timeout_sec: StatusUserQueries._server_proc_timeout_sec()
                 },