DRAFT: Refine views

cloud

@@ -315,6 +315,7 @@ function execute_command() {
         --database="$database"
         --clean-test-databases="$clean_test_databases"
         --empty-test-databases="$empty_test_databases"
+        --auto-sync="$test"
     )
 
     # Handle "env" command

kcidb/cloud/functions.sh

@@ -32,6 +32,7 @@ declare _FUNCTIONS_SH=
 #       --database=SPEC
 #       --clean-test-databases=SPEC_LIST
 #       --empty-test-databases=SPEC_LIST
+#       --auto-sync=true|false
 function functions_env() {
     declare params
     params="$(getopt_vars format \
@@ -57,6 +58,7 @@ function functions_env() {
                           database \
                           clean_test_databases \
                           empty_test_databases \
+                          auto_sync \
                           -- "$@")"
     eval "$params"
     declare -A env=(
@@ -107,6 +109,9 @@ function functions_env() {
     if "$updated_publish"; then
         env[KCIDB_UPDATED_PUBLISH]="1"
     fi
+    if "$auto_sync"; then
+        env[KCIDB_AUTO_SYNC]="1"
+    fi
     if [ "$format" == "yaml" ]; then
         # Silly Python and its significant whitespace
         sed -E 's/^[[:blank:]]+//' <<<'

kcidb/db/__init__.py

@@ -749,8 +749,13 @@ def load_main():
         help='Load metadata fields as well',
         action='store_true'
     )
+    parser.add_argument(
+        '--sync',
+        help='Propagate database changes immediately, if needed',
+        action='store_true'
+    )
     args = parser.parse_args()
-    client = Client(args.database)
+    client = Client(args.database, auto_sync=args.sync)
     if not client.is_initialized():
         raise Exception(f"Database {args.database!r} is not initialized")
     io_schema = client.get_schema()[1]
@@ -885,8 +890,13 @@ def empty_main():
     description = 'kcidb-db-empty - Remove all data from a ' \
         'Kernel CI report database'
     parser = ArgumentParser(description=description)
+    parser.add_argument(
+        '--sync',
+        help='Propagate database changes immediately, if needed',
+        action='store_true'
+    )
     args = parser.parse_args()
-    client = Client(args.database)
+    client = Client(args.database, auto_sync=args.sync)
     if client.is_initialized():
         client.empty()
     else:
@@ -909,8 +919,26 @@ def purge_main():
         "be *preserved* should've arrived. "
         "No data is removed if not specified."
     )
+    parser.add_argument(
+        '--sync',
+        help='Propagate database changes immediately, if needed',
+        action='store_true'
+    )
     args = parser.parse_args()
-    client = Client(args.database)
+    client = Client(args.database, auto_sync=args.sync)
     if not client.is_initialized():
         raise Exception(f"Database {args.database!r} is not initialized")
     return 0 if client.purge(before=args.before) else 2
+
+
+def sync_main():
+    """Execute the kcidb-db-sync command-line tool"""
+    sys.excepthook = kcidb.misc.log_and_print_excepthook
+    description = 'kcidb-db-sync - Propagate database changes. ' \
+        'Exit with status 2, if not needed/supported by database.'
+    parser = ArgumentParser(description=description)
+    args = parser.parse_args()
+    client = Client(args.database)
+    if not client.is_initialized():
+        raise Exception(f"Database {args.database!r} is not initialized")
+    return 0 if client.sync() else 2

kcidb/db/bigquery/v04_01.py

@@ -222,12 +222,21 @@ class Schema(PreviousSchema):
                  "        present,\n"
                  "        comment,\n"
                  "        misc,\n"
-                 "        DENSE_RANK() OVER (\n"
+                 "        RANK() OVER (\n"
                  "            PARTITION BY\n"
                  "                issue_id, build_id, test_id\n"
                  "            ORDER BY issue_version DESC\n"
                  "        ) AS precedence\n"
                  "    FROM incidents\n"
+                 "    WHERE\n"
+                 "        present IS NOT NULL AND\n"
+                 "        EXISTS (\n"
+                 "            SELECT TRUE\n"
+                 "            FROM issues\n"
+                 "            WHERE\n"
+                 "                incidents.issue_id = issues.id AND\n"
+                 "                incidents.issue_version = issues.version\n"
+                 "        )\n"
                  ")\n"
                  "WHERE precedence = 1",
     )

kcidb/db/postgresql/v04_09.py

@@ -27,7 +27,7 @@ class Schema(PreviousSchema):
 
     # A map of view names and schemas
     VIEWS = merge_dicts(PreviousSchema.VIEWS, dict(
-        current_bugs=View("""\
+        current_bugs=View(textwrap.dedent("""\
             SELECT
                 MAX(_timestamp) AS _timestamp,
                 report_url AS url,
@@ -47,8 +47,8 @@ class Schema(PreviousSchema):
                 ORDER BY id, version DESC
             ) AS current_issues
             GROUP BY report_url
-        """, refresh_period=5),
-        current_issues=View("""\
+        """), refresh_period=5),
+        current_issues=View(textwrap.dedent("""\
             SELECT DISTINCT ON (id)
                 _timestamp,
                 id,
@@ -65,26 +65,51 @@ class Schema(PreviousSchema):
                 misc
             FROM issues
             ORDER BY id, version DESC
-        """, refresh_period=5),
-        current_incidents=View("""\
-            SELECT DISTINCT ON (incidents.issue_id)
-                incidents._timestamp,
-                incidents.id,
-                incidents.origin,
-                incidents.issue_id,
-                incidents.issue_version,
-                incidents.build_id,
-                incidents.test_id,
-                incidents.present,
-                incidents.comment,
-                incidents.misc
-            FROM incidents
-            INNER JOIN issues ON
-                incidents.issue_id = issues.id AND
-                incidents.issue_version = issues.version
-            WHERE present IS NOT NULL
-            ORDER BY incidents.issue_id, incidents.issue_version DESC
-        """, refresh_period=5),
+        """), refresh_period=5),
+        current_incidents=View(textwrap.dedent("""\
+            SELECT
+                _timestamp,
+                id,
+                origin,
+                issue_id,
+                issue_version,
+                build_id,
+                test_id,
+                present,
+                comment,
+                misc
+            FROM (
+                SELECT
+                    _timestamp,
+                    id,
+                    origin,
+                    issue_id,
+                    issue_version,
+                    build_id,
+                    test_id,
+                    present,
+                    comment,
+                    misc,
+                    RANK() OVER (
+                        PARTITION BY
+                            issue_id,
+                            build_id,
+                            test_id
+                        ORDER BY issue_version DESC
+                    ) AS precedence
+                FROM incidents
+                WHERE
+                    present IS NOT NULL AND
+                    EXISTS (
+                        SELECT TRUE
+                        FROM issues
+                        WHERE
+                            incidents.issue_id = issues.id AND
+                            incidents.issue_version = issues.version
+                    )
+            ) AS prioritized_known_incidents
+            WHERE prioritized_known_incidents.precedence = 1
+        """), refresh_period=5),
     ))
 
     # A map of index names and schemas

kcidb/db/sqlite/v04_01.py

@@ -150,12 +150,22 @@ class Schema(PreviousSchema):
                       "        present,\n"
                       "        comment,\n"
                       "        misc,\n"
-                      "        DENSE_RANK() OVER (\n"
+                      "        RANK() OVER (\n"
                       "            PARTITION BY\n"
                       "                issue_id, build_id, test_id\n"
                       "            ORDER BY issue_version DESC\n"
                       "        ) AS precedence\n"
                       "    FROM incidents\n"
+                      "    WHERE\n"
+                      "        present IS NOT NULL AND\n"
+                      "        EXISTS (\n"
+                      "            SELECT TRUE\n"
+                      "            FROM issues\n"
+                      "            WHERE\n"
+                      "                incidents.issue_id = issues.id AND\n"
+                      "                incidents.issue_version = "
+                      "issues.version\n"
+                      "        )\n"
                       ")\n"
                       "WHERE precedence = 1",
         ),

kcidb/test_db.py

@@ -84,7 +84,8 @@ def test_empty_main():
         with patch("kcidb.db.Client", return_value=client) as \
                 Client:
             status = function()
-        Client.assert_called_once_with("bigquery:project.dataset")
+        Client.assert_called_once_with("bigquery:project.dataset",
+                                       auto_sync=False)
         client.empty.assert_called_once()
         return status
     """)
@@ -207,7 +208,8 @@ def test_load_main():
         client.load = Mock()
         with patch("kcidb.db.Client", return_value=client) as Client:
             status = function()
-        Client.assert_called_once_with("bigquery:project.dataset")
+        Client.assert_called_once_with("bigquery:project.dataset",
+                                       auto_sync=False)
         client.load.assert_called_once_with({repr(empty)},
                                             with_metadata=False)
         return status
@@ -223,7 +225,8 @@ def test_load_main():
         client.load = Mock()
         with patch("kcidb.db.Client", return_value=client) as Client:
             status = function()
-        Client.assert_called_once_with("bigquery:project.dataset")
+        Client.assert_called_once_with("bigquery:project.dataset",
+                                       auto_sync=False)
         assert client.load.call_count == 2
         client.load.assert_has_calls([
             call({repr(empty)}, with_metadata=False),

main.py

@@ -102,6 +102,8 @@
 _UPDATED_QUEUE_PUBLISHER = None
 # KCIDB cache storage bucket name
 CACHE_BUCKET_NAME = os.environ.get("KCIDB_CACHE_BUCKET_NAME")
+# True if the database changes should be synchronized immediately
+AUTO_SYNC = bool(os.environ.get("KCIDB_AUTO_SYNC", ""))
 
 
 def get_smtp_publisher():
@@ -190,7 +192,7 @@ def get_db_client(database):
         # Get the credentials
         get_db_credentials()
         # Create the client
-        _DB_CLIENTS[database] = kcidb.db.Client(database)
+        _DB_CLIENTS[database] = kcidb.db.Client(database, auto_sync=AUTO_SYNC)
     return _DB_CLIENTS[database]
 
 

setup.py

@@ -89,6 +89,7 @@
             "kcidb-db-load = kcidb.db:load_main",
             "kcidb-db-dump = kcidb.db:dump_main",
             "kcidb-db-query = kcidb.db:query_main",
+            "kcidb-db-sync = kcidb.db:sync_main",
             "kcidb-mq-email-publisher = kcidb.mq:email_publisher_main",
             "kcidb-mq-email-subscriber = kcidb.mq:email_subscriber_main",
             "kcidb-mq-io-publisher = kcidb.mq:io_publisher_main",