Support v4.4 I/O schema with test enhancements

kcidb/db/bigquery/__init__.py

@@ -2,7 +2,7 @@
 
 import textwrap
 from kcidb.db.schematic import Driver as SchematicDriver
-from kcidb.db.bigquery.v04_02 import Schema as LatestSchema
+from kcidb.db.bigquery.v04_03 import Schema as LatestSchema
 
 
 class Driver(SchematicDriver):

kcidb/db/bigquery/v04_00.py

@@ -542,8 +542,12 @@ class Schema(AbstractSchema):
              "    origin,\n"
              "    path,\n"
              "    environment.comment AS environment_comment,\n"
+             "    NULL AS environment_compatible,\n"
              "    environment.misc AS environment_misc,\n"
              "    status,\n"
+             "    NULL AS number_value,\n"
+             "    NULL AS number_unit,\n"
+             "    NULL AS number_prefix,\n"
              "    waived,\n"
              "    start_time,\n"
              "    duration,\n"

kcidb/db/bigquery/v04_03.py

@@ -0,0 +1,117 @@
+"""Kernel CI report database - BigQuery schema v4.3"""
+
+import logging
+from google.cloud.bigquery.schema import SchemaField as Field
+from google.cloud.bigquery.table import Table
+import kcidb.io as io
+from kcidb.misc import merge_dicts
+from .v04_02 import Schema as PreviousSchema
+
+# Module's logger
+LOGGER = logging.getLogger(__name__)
+
+
+class Schema(PreviousSchema):
+    """BigQuery database schema v4.3"""
+
+    # The schema's version.
+    version = (4, 3)
+    # The I/O schema the database schema supports
+    io = io.schema.V4_4
+
+    # Test environment fields
+    ENVIRONMENT_FIELDS = PreviousSchema.ENVIRONMENT_FIELDS + (
+        Field(
+            "compatible", "STRING", mode="REPEATED",
+            description="The values from the root-level "
+                        "'compatible' property of the system's "
+                        "device tree, if any, in the same order"
+        ),
+    )
+
+    # Number fields
+    NUMBER_FIELDS = (
+        Field(
+            "value", "FLOAT64",
+            description="The floating-point output value",
+        ),
+        Field(
+            "unit", "STRING",
+            description="The (compound) unit symbol(s) the value "
+                        "is measured in",
+        ),
+        Field(
+            "prefix", "STRING",
+            description="The type of prefix to add to the "
+                        "unit, after the value is scaled. "
+                        "If unit is not specified, the prefix "
+                        "alone is used in its place. "
+                        "Either \"metric\" or \"binary\".",
+        ),
+    )
+
+    # A map of table names to their BigQuery schemas
+    TABLE_MAP = merge_dicts(
+        PreviousSchema.TABLE_MAP,
+        tests=list(filter(
+            lambda f: f.name != "environment",
+            PreviousSchema.TABLE_MAP["tests"]
+        )) + [
+            Field(
+                "environment", "RECORD", fields=ENVIRONMENT_FIELDS,
+                description="The environment the test ran in. "
+                            "E.g. a host, a set of hosts, or a lab; "
+                            "amount of memory/storage/CPUs, for each host; "
+                            "process environment variables, etc.",
+            ),
+            Field(
+                "number", "RECORD", fields=NUMBER_FIELDS,
+                description="The numerical output produced by the test",
+            ),
+        ]
+    )
+
+    # Queries for each type of raw object-oriented data
+    OO_QUERIES = merge_dicts(
+        PreviousSchema.OO_QUERIES,
+        test="SELECT\n"
+             "    id,\n"
+             "    build_id,\n"
+             "    origin,\n"
+             "    path,\n"
+             "    environment.comment AS environment_comment,\n"
+             "    environment.compatible AS environment_compatible,\n"
+             "    environment.misc AS environment_misc,\n"
+             "    status,\n"
+             "    number.value AS number_value,\n"
+             "    number.unit AS number_unit,\n"
+             "    number.prefix AS number_prefix,\n"
+             "    waived,\n"
+             "    start_time,\n"
+             "    duration,\n"
+             "    output_files,\n"
+             "    log_url,\n"
+             "    log_excerpt,\n"
+             "    comment,\n"
+             "    misc\n"
+             "FROM tests",
+    )
+
+    @classmethod
+    def _inherit(cls, conn):
+        """
+        Inerit the database data from the previous schema version (if any).
+
+        Args:
+            conn:   Connection to the database to inherit. The database must
+                    comply with the previous version of the schema.
+        """
+        assert isinstance(conn, cls.Connection)
+        # Add the new fields to the "tests" table.
+        tests_table = Table(conn.dataset_ref.table("_tests"))
+        tests_table.schema = cls.TABLE_MAP["tests"]
+        conn.client.update_table(tests_table, ["schema"])
+        # Update the view
+        tests_view = Table(conn.dataset_ref.table("tests"))
+        tests_view.view_query = cls._format_view_query(conn, "tests")
+        conn.client.update_table(tests_view, ["view_query"])

kcidb/db/postgresql/__init__.py

@@ -2,7 +2,7 @@
 
 import textwrap
 from kcidb.db.schematic import Driver as SchematicDriver
-from kcidb.db.postgresql.v04_07 import Schema as LatestSchema
+from kcidb.db.postgresql.v04_08 import Schema as LatestSchema
 
 
 class Driver(SchematicDriver):

kcidb/db/postgresql/schema.py

@@ -89,6 +89,32 @@ def __init__(self, constraint=None,
                          metadata_expr=metadata_expr)
 
 
+class TextArrayColumn(Column):
+    """A text array column schema"""
+
+    def __init__(self, constraint=None,
+                 conflict_func=None, metadata_expr=None):
+        """
+        Initialize the column schema.
+
+        Args:
+            constraint:     The column's constraint.
+                            A member of the Constraint enum, or None,
+                            meaning no constraint.
+            conflict_func:  The (non-empty) string containing the name of the
+                            SQL function to use to resolve insertion conflicts
+                            for this column. None to resolve
+                            non-deterministically.
+            metadata_expr:  A (non-empty) SQL expression string to use as the
+                            value for this (metadata) column, if not supplied
+                            explicitly. None to consider this a normal column.
+        """
+        assert constraint is None or isinstance(constraint, Constraint)
+        super().__init__("TEXT[]", constraint=constraint,
+                         conflict_func=conflict_func,
+                         metadata_expr=metadata_expr)
+
+
 class JSONColumn(Column):
     """A JSON column schema"""
 
@@ -230,7 +256,7 @@ def __init__(self, columns, primary_key=None):
 
 class Index(_SQLIndex):
     """An index schema"""
-    def __init__(self, table, columns):
+    def __init__(self, table, columns, method=None):
         """
         Initialize the index schema.
 
@@ -239,9 +265,30 @@ def __init__(self, table, columns):
             columns:    The list of names of table columns belonging to the
                         index. The names consist of dot-separated parts, same
                         as used for the Table creation parameters.
+            method:     The index method string, if different from default.
+                        None, if the default should be used.
         """
         assert isinstance(table, str)
         assert isinstance(columns, list)
         assert all(isinstance(c, str) for c in columns)
+        assert method is None or isinstance(method, str) and method
         # TODO: Switch to hardcoding "_" key_sep in base class
         super().__init__(table, columns, key_sep="_")
+        self.method = method
+
+    def format_create(self, name):
+        """
+        Format the "CREATE INDEX" command for the table.
+
+        Args:
+            name:       The name of the target index of the command.
+
+        Returns:
+            The formatted "CREATE INDEX" command.
+        """
+        method = "" if self.method is None else f" USING {self.method}"
+        return (
+            f"CREATE INDEX IF NOT EXISTS {name} ON {self.table}{method} (" +
+            ", ".join(self.columns.values()) +
+            ")"
+        )

kcidb/db/postgresql/v04_00.py

@@ -15,7 +15,7 @@
     Connection as AbstractConnection
 from kcidb.db.postgresql.schema import \
     Constraint, BoolColumn, FloatColumn, IntegerColumn, TimestampColumn, \
-    VarcharColumn, TextColumn, JSONColumn, Table
+    VarcharColumn, TextColumn, TextArrayColumn, JSONColumn, Table
 
 # Module's logger
 LOGGER = logging.getLogger(__name__)
@@ -381,10 +381,14 @@ class Schema(AbstractSchema):
                       "    origin,\n"
                       "    path,\n"
                       "    environment_comment,\n"
+                      "    NULL AS environment_compatible,\n"
                       "    environment_misc,\n"
                       "    log_url,\n"
                       "    log_excerpt,\n"
                       "    status,\n"
+                      "    NULL AS number_value,\n"
+                      "    NULL AS number_unit,\n"
+                      "    NULL AS number_prefix,\n"
                       "    waived,\n"
                       "    start_time,\n"
                       "    duration,\n"
@@ -398,10 +402,14 @@ class Schema(AbstractSchema):
                 origin=TextColumn(),
                 path=TextColumn(),
                 environment_comment=TextColumn(),
+                environment_compatible=TextArrayColumn(),
                 environment_misc=JSONColumn(),
                 log_url=TextColumn(),
                 log_excerpt=TextColumn(),
                 status=TextColumn(),
+                number_value=FloatColumn(),
+                number_unit=TextColumn(),
+                number_prefix=TextColumn(),
                 waived=BoolColumn(),
                 start_time=TimestampColumn(),
                 duration=FloatColumn(),