feat: job configuration enhancements: added description and metadata …

…fields and refactored hive source partition filtering (#28) - changes in Job Configuration model: - added description field for all DQ entities (where it was previously missed). - added metadata fields for all DQ entitities to allow definition of custom parameters related to specific entity. - added possibility to filter hive source partitions using SQL expression. Expression can only contain reference to a partition column, literals and sql functions. - refactored encryption configuration naming. - changes in results saving and targets processing: - added application and job configuration API versions info to job state. - added jobState type of result to Result Targets. - added SQL scripts to support migration to a new tables structure that includes both description and metadata columns. - added scripts for DB migration to change job_state table structure. - test updates: - refactored tests in accordance with new changes. - added tests for SQL expression parser used to validate partition filter expressions. - refactored semantic-release process: - added script to fix heading levels in changelog file during release. - changed release configuration to fix to run script above at the `prepare` stage after changelog was updated. - updated semantic-release script that changes application versions in related documents. - other minor fixes. - documentation update to reflect aforementioned changes.
Raiffeisen-DGTL · Mar 4, 2024 · 7cf24db · 7cf24db
1 parent d13a5a6
commit 7cf24db
Show file tree

Hide file tree

Showing 60 changed files with 3,738 additions and 1,413 deletions.
diff --git a/.prepare_changelog.sh b/.prepare_changelog.sh
@@ -0,0 +1,4 @@
+set -eu
+
+echo "Updating changelog headers' levels ..."
+sed -rie 's/^# \[/## \[/g' docs/changelog/CHANGELOG.md
diff --git a/.prepare_release.sh b/.prepare_release.sh
@@ -3,9 +3,11 @@ set -eu
 NEW_DOC_VER="**Latest Version: $1**"
 NEW_DOC_VER_RU="**Актуальная версия: $1**"
 NEW_BUILD_VER="  val releaseVersion = \"$1\""
+NEW_APP_VER="app-version = $1"
 
 echo "Updating versions to $1 ..."
 sed -i '/**Latest Version/c\'"$NEW_DOC_VER" docs/en/index.md
 sed -i '/**Актуальная версия/c\'"$NEW_DOC_VER_RU" docs/ru/index.md
 sed -i '/**Latest Version/c\'"$NEW_DOC_VER" README.md
-sed -i '/  val releaseVersion/c\'"$NEW_BUILD_VER" project/Version.scala
+sed -i '/  val releaseVersion/c\'"$NEW_BUILD_VER" project/Version.scala
+sed -i '/app-version/c\'"$NEW_APP_VER" checkita-core/src/main/resources/version-info.properties
diff --git a/.releaserc.json b/.releaserc.json
@@ -6,15 +6,16 @@
         "@semantic-release/commit-analyzer",
         "@semantic-release/release-notes-generator",
         [
-            "@semantic-release/exec",
+            "@semantic-release/changelog",
             {
-                "verifyReleaseCmd": "./.prepare_release.sh ${nextRelease.version}"
+                "changelogFile": "docs/changelog/CHANGELOG.md"
             }
         ],
         [
-            "@semantic-release/changelog",
+            "@semantic-release/exec",
             {
-                "changelogFile": "docs/changelog/CHANGELOG.md"
+                "verifyReleaseCmd": "./.prepare_release.sh ${nextRelease.version}",
+                "prepare": "./.prepare_changelog.sh"
             }
         ],
         [

diff --git a/checkita-core/src/main/resources/db/specific/h2/V1.2__add_metadata_column.sql b/checkita-core/src/main/resources/db/specific/h2/V1.2__add_metadata_column.sql
@@ -0,0 +1,185 @@
+ALTER TABLE "${defaultSchema}"."results_metric_regular" RENAME TO "results_metric_regular_backup";
+CREATE TABLE "${defaultSchema}"."results_metric_regular"
+(
+    "job_id"            VARCHAR(512)     NOT NULL,
+    "metric_id"         VARCHAR(512)     NOT NULL,
+    "metric_name"       VARCHAR(512)     NOT NULL,
+    "description"       TEXT,
+    "metadata"          TEXT,
+    "source_id"         VARCHAR(512)     NOT NULL,
+    "column_names"      TEXT,
+    "params"            TEXT,
+    "result"            DOUBLE PRECISION NOT NULL,
+    "additional_result" VARCHAR(2048),
+    "reference_date"    TIMESTAMP        NOT NULL,
+    "execution_date"    TIMESTAMP        NOT NULL,
+    UNIQUE ("job_id", "metric_id", "reference_date")
+);
+INSERT INTO "${defaultSchema}"."results_metric_regular" (
+    "job_id",
+    "metric_id",
+    "metric_name",
+    "description",
+    "metadata",
+    "source_id",
+    "column_names",
+    "params",
+    "result",
+    "additional_result",
+    "reference_date",
+    "execution_date"
+) SELECT "job_id",
+         "metric_id",
+         "metric_name",
+         "description",
+         null,
+         "source_id",
+         "column_names",
+         "params",
+         "result",
+         "additional_result",
+         "reference_date",
+         "execution_date"
+FROM "${defaultSchema}"."results_metric_regular_backup";
+DROP TABLE "${defaultSchema}"."results_metric_regular_backup";
+
+
+ALTER TABLE "${defaultSchema}"."results_metric_composed" RENAME TO "results_metric_composed_backup";
+CREATE TABLE "${defaultSchema}"."results_metric_composed"
+(
+    "job_id"            VARCHAR(512)     NOT NULL,
+    "metric_id"         VARCHAR(512)     NOT NULL,
+    "metric_name"       VARCHAR(512)     NOT NULL,
+    "description"       TEXT,
+    "metadata"          TEXT,
+    "source_id"         VARCHAR(512)     NOT NULL,
+    "formula"           TEXT             NOT NULL,
+    "result"            DOUBLE PRECISION NOT NULL,
+    "additional_result" VARCHAR(2048),
+    "reference_date"    TIMESTAMP        NOT NULL,
+    "execution_date"    TIMESTAMP        NOT NULL,
+    UNIQUE ("job_id", "metric_id", "reference_date")
+);
+INSERT INTO "${defaultSchema}"."results_metric_composed" (
+    "job_id",
+    "metric_id",
+    "metric_name",
+    "description",
+    "metadata",
+    "source_id",
+    "formula",
+    "result",
+    "additional_result",
+    "reference_date",
+    "execution_date"
+) SELECT "job_id",
+         "metric_id",
+         "metric_name",
+         "description",
+         null,
+         "source_id",
+         "formula",
+         "result",
+         "additional_result",
+         "reference_date",
+         "execution_date"
+FROM "${defaultSchema}"."results_metric_composed_backup";
+DROP TABLE "${defaultSchema}"."results_metric_composed_backup";
+
+
+ALTER TABLE "${defaultSchema}"."results_check" RENAME TO "results_check_backup";
+CREATE TABLE "${defaultSchema}"."results_check"
+(
+    "job_id"             VARCHAR(512) NOT NULL,
+    "check_id"           VARCHAR(512) NOT NULL,
+    "check_name"         VARCHAR(512) NOT NULL,
+    "description"        TEXT,
+    "metadata"           TEXT,
+    "source_id"          VARCHAR(512) NOT NULL,
+    "base_metric"        VARCHAR(512) NOT NULL,
+    "compared_metric"    VARCHAR(512),
+    "compared_threshold" DOUBLE PRECISION,
+    "lower_bound"        DOUBLE PRECISION,
+    "upper_bound"        DOUBLE PRECISION,
+    "status"             VARCHAR(512) NOT NULL,
+    "message"            TEXT,
+    "reference_date"     TIMESTAMP NOT NULL,
+    "execution_date"     TIMESTAMP NOT NULL,
+    UNIQUE ("job_id", "check_id", "reference_date")
+);
+INSERT INTO "${defaultSchema}"."results_check" (
+    "job_id",
+    "check_id",
+    "check_name",
+    "description",
+    "metadata",
+    "source_id",
+    "base_metric",
+    "compared_metric",
+    "compared_threshold",
+    "lower_bound",
+    "upper_bound",
+    "status",
+    "message",
+    "reference_date",
+    "execution_date"
+) SELECT "job_id",
+         "check_id",
+         "check_name",
+         "description",
+         null,
+         "source_id",
+         "base_metric",
+         "compared_metric",
+         "compared_threshold",
+         "lower_bound",
+         "upper_bound",
+         "status",
+         "message",
+         "reference_date",
+         "execution_date"
+FROM "${defaultSchema}"."results_check_backup";
+DROP TABLE "${defaultSchema}"."results_check_backup";
+
+
+ALTER TABLE "${defaultSchema}"."results_check_load" RENAME TO "results_check_load_backup";
+CREATE TABLE "${defaultSchema}"."results_check_load"
+(
+    "job_id"         VARCHAR(512) NOT NULL,
+    "check_id"       VARCHAR(512) NOT NULL,
+    "check_name"     VARCHAR(512) NOT NULL,
+    "description"    VARCHAR(512),
+    "metadata"       TEXT,
+    "source_id"      VARCHAR(512) NOT NULL,
+    "expected"       VARCHAR(512) NOT NULL,
+    "status"         VARCHAR(512) NOT NULL,
+    "message"        TEXT,
+    "reference_date" TIMESTAMP NOT NULL,
+    "execution_date" TIMESTAMP NOT NULL,
+    UNIQUE ("job_id", "check_id", "reference_date")
+);
+INSERT INTO "${defaultSchema}"."results_check_load" (
+    "job_id",
+    "check_id",
+    "check_name",
+    "description",
+    "metadata",
+    "source_id",
+    "expected",
+    "status",
+    "message",
+    "reference_date",
+    "execution_date"
+) SELECT "job_id",
+         "check_id",
+         "check_name",
+         null,
+         null,
+         "source_id",
+         "expected",
+         "status",
+         "message",
+         "reference_date",
+         "execution_date"
+FROM "${defaultSchema}"."results_check_load_backup";
+DROP TABLE "${defaultSchema}"."results_check_load_backup";
diff --git a/checkita-core/src/main/resources/db/specific/h2/V1.3__add_version_info.sql b/checkita-core/src/main/resources/db/specific/h2/V1.3__add_version_info.sql
@@ -0,0 +1,23 @@
+ALTER TABLE "${defaultSchema}"."job_state" RENAME TO "job_state_backup";
+CREATE TABLE "${defaultSchema}"."job_state"
+(
+    "job_id"            VARCHAR(512)     NOT NULL,
+    "config"            TEXT             NOT NULL,
+    "version_info"      VARCHAR(512)     NOT NULL,
+    "reference_date"    TIMESTAMP        NOT NULL,
+    "execution_date"    TIMESTAMP        NOT NULL,
+    UNIQUE ("job_id", "reference_date")
+);
+INSERT INTO "${defaultSchema}"."job_state" (
+    "job_id",
+    "config",
+    "version_info",
+    "reference_date",
+    "execution_date"
+) SELECT "job_id",
+         "config",
+         '{"appVersion":"<unknown>","configAPIVersion":"<unknown>"}',
+         "reference_date",
+         "execution_date"
+FROM "${defaultSchema}"."job_state_backup";
+DROP TABLE "${defaultSchema}"."job_state_backup";