Split checks into stages

Author: greenape

CHANGELOG.md

@@ -107,11 +107,7 @@ def get_available_dates(engine: Engine):
 
     templates = (
         QaTemplate(
-            display_name=(
-                ".".join((Path(t).stem, Path(t).parent.stem))
-                if Path(t).parent != Path(".")
-                else Path(t).stem
-            ),
+            display_name=Path(t).stem,
             template=env.get_template(t),
             event_type=Path(t).parent.stem if Path(t).parent != Path(".") else "any",
         )

flowdb/testdata/bin/run_qa_checks.py

@@ -50,7 +50,7 @@ ADD --chown=postgres flowdb/testdata/test_data/sql/admin*.sql /docker-entrypoint
 ADD --chown=postgres flowdb/testdata/synthetic_data/data/NPL_admbnda_adm3_Districts_simplified.geojson /opt/synthetic_data/
 # Copy QA templates from flowetl
 
-COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/qa_checks /docker-entrypoint-initdb.d/qa_checks
+COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/final /docker-entrypoint-initdb.d/qa_checks
 
 # Need to make postgres is owner of any subdirectrories
 RUN mkdir docker-entrypoint-initdb.d/sql/syntheticdata/sql &&  chown -R postgres /docker-entrypoint-initdb.d

flowdb_synthetic_data.Dockerfile

@@ -19,4 +19,4 @@
 !flowdb/testdata/bin/9910_run_synthetic_dfs_data_generation_script.sh
 !flowdb/testdata/bin/run_qa_checks.py
 !flowdb/testdata/test_data/py/
-!flowetl/flowetl/flowetl/qa_checks/qa_checks
+!flowetl/flowetl/flowetl/qa_checks/

flowdb_synthetic_data.Dockerfile.dockerignore

@@ -43,7 +43,7 @@ ADD --chown=postgres flowdb/testdata/test_data/sql/* /docker-entrypoint-initdb.d
 ADD --chown=postgres flowdb/testdata/test_data/py/* /docker-entrypoint-initdb.d/py/testdata/
 ADD --chown=postgres flowdb/testdata/test_data/data/ /docker-entrypoint-initdb.d/data/
 COPY --chown=postgres flowdb/testdata/test_data/data/*.csv /docker-entrypoint-initdb.d/data/csv/
-COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/qa_checks /docker-entrypoint-initdb.d/qa_checks
+COPY --chown=postgres flowetl/flowetl/flowetl/qa_checks/final /docker-entrypoint-initdb.d/qa_checks
 # Need to make postgres owner of any subdirectories
 RUN chown -R postgres /docker-entrypoint-initdb.d
 # Explicitly set number of days of dfs data to match test data

flowdb_testdata.Dockerfile

@@ -19,5 +19,5 @@
 !flowdb/testdata/test_data/sql/
 !flowdb/testdata/test_data/py/
 !flowdb/testdata/test_data/data/
-!flowetl/flowetl/flowetl/qa_checks/qa_checks
+!flowetl/flowetl/flowetl/qa_checks/
 

flowdb_testdata.Dockerfile.dockerignore

@@ -2,3 +2,4 @@ include versioneer.py
 include flowetl/_version.py
 include flowetl/qa_checks/*/*
 include flowetl/qa_checks/*/*/*
+include flowetl/qa_checks/*/*/*/*

flowetl/flowetl/MANIFEST.in

@@ -7,7 +7,7 @@
 QACheckOperator = wrapped_sql_operator(
     sql="""INSERT INTO etl.post_etl_queries(cdr_date, cdr_type, type_of_query_or_check, outcome, timestamp) 
             VALUES
-        (date '{{{{ ds }}}}', '{{{{ params.cdr_type }}}}', '{{{{ task.task_id }}}}', ({sql}), NOW())
+        (date '{{{{ ds }}}}', '{{{{ params.cdr_type }}}}', '{{{{ task.task_id.split(".")[0] }}}}', ({sql}), NOW())
     """,
     class_name="QACheckOperator",
 )