diff --git a/dot/self_tests/data/expected/extract_df_from_dbt_test_results_json.csv b/dot/self_tests/data/expected/extract_df_from_dbt_test_results_json.csv index adf252f..d3f1719 100644 --- a/dot/self_tests/data/expected/extract_df_from_dbt_test_results_json.csv +++ b/dot/self_tests/data/expected/extract_df_from_dbt_test_results_json.csv @@ -1,10 +1,10 @@ ,run_id,test_id,entity_id,test_type,column_name,id_column_name,test_parameters,test_status,test_status_message,failed_tests_view,failed_tests_view_sql -test.dbt_model_1.not_negative_string_column_dot_model__fpview_registration_value__value.e15d766b3b,4541476c-814e-43fe-ab38-786f36beecbc,54196c00-98cf-3b57-a5e0-155c726552da,95bd0f60-ab59-48fc-a62e-f256f5f3e6de,not_negative_string_column,value,,{'name': 'value'},fail,"got 1 result, configured to fail if != 0",tr_dot_model__fpview_registration_value," SELECT array_agg(dot_model__fpview_registration.uuid) AS uuid_list +test.dbt_model_1.not_negative_string_column_dot_model__fpview_registration_value__value.e15d766b3b,4541476c-814e-43fe-ab38-786f36beecbc,85106e71-378d-3f8e-a9d6-2f056becf09b,b22a0dd5-f26a-3f7a-a413-867b4c5b1882,not_negative_string_column,value,,{'name': 'value'},fail,"got 1 result, configured to fail if != 0",tr_dot_model__fpview_registration_value," SELECT array_agg(dot_model__fpview_registration.uuid) AS uuid_list FROM self_tests_public_tests.dot_model__fpview_registration WHERE dot_model__fpview_registration.value::character varying::text ~~ '-%'::text HAVING count(*) > 0;" -test.dbt_model_1.not_null_dot_model__ancview_pregnancy_patient_id.af4157ba8c,4541476c-814e-43fe-ab38-786f36beecbc,8957b754-5db0-35e6-b648-98c932b70ea1,66f5d13a-8f74-4f97-836b-334d97932781,not_null,patient_id,,{},fail,"got 1 result, configured to fail if != 0",tr_dot_model__ancview_pregnancy_not_null_patient_id," SELECT dot_model__ancview_pregnancy.patient_id, +test.dbt_model_1.not_null_dot_model__ancview_pregnancy_patient_id.af4157ba8c,4541476c-814e-43fe-ab38-786f36beecbc,97f6844e-ec43-3a55-b8ba-0a5c27671ff6,3ed4d6ed-74b1-32cb-b74f-b51ebaa13294,not_null,patient_id,,{},fail,"got 1 result, configured to fail if != 0",tr_dot_model__ancview_pregnancy_not_null_patient_id," SELECT dot_model__ancview_pregnancy.patient_id, dot_model__ancview_pregnancy.value FROM self_tests_public_tests.dot_model__ancview_pregnancy WHERE dot_model__ancview_pregnancy.patient_id IS NULL;" -test.dbt_model_1.not_null_dot_model__fpview_registration_uuid.d2f7403403,4541476c-814e-43fe-ab38-786f36beecbc,8f15a2d1-3c4b-3a07-9143-e1e8c62a99c0,95bd0f60-ab59-48fc-a62e-f256f5f3e6de,not_null,uuid,,{},pass,,, +test.dbt_model_1.not_null_dot_model__fpview_registration_uuid.d2f7403403,4541476c-814e-43fe-ab38-786f36beecbc,38df794b-8f95-3404-8a83-e777a085a53b,b22a0dd5-f26a-3f7a-a413-867b4c5b1882,not_null,uuid,,{},pass,,, \ No newline at end of file diff --git a/dot/self_tests/data/queries/configured_tests_dbt_core.sql b/dot/self_tests/data/queries/configured_tests_dbt_core.sql index 9fdeb18..7e47ae2 100644 --- a/dot/self_tests/data/queries/configured_tests_dbt_core.sql +++ b/dot/self_tests/data/queries/configured_tests_dbt_core.sql @@ -2,14 +2,11 @@ INSERT INTO self_tests_dot.scenarios VALUES('MISSING-1', 'Missing fields', 'Data entry error', 'Form data entry error', 'Null fields', 'Blank fields'); INSERT INTO self_tests_dot.scenarios VALUES('INCONSISTENT-1', 'Inconsistent data', 'Data entry error', 'Form data entry error', 'Outliers', 'Jaundice alert=No when fever+jaundice; Incorrect LMP, wrong visit dates'); -INSERT INTO self_tests_dot.test_types VALUES('not_negative_string_column', 'dbt', 'Test to confirm all positive', 'column', 'name: patient_age_in_years'); -INSERT INTO self_tests_dot.test_types VALUES('not_null', 'dbt', 'Test to confirm if null', 'column', ''); - -INSERT INTO self_tests_dot.scenario_test_types VALUES('MISSING-1', 'not_null'); -INSERT INTO self_tests_dot.scenario_test_types VALUES('INCONSISTENT-1', 'not_negative_string_column'); +INSERT INTO self_tests_dot.test_types VALUES('not_negative_string_column', 'dbt', 'Test to confirm all positive', 'column', false, true); +INSERT INTO self_tests_dot.test_types VALUES('not_null', 'dbt', 'Test to confirm if null', 'column', false, true); -- non static data -entities and tests + -INSERT INTO self_tests_dot.projects SELECT 'Muso', 'Muso project', '2021-12-07 00:00:00+00', 'true', 'public'; +INSERT INTO self_tests_dot.projects SELECT 'Muso', 'Muso project', true, 'public', null, '2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'; INSERT INTO self_tests_dot.entity_categories VALUES('anc', 'antenatal care'); INSERT INTO self_tests_dot.entity_categories VALUES('fp', 'Family planning'); @@ -18,7 +15,7 @@ INSERT INTO self_tests_dot.entity_categories VALUES('fp', 'Family planning'); INSERT INTO self_tests_dot.configured_entities VALUES('66f5d13a-8f74-4f97-836b-334d97932781', 'ancview_pregnancy', 'anc', '{{ config(materialized=''view'') }} {% set schema = %} select * from -(values (''patient-id1'', ''1''), (NULL, ''2'')) x(patient_id, value)'); +(values (''patient-id1'', ''1''), (NULL, ''2'')) x(patient_id, value)','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); CREATE SCHEMA IF NOT EXISTS self_tests_public; CREATE TABLE self_tests_public.ancview_pregnancy ( @@ -28,13 +25,13 @@ CREATE TABLE self_tests_public.ancview_pregnancy ( INSERT INTO self_tests_public.ancview_pregnancy SELECT * FROM (values ('patient-id1', '1'), (NULL, '2')) x(patient_id, value); -INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '549c0575-e64c-3605-85a9-70356a23c4d2', 'MISSING-1', 3, 'Patient ID is not null', '', '', '66f5d13a-8f74-4f97-836b-334d97932781', 'not_null', 'patient_id', '', NULL, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Example'); +INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '549c0575-e64c-3605-85a9-70356a23c4d2', 'MISSING-1', 3, 'Patient ID is not null', '', '', '3ed4d6ed-74b1-32cb-b74f-b51ebaa13294', 'not_null', 'patient_id', '', NULL, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Example'); INSERT INTO self_tests_dot.configured_entities VALUES('95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'fpview_registration', 'fp', '{{ config(materialized=''view'') }} {% set schema = %} select * from -(values (''patient-id1'', ''1''), (''patient_id2'', ''2''), (''patient_id3'', ''-3'')) x(uuid, value)'); +(values (''patient-id1'', ''1''), (''patient_id2'', ''2''), (''patient_id3'', ''-3'')) x(uuid, value)','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); CREATE TABLE self_tests_public.fpview_registration ( uuid VARCHAR(300), @@ -43,8 +40,8 @@ CREATE TABLE self_tests_public.fpview_registration ( INSERT INTO self_tests_public.fpview_registration SELECT * FROM (values ('patient-id1', '1'), ('patient_id2', '2'), ('patient_id3', '-3')) x(uuid, value); -INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '549c0575-e64c-3605-85a9-70356a23c4d2', 'MISSING-1', 3, 'UUID is not null', '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'not_null', 'uuid', '', NULL, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Example'); -INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '8aca2bee-9e95-3f8a-90e9-153714e05367', 'INCONSISTENT-1', 5, 'value not negative', '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'not_negative_string_column', 'value', '', $${"name": "value"}$$, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Example'); +INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '549c0575-e64c-3605-85a9-70356a23c4d2', 'MISSING-1', 3, 'UUID is not null', '', '', 'b22a0dd5-f26a-3f7a-a413-867b4c5b1882', 'not_null', 'uuid', '', NULL, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Example'); +INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '8aca2bee-9e95-3f8a-90e9-153714e05367', 'INCONSISTENT-1', 5, 'value not negative', '', '', 'b22a0dd5-f26a-3f7a-a413-867b4c5b1882', 'not_negative_string_column', 'value', '', $${"name": "value"}$$, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Example'); -- these are generated by the dbt stages; TODO split the file in 2 for some tests CREATE OR REPLACE VIEW self_tests_public_tests.dot_model__ancview_pregnancy diff --git a/dot/self_tests/data/queries/configured_tests_sample-improved.sql b/dot/self_tests/data/queries/configured_tests_sample-improved.sql index ce1e85d..5864b27 100644 --- a/dot/self_tests/data/queries/configured_tests_sample-improved.sql +++ b/dot/self_tests/data/queries/configured_tests_sample-improved.sql @@ -2,8 +2,8 @@ INSERT INTO self_tests_dot.scenarios VALUES('DUPLICATE-1', 'Duplicate data', 'Data entry error', 'Duplicate data entered', 'Duplicate records', 'Multiple person records for the same person'); INSERT INTO self_tests_dot.scenarios VALUES('TREAT-1', 'Incorrect treatment', 'Process errors', 'Incorrect treatment', 'Outliers', 'Drug protocol not followed for Malaria treatment; FP for people on tubal ligation, pregnant or had vasectomy'); -INSERT INTO self_tests_dot.test_types VALUES('possible_duplicate_forms', 'dbt', 'Test to confirm duplicate records', 'single_table', 'table_specific_reported_date: reported| table_specific_patient_uuid: patient_id| table_specific_uuid: uuid'); -INSERT INTO self_tests_dot.test_types VALUES('custom_sql', 'dbt', 'Custom SQL, if rows returned test failed', 'any', '""select 1""'); +INSERT INTO self_tests_dot.test_types VALUES('possible_duplicate_forms', 'dbt', 'Test to confirm duplicate records', 'single_table', true, false); +INSERT INTO self_tests_dot.test_types VALUES('custom_sql', 'dbt', 'Custom SQL, if rows returned test failed', 'any', true, false); INSERT INTO self_tests_dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_reported_date', 'entity date field', 'reported', 'Column which indicates when form created'); INSERT INTO self_tests_dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_patient_uuid', 'entity id field', 'patient_id', 'Column which holds to patient uuid'); @@ -14,7 +14,7 @@ INSERT INTO self_tests_dot.scenario_test_types VALUES('DUPLICATE-1', 'possible_d INSERT INTO self_tests_dot.scenario_test_types VALUES('TREAT-1', 'custom_sql'); -- non static data -entities and tests + -INSERT INTO self_tests_dot.projects SELECT 'Muso', 'Muso project', '2021-12-07 00:00:00+00', 'true', 'public'; +INSERT INTO self_tests_dot.projects SELECT 'Muso', 'Muso project', true, 'public', null, '2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'; INSERT INTO self_tests_dot.entity_categories VALUES('anc', 'antenatal care'); INSERT INTO self_tests_dot.entity_categories VALUES('fp', 'Family planning'); @@ -24,7 +24,7 @@ INSERT INTO self_tests_dot.configured_entities VALUES('66f5d13a-8f74-4f97-836b-3 {% set schema = %} select * -from {{ schema }}.ancview_delivery'); +from {{ schema }}.ancview_delivery','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); -- same as above, this configured test cannot be run -- INSERT INTO self_tests_dot.configured_tests @@ -32,13 +32,13 @@ from {{ schema }}.ancview_delivery'); -- VALUES(TRUE, 'Muso', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', '', '', '66f5d13a-8f74-4f97-836b-334d97932781', 'possible_duplicate_forms', '', '', 'id_column', 'table_specific_reported_date: delivery_date| table_specific_patient_uuid: patient_id| table_specific_uuid: uuid| table_specific_period: day', '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Lorenzo'); INSERT INTO self_tests_dot.configured_tests (test_activated, project_id, test_id, scenario_id, priority, description, impact, proposed_remediation, entity_id, test_type, column_name, column_description, id_column_name, test_parameters, date_added, date_modified, last_updated_by) -VALUES(TRUE, 'Muso', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', '', '', '66f5d13a-8f74-4f97-836b-334d97932781', 'possible_duplicate_forms', '', '', 'id_column', +VALUES(TRUE, 'Muso', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', '', '', 'adc007dd-2407-3dc2-95a7-002067e741f9', 'possible_duplicate_forms', '', '', 'id_column', $${"table_specific_reported_date": "delivery_date", "table_specific_patient_uuid": "patient_id", "table_specific_uuid": "uuid", "table_specific_period": "day"}$$ , '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Lorenzo'); INSERT INTO self_tests_dot.configured_entities VALUES('95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'fpview_registration', 'fp', 'select * from (values (''patient-id1'', ''1''), (''patient_id2'', ''2''), (''patient_id3'', ''3'')) x(patient_id, value) -'); +','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); -- NFP-1 -- INSERT INTO self_tests_dot.configured_tests @@ -52,6 +52,6 @@ INSERT INTO self_tests_dot.configured_entities VALUES('95bd0f60-ab59-48fc-a62e-f -- LIMIT 2', '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Leah'); INSERT INTO self_tests_dot.configured_tests (test_activated, project_id, test_id, scenario_id, priority, description, impact, proposed_remediation, entity_id, test_type, column_name, column_description, id_column_name, test_parameters, date_added, date_modified, last_updated_by) -VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'custom_sql', '', '', 'patient_id', +VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', '', '', '52aa8e99-5221-3aac-bca5-b52b80b90929', 'custom_sql', '', '', 'patient_id', $${"query": "SELECT patient_id as primary_table_id_field, 'dot_model__fpview_registration' as primary_table, value FROM {{ ref('dot_model__fpview_registration') }} a LIMIT 2"}$$ , '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Leah'); diff --git a/dot/self_tests/data/queries/configured_tests_sample.sql b/dot/self_tests/data/queries/configured_tests_sample.sql index 14d4fb9..a1b14aa 100644 --- a/dot/self_tests/data/queries/configured_tests_sample.sql +++ b/dot/self_tests/data/queries/configured_tests_sample.sql @@ -2,8 +2,8 @@ INSERT INTO self_tests_dot.scenarios VALUES('DUPLICATE-1', 'Duplicate data', 'Data entry error', 'Duplicate data entered', 'Duplicate records', 'Multiple person records for the same person'); INSERT INTO self_tests_dot.scenarios VALUES('TREAT-1', 'Incorrect treatment', 'Process errors', 'Incorrect treatment', 'Outliers', 'Drug protocol not followed for Malaria treatment; FP for people on tubal ligation, pregnant or had vasectomy'); -INSERT INTO self_tests_dot.test_types VALUES('possible_duplicate_forms', 'dbt', 'Test to confirm duplicate records', 'single_table', 'table_specific_reported_date: reported| table_specific_patient_uuid: patient_id| table_specific_uuid: uuid'); -INSERT INTO self_tests_dot.test_types VALUES('custom_sql', 'dbt', 'Custom SQL, if rows returned test failed', 'any', '""select 1""'); +INSERT INTO self_tests_dot.test_types VALUES('possible_duplicate_forms', 'dbt', 'Test to confirm duplicate records', 'single_table', true, false); +INSERT INTO self_tests_dot.test_types VALUES('custom_sql', 'dbt', 'Custom SQL, if rows returned test failed', 'any', true, false); INSERT INTO self_tests_dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_reported_date', 'entity date field', 'reported', 'Column which indicates when form created'); INSERT INTO self_tests_dot.test_parameters_interface VALUES('possible_duplicate_forms', 'table_specific_patient_uuid', 'entity id field', 'patient_id', 'Column which holds to patient uuid'); @@ -14,7 +14,7 @@ INSERT INTO self_tests_dot.scenario_test_types VALUES('DUPLICATE-1', 'possible_d INSERT INTO self_tests_dot.scenario_test_types VALUES('TREAT-1', 'custom_sql'); -- non static data -entities and tests + -INSERT INTO self_tests_dot.projects SELECT 'Muso', 'Muso project', '2021-12-07 00:00:00+00', 'true', 'public'; +INSERT INTO self_tests_dot.projects SELECT 'Muso', 'Muso project', true, 'public', null, '2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'; INSERT INTO self_tests_dot.entity_categories VALUES('anc', 'antenatal care'); INSERT INTO self_tests_dot.entity_categories VALUES('fp', 'Family planning'); @@ -24,17 +24,16 @@ INSERT INTO self_tests_dot.configured_entities VALUES('66f5d13a-8f74-4f97-836b-3 {% set schema = %} select * -from {{ schema }}.ancview_delivery'); +from {{ schema }}.ancview_delivery','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); -- same as above, this configured test cannot be run --- INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', '', '', '66f5d13a-8f74-4f97-836b-334d97932781', 'possible_duplicate_forms', '', '', 'table_specific_reported_date: delivery_date| table_specific_patient_uuid: patient_id| table_specific_uuid: uuid| table_specific_period: day', '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Lorenzo'); -INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', '', '', '66f5d13a-8f74-4f97-836b-334d97932781', 'possible_duplicate_forms', '', '', +INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', '7f78de0e-8268-3da6-8845-9a445457cc9a', 'DUPLICATE-1', 3, '', '', '', 'adc007dd-2407-3dc2-95a7-002067e741f9', 'possible_duplicate_forms', '', '', $${"table_specific_uuid": "uuid", "table_specific_period": "day", "table_specific_patient_uuid": "patient_id", "table_specific_reported_date": "delivery_date"}$$ , '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Lorenzo'); INSERT INTO self_tests_dot.configured_entities VALUES('95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'fpview_registration', 'fp', 'select * from (values (''patient-id1'', ''1''), (''patient_id2'', ''2''), (''patient_id3'', ''3'')) x(patient_id, value) -'); +','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); -- NFP-1 --INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'custom_sql', '', '', ' @@ -46,5 +45,5 @@ INSERT INTO self_tests_dot.configured_entities VALUES('95bd0f60-ab59-48fc-a62e-f -- LIMIT 2', '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Leah'); -- INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'custom_sql', '', '', $${"query": "SELECT patient_id as primary_table_id_field, 'dot_model__fpview_registration' as primary_table, value FROM {{ ref('dot_model__fpview_registration') }} a LIMIT 2"}$$, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Leah'); -INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'custom_sql', '', '', +INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', '', '', '52aa8e99-5221-3aac-bca5-b52b80b90929', 'custom_sql', '', '', $${"query": "SELECT patient_id as primary_table_id_field, 'dot_model__fpview_registration' as primary_table, value FROM {{ ref('dot_model__fpview_registration') }} a LIMIT 2"}$$, '2021-12-23 19:00:00.000 -0500', '2021-12-23 19:00:00.000 -0500', 'Leah'); \ No newline at end of file diff --git a/dot/self_tests/data/queries/core_entities_creation.sql b/dot/self_tests/data/queries/core_entities_creation.sql index cd44706..e2d96a9 100644 --- a/dot/self_tests/data/queries/core_entities_creation.sql +++ b/dot/self_tests/data/queries/core_entities_creation.sql @@ -4,6 +4,6 @@ INSERT INTO self_tests_dot.configured_entities VALUES('b05f1f9c-2176-46b0-8e8f-d {% set schema = %} select * -from {{ schema }}.ancview_danger_sign'); +from {{ schema }}.ancview_danger_sign','2021-12-07 00:00:00+00','2021-12-07 00:00:00+00','Matt'); diff --git a/dot/self_tests/unit/test_dbt.py b/dot/self_tests/unit/test_dbt.py index 9f8b5ef..6a2a1c0 100644 --- a/dot/self_tests/unit/test_dbt.py +++ b/dot/self_tests/unit/test_dbt.py @@ -42,6 +42,10 @@ def test_extract_df_from_dbt_test_results_json( logger=setup_custom_logger("self_tests/output/test.log", logging.INFO), target_path="self_tests/data/dot_output_files/dbt/target", ) + + output.to_csv('self_tests/data/op.csv') + + expected = pd.read_csv( "self_tests/data/expected/extract_df_from_dbt_test_results_json.csv", index_col=0, diff --git a/dot/self_tests/unit/test_utils.py b/dot/self_tests/unit/test_utils.py index efc7c38..4edc6ec 100644 --- a/dot/self_tests/unit/test_utils.py +++ b/dot/self_tests/unit/test_utils.py @@ -48,8 +48,8 @@ def get_test_summary() -> Tuple[pd.DataFrame, uuid.UUID]: run_id = uuid.UUID("4541476c-814e-43fe-ab38-786f36beecbc") test_summary_row = { "run_id": run_id, - "test_id": "ef6bb39d-7a89-3972-b5b6-719d4435e7f9", - "entity_id": "95bd0f60-ab59-48fc-a62e-f256f5f3e6de", + "test_id": "8317ea22-70d1-32d2-b061-e25e0aac9f8e", + "entity_id": "52aa8e99-5221-3aac-bca5-b52b80b90929", "test_type": "custom_sql", "column_name": "", # "id_column_name": "patient_id", @@ -72,7 +72,7 @@ def test_get_test_id( generated_test_id = get_test_id( test_type="possible_duplicate_forms", - entity_id="66f5d13a-8f74-4f97-836b-334d97932781", + entity_id="adc007dd-2407-3dc2-95a7-002067e741f9", column="", project_id="Muso", test_parameters='''$$ @@ -84,7 +84,7 @@ def test_get_test_id( }$$ '''.replace('\n','') ) - expected_test_id = "0a055ffd-c753-3c27-9de9-a4665352513f" + expected_test_id = "5501f109-a017-3594-96ea-eb8438187509" self.assertEqual( expected_test_id, generated_test_id, @@ -107,7 +107,7 @@ def test_possible_duplicate_forms_test_malformed( "'7f78de0e-8268-3da6-8845-9a445457cc9a'," "'DUPLICATE-1'," "3, '', '', '', " - "'66f5d13a-8f74-4f97-836b-334d97932781'," + "'adc007dd-2407-3dc2-95a7-002067e741f9'," "'possible_duplicate_forms', '', ''," '''$${ 'table_specific_uuid': 'uuid', @@ -135,7 +135,7 @@ def test_custom_sql_test_malformed( INSERT INTO self_tests_dot.configured_tests VALUES(TRUE, 'Muso', 'c4a3da8f-32f4-4e9b-b135-354de203ca70', 'TREAT-1', 5, 'Test for new family planning method (NFP-1)', - '', '', '95bd0f60-ab59-48fc-a62e-f256f5f3e6de', 'custom_sql', + '', '', '52aa8e99-5221-3aac-bca5-b52b80b90929', 'custom_sql', '', '', $${"query":"SELECT patient_id as primary_table_id_field, value @@ -158,7 +158,7 @@ def test_get_configured_tests_row_reference_error( with self.assertRaises(ReferenceError): _ = get_configured_tests_row( test_type="possible_duplicate_forms", - entity_id="66f5d13a-8f74-4f97-836b-334d97932781", + entity_id="adc007dd-2407-3dc2-95a7-002067e741f9", column="", project_id="Muso", test_parameters=''' @@ -180,13 +180,13 @@ def test_get_configured_tests_row( configured_tests_row = get_configured_tests_row( test_type="possible_duplicate_forms", - entity_id="66f5d13a-8f74-4f97-836b-334d97932781", + entity_id="adc007dd-2407-3dc2-95a7-002067e741f9", column="", project_id="Muso", test_parameters="{'table_specific_uuid': 'uuid', 'table_specific_period': 'day', 'table_specific_patient_uuid': 'patient_id', 'table_specific_reported_date': 'delivery_date'}", ) - expected_test_id = "0a055ffd-c753-3c27-9de9-a4665352513f" + expected_test_id = "5501f109-a017-3594-96ea-eb8438187509" self.assertEqual( expected_test_id, configured_tests_row["test_id"], @@ -202,7 +202,7 @@ def test_get_configured_tests_row( "description": "", "impact": "", "proposed_remediation": "", - "entity_id": "66f5d13a-8f74-4f97-836b-334d97932781", + "entity_id": "adc007dd-2407-3dc2-95a7-002067e741f9", "test_type": "possible_duplicate_forms", "column_name": "", "column_description": "", @@ -224,7 +224,7 @@ def test_get_entity_id_from_name( mock_get_filename_safely.side_effect = self.mock_get_filename_safely self.assertEqual( - "66f5d13a-8f74-4f97-836b-334d97932781", + "adc007dd-2407-3dc2-95a7-002067e741f9", get_entity_id_from_name("Muso", "dot_model__ancview_delivery"), ) @@ -237,7 +237,7 @@ def test_get_entity_name_from_id( self.assertEqual( "dot_model__ancview_delivery", - get_entity_name_from_id("Muso", "66f5d13a-8f74-4f97-836b-334d97932781"), + get_entity_name_from_id("Muso", "adc007dd-2407-3dc2-95a7-002067e741f9"), ) @patch("utils.configuration_utils._get_filename_safely") diff --git a/dot/self_tests/unit/test_utils_schema_improved.py b/dot/self_tests/unit/test_utils_schema_improved.py index 8490c8b..86a60a9 100644 --- a/dot/self_tests/unit/test_utils_schema_improved.py +++ b/dot/self_tests/unit/test_utils_schema_improved.py @@ -49,12 +49,12 @@ def test_get_configured_tests_row( configured_tests_row = get_configured_tests_row( test_type="possible_duplicate_forms", - entity_id="66f5d13a-8f74-4f97-836b-334d97932781", + entity_id="adc007dd-2407-3dc2-95a7-002067e741f9", column="", project_id="Muso", test_parameters="{'table_specific_uuid': 'uuid', 'table_specific_period': 'day', 'table_specific_patient_uuid': 'patient_id', 'table_specific_reported_date': 'delivery_date'}" ) - expected_test_id = "0a055ffd-c753-3c27-9de9-a4665352513f" + expected_test_id = "5501f109-a017-3594-96ea-eb8438187509" self.assertEqual( expected_test_id, configured_tests_row["test_id"], @@ -70,7 +70,7 @@ def test_get_configured_tests_row( "description": "", "impact": "", "proposed_remediation": "", - "entity_id": "66f5d13a-8f74-4f97-836b-334d97932781", + "entity_id": "adc007dd-2407-3dc2-95a7-002067e741f9", "test_type": "possible_duplicate_forms", "column_name": "", "column_description": "", @@ -151,7 +151,7 @@ def test_get_test_rows( test_summary_row = { "run_id": run_id, "test_id": "ef6bb39d-7a89-3972-b5b6-719d4435e7f9", - "entity_id": "95bd0f60-ab59-48fc-a62e-f256f5f3e6de", + "entity_id": "52aa8e99-5221-3aac-bca5-b52b80b90929", "test_type": "custom_sql", "column_name": "", "id_column_name": "patient_id",