From c95bcbb417cc2c4302c7a7783dcf8725a447d540 Mon Sep 17 00:00:00 2001 From: AlaikseiKatyshou Date: Wed, 26 Jun 2024 16:18:36 +0300 Subject: [PATCH 1/3] Add two parameters in ConceptAncestorCore() and pManualConceptAncestor(): - pIncludeNonStandard: to include ancestors with non-standard concepts - pIncludeInvalidReason: to include ancestors with invalid reasons Modify text in email notification in pManualConceptAncestor() --- .../vocabulary_pack/ConceptAncestorCore.sql | 51 +++++++++++-------- .../pManualConceptAncestor.sql | 18 +++++-- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/working/packages/vocabulary_pack/ConceptAncestorCore.sql b/working/packages/vocabulary_pack/ConceptAncestorCore.sql index 027a89ac6..c70e54166 100644 --- a/working/packages/vocabulary_pack/ConceptAncestorCore.sql +++ b/working/packages/vocabulary_pack/ConceptAncestorCore.sql @@ -1,4 +1,9 @@ -CREATE OR REPLACE FUNCTION vocabulary_pack.ConceptAncestorCore (pVocabularies TEXT, pDebugLoops BOOLEAN) +CREATE OR REPLACE FUNCTION vocabulary_pack.ConceptAncestorCore ( + pVocabularies TEXT, + pDebugLoops BOOLEAN, + pIncludeNonStandard BOOLEAN DEFAULT FALSE, + pIncludeInvalidReason BOOLEAN DEFAULT FALSE +) RETURNS VOID AS $BODY$ /* @@ -38,18 +43,18 @@ BEGIN JOIN relationship s ON s.relationship_id = r.relationship_id AND s.defines_ancestry = 1 JOIN concept c1 ON c1.concept_id = r.concept_id_1 - AND c1.invalid_reason IS NULL + AND (pIncludeInvalidReason OR c1.invalid_reason IS NULL) AND ( c1.vocabulary_id = ANY (iVocabularies) OR iVocabularies IS NULL ) JOIN concept c2 ON c2.concept_id = r.concept_id_2 - AND c2.invalid_reason IS NULL + AND (pIncludeInvalidReason OR c2.invalid_reason IS NULL) AND ( c2.vocabulary_id = ANY (iVocabularies) OR iVocabularies IS NULL ) - WHERE r.invalid_reason IS NULL; + WHERE (pIncludeInvalidReason OR r.invalid_reason IS NULL); CREATE INDEX idx_temp_ca_base$ ON temporary_ca_base$ (ancestor_concept_id) INCLUDE (descendant_concept_id, levels_of_separation) WITH (FILLFACTOR=100); ANALYZE temporary_ca_base$; @@ -84,12 +89,12 @@ BEGIN ca.levels_of_separation, ARRAY [descendant_concept_id] AS full_path FROM temporary_ca_base$ ca - WHERE EXISTS ( + WHERE (pIncludeNonStandard OR EXISTS ( SELECT 1 FROM concept c_int WHERE c_int.concept_id = ca.ancestor_concept_id AND c_int.standard_concept IS NOT NULL --remove non-standard records in ancestor_concept_id - ) + )) AND ca.ancestor_concept_id > cRecord.ancestor_concept_id_min AND ca.ancestor_concept_id <= cRecord.ancestor_concept_id_max @@ -120,23 +125,27 @@ BEGIN hc.descendant_concept_id; END LOOP; - --remove non-standard records in descendant_concept_id - DELETE - FROM temporary_ca$ ca - USING concept c - WHERE c.standard_concept IS NULL - AND c.concept_id = ca.descendant_concept_id; - + IF NOT pIncludeNonStandard THEN + --remove non-standard records in descendant_concept_id + DELETE + FROM temporary_ca$ ca + USING concept c + WHERE c.standard_concept IS NULL + AND c.concept_id = ca.descendant_concept_id; + END IF; + IF pDebugLoops THEN --in debug mode we preserve the temporary_ca$ for future use RETURN; END IF; - PERFORM FROM temporary_ca$ ca WHERE ca.ancestor_concept_id=ca.descendant_concept_id LIMIT 1; - IF FOUND THEN - --loops found, e.g. code1 'Subsumes' code2 'Subsumes' code1, which leads to the creation of an code1-code1 chain - RAISE EXCEPTION $$Loops in the ancestor were found, please run SELECT vocabulary_pack.GetAncestorLoops(pVocabularies=>'%'); For view chains you can also use SELECT * FROM vocabulary_pack.GetAncestorPath_in_DEV(ancestor_id, descendant_id)$$, pVocabularies; - END IF; + IF NOT (pIncludeNonStandard OR pIncludeInvalidReason) THEN + PERFORM FROM temporary_ca$ ca WHERE ca.ancestor_concept_id=ca.descendant_concept_id LIMIT 1; + IF FOUND THEN + --loops found, e.g. code1 'Subsumes' code2 'Subsumes' code1, which leads to the creation of an code1-code1 chain + RAISE EXCEPTION $$Loops in the ancestor were found, please run SELECT vocabulary_pack.GetAncestorLoops(pVocabularies=>'%'); For view chains you can also use SELECT * FROM vocabulary_pack.GetAncestorPath_in_DEV(ancestor_id, descendant_id)$$, pVocabularies; + END IF; + END IF; --Add connections to self for those vocabs having at least one concept in the concept_relationship table INSERT INTO temporary_ca$ @@ -149,10 +158,10 @@ BEGIN SELECT c_int.vocabulary_id FROM concept_relationship cr_int JOIN concept c_int ON c_int.concept_id = cr_int.concept_id_1 - WHERE cr_int.invalid_reason IS NULL + WHERE (pIncludeInvalidReason OR cr_int.invalid_reason IS NULL) ) - AND c.invalid_reason IS NULL - AND c.standard_concept IS NOT NULL; + AND (pIncludeInvalidReason OR c.invalid_reason IS NULL) + AND (pIncludeNonStandard OR c.standard_concept IS NOT NULL); CREATE INDEX idx_tmp_ca$ ON temporary_ca$ (ancestor_concept_id, descendant_concept_id) INCLUDE (min_levels_of_separation, max_levels_of_separation) WITH (FILLFACTOR=100); ANALYZE temporary_ca$; diff --git a/working/packages/vocabulary_pack/pManualConceptAncestor.sql b/working/packages/vocabulary_pack/pManualConceptAncestor.sql index d8768a287..d31767393 100644 --- a/working/packages/vocabulary_pack/pManualConceptAncestor.sql +++ b/working/packages/vocabulary_pack/pManualConceptAncestor.sql @@ -1,4 +1,8 @@ -CREATE OR REPLACE FUNCTION vocabulary_pack.pManualConceptAncestor (pVocabularies TEXT) +CREATE OR REPLACE FUNCTION vocabulary_pack.pManualConceptAncestor ( + pVocabularies TEXT, + pIncludeNonStandard BOOLEAN DEFAULT FALSE, + pIncludeInvalidReason BOOLEAN DEFAULT FALSE +) RETURNS VOID AS $BODY$ /* @@ -17,10 +21,18 @@ DECLARE cStartTime TIMESTAMP:=CLOCK_TIMESTAMP(); cWorkTime NUMERIC; BEGIN - PERFORM vocabulary_pack.ConceptAncestorCore(pVocabularies, FALSE); + PERFORM vocabulary_pack.ConceptAncestorCore(pVocabularies, FALSE, pIncludeNonStandard, pIncludeInvalidReason); cWorkTime:=ROUND((EXTRACT(EPOCH FROM CLOCK_TIMESTAMP()-cStartTime)/60)::NUMERIC,1); - PERFORM devv5.SendMailHTML (iSmallCA_emails, 'Manual concept ancestor in '||UPPER(CURRENT_SCHEMA)||' [ok]', 'Manual concept ancestor in '||UPPER(CURRENT_SCHEMA)||' completed'||crlf||'Execution time: '||cWorkTime||' min'); + PERFORM devv5.SendMailHTML ( + iSmallCA_emails, + 'Manual concept ancestor in '||UPPER(CURRENT_SCHEMA)||' [ok]', + 'Manual concept ancestor in '||UPPER(CURRENT_SCHEMA)||' completed'||crlf|| + CASE WHEN pIncludeNonStandard THEN 'incliding non-standard concepts' ELSE '' END||crlf|| + CASE WHEN pIncludeInvalidReason THEN 'including invalid reason ' ELSE '' END||crlf|| + 'Execution time: '||cWorkTime||' min'||crlf|| + + ); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS cRet = PG_EXCEPTION_CONTEXT, cRet2 = PG_EXCEPTION_DETAIL; From 023efefa74f2d4f312a2704b4ebb28deb109ab8a Mon Sep 17 00:00:00 2001 From: AlaikseiKatyshou Date: Wed, 26 Jun 2024 16:58:41 +0300 Subject: [PATCH 2/3] Fix notification text --- working/packages/vocabulary_pack/pManualConceptAncestor.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/working/packages/vocabulary_pack/pManualConceptAncestor.sql b/working/packages/vocabulary_pack/pManualConceptAncestor.sql index d31767393..8e216777f 100644 --- a/working/packages/vocabulary_pack/pManualConceptAncestor.sql +++ b/working/packages/vocabulary_pack/pManualConceptAncestor.sql @@ -30,7 +30,7 @@ BEGIN 'Manual concept ancestor in '||UPPER(CURRENT_SCHEMA)||' completed'||crlf|| CASE WHEN pIncludeNonStandard THEN 'incliding non-standard concepts' ELSE '' END||crlf|| CASE WHEN pIncludeInvalidReason THEN 'including invalid reason ' ELSE '' END||crlf|| - 'Execution time: '||cWorkTime||' min'||crlf|| + 'Execution time: '||cWorkTime||' min' ); From 053933116a80a58d8c7562ee3b99e66dd1d46276 Mon Sep 17 00:00:00 2001 From: AlaikseiKatyshou Date: Wed, 14 Aug 2024 18:18:59 +0300 Subject: [PATCH 3/3] Fix ConceptAncestorCore - use only valid relationships --- working/packages/vocabulary_pack/ConceptAncestorCore.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/working/packages/vocabulary_pack/ConceptAncestorCore.sql b/working/packages/vocabulary_pack/ConceptAncestorCore.sql index c70e54166..8c1ee1bd9 100644 --- a/working/packages/vocabulary_pack/ConceptAncestorCore.sql +++ b/working/packages/vocabulary_pack/ConceptAncestorCore.sql @@ -54,7 +54,7 @@ BEGIN c2.vocabulary_id = ANY (iVocabularies) OR iVocabularies IS NULL ) - WHERE (pIncludeInvalidReason OR r.invalid_reason IS NULL); + WHERE r.invalid_reason IS NULL; CREATE INDEX idx_temp_ca_base$ ON temporary_ca_base$ (ancestor_concept_id) INCLUDE (descendant_concept_id, levels_of_separation) WITH (FILLFACTOR=100); ANALYZE temporary_ca_base$; @@ -158,7 +158,7 @@ BEGIN SELECT c_int.vocabulary_id FROM concept_relationship cr_int JOIN concept c_int ON c_int.concept_id = cr_int.concept_id_1 - WHERE (pIncludeInvalidReason OR cr_int.invalid_reason IS NULL) + WHERE r_int.invalid_reason IS NULL ) AND (pIncludeInvalidReason OR c.invalid_reason IS NULL) AND (pIncludeNonStandard OR c.standard_concept IS NOT NULL);