ICD CDE

CIM10/manual_work/crm_changes.sql

@@ -17,6 +17,8 @@
 * Date: 2021
 **************************************************************************/
 -- 1. Update the concept_relationship_stage table
+CREATE TABLE concept_relationship_manual_bu as (SELECT * FROM concept_relationship_manual);
+INSERT INTO concept_relationship_manual (SELECT * FROM concept_relationship_manual_bu);
 TRUNCATE TABLE dev_CIM10.concept_relationship_manual;
 INSERT INTO concept_relationship_manual (concept_code_1, concept_code_2, vocabulary_id_1, vocabulary_id_2, relationship_id, valid_start_date, valid_end_date, invalid_reason)
 SELECT DISTINCT
@@ -108,23 +110,4 @@ INSERT INTO concept_relationship_manual(concept_code_1, concept_code_2, vocabula
                        vocabulary_id_2,
                        relationship_id FROM concept_relationship_manual)
     )
-;
-
--- Some manual fixes
---INSERT INTO concept_relationship_manual VALUES
---('Z52.80', 'OMOP5165859', 'CIM10', 'OMOP Extension', 'Maps to', '2024-02-27', '2099-12-31', null);
---
---INSERT INTO concept_relationship_manual VALUES
---('Z52.80', '53958007', 'CIM10', 'SNOMED', 'Maps to value', '2024-02-27', '2099-12-31', null);
---
---INSERT INTO concept_relationship_manual VALUES
---('Z52.88', 'OMOP5165859', 'CIM10', 'OMOP Extension', 'Maps to', '2024-02-27', '2099-12-31', null);
---
---INSERT INTO concept_relationship_manual VALUES
---('Z52.88', '53958007', 'CIM10', 'SNOMED', 'Maps to value', '2024-02-27', '2099-12-31', null);
---
---INSERT INTO concept_relationship_manual VALUES
---('Z95.80', 'OMOP5165859', 'CIM10', 'OMOP Extension', 'Maps to', '2024-02-27', '2099-12-31', null);
---
---INSERT INTO concept_relationship_manual VALUES
---('Z95.80', '429381005', 'CIM10', 'SNOMED', 'Maps to value', '2024-02-27', '2099-12-31', null);
+;

CIM10/manual_work/mapping_refresh.sql

@@ -13,7 +13,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
-* Authors: Dmitry Dymshyts, Polina Talapova, Daryna Ivakhnenko
+* Authors: Irina Zherko, Dmitry Dymshyts, Polina Talapova, Daryna Ivakhnenko
 * Date: 2021
 **************************************************************************/
 --Create table CIM10_refresh
@@ -62,7 +62,7 @@ with deprecated_mappings as
     FROM concept_relationship_stage crs
     WHERE (concept_code_1, vocabulary_id_1, concept_code_2, vocabulary_id_2, relationship_id) IN
     (SELECT concept_code_1, vocabulary_id_1, concept_code_2, vocabulary_id_2, relationship_id
-    FROM concept_relationship_manual)
+    FROM devv5.base_concept_relationship_manual WHERE vocabulary_id_1 = 'CIM10')
     and invalid_reason = 'D'
     and valid_end_date in (SELECT DISTINCT GREATEST(crs.valid_start_date, (
 				SELECT MAX(v.latest_update) - 1
@@ -165,7 +165,7 @@ and c.vocabulary_id = 'SNOMED'
 JOIN concept_stage cs ON crs.concept_code_1 = cs.concept_code
 WHERE (concept_code_1, vocabulary_id_1, concept_code_2, vocabulary_id_2, relationship_id) IN
     (SELECT concept_code_1, vocabulary_id_1, concept_code_2, vocabulary_id_2, relationship_id
-    FROM concept_relationship_manual)
+    FROM devv5.base_concept_relationship_manual WHERE vocabulary_id_1 = 'CIM10')
     and crs.invalid_reason = 'D'
     and crs.valid_end_date in (SELECT DISTINCT GREATEST(crs.valid_start_date, (
 				SELECT MAX(v.latest_update) - 1
@@ -199,7 +199,7 @@ WHERE (concept_code_1, vocabulary_id_1, concept_code_2, vocabulary_id_2, relatio
        AND c.standard_concept = 'S'
        AND c.invalid_reason is null);
 
---Insert the rest of crm concepts
+--Insert the rest of crm relationships which are not represented in the icd_cde_source table
 INSERT INTO CIM10_refresh
     (source_code,
      source_code_description,
@@ -234,11 +234,13 @@ SELECT
     crm.valid_start_date as valid_start_date,
     crm.valid_end_date as valid_end_date,
     'crm' as mapping_origin
-FROM concept_relationship_manual crm
+FROM devv5.base_concept_relationship_manual crm
 LEFT JOIN concept c on crm.concept_code_1 = c.concept_code and crm.vocabulary_id_1 = c.vocabulary_id
 LEFT JOIN concept c2 on crm.concept_code_2 = c2.concept_code and crm.vocabulary_id_2 = c2.vocabulary_id
-WHERE (crm.concept_code_1, crm.vocabulary_id_1, crm.relationship_id) not in (SELECT source_code, source_vocabulary_id, relationship_id FROM CIM10_refresh)
-AND crm.vocabulary_id_1 = 'CIM10';
+WHERE (crm.concept_code_1, crm.vocabulary_id_1, crm.relationship_id) NOT IN (SELECT source_code, source_vocabulary_id, relationship_id FROM CIM10_refresh)
+AND crm.vocabulary_id_1 = 'CIM10'
+AND (crm.concept_code_1, crm.vocabulary_id_1, crm.concept_code_2. crm.vocabulary_id_2) NOT IN
+    (SELECT source_code, source_vocabulary_id, target_concept_code, target_vocabulary_id FROM dev_icd10.icd_cde_source);
 
 --Insert concepts without mapping --Not used at every refresh
 INSERT INTO CIM10_refresh
@@ -255,258 +257,4 @@ SELECT cs.concept_code as source_code,
 FROM concept_stage cs LEFT JOIN concept_relationship_stage crs on cs.concept_code = crs.concept_code_1
 WHERE crs.concept_code_2 is null
 and cs.invalid_reason is null
-and cs.concept_class_id NOT IN ('ICD10 Chapter','ICD10 SubChapter', 'ICD10 Hierarchy');
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-DROP TABLE IF EXISTS refresh_lookup;
-CREATE TABLE refresh_lookup AS WITH miss_map
-AS
-(
-	-- 'deprecated mapping'
-	SELECT c.concept_code AS icd_code,
-       c.concept_name AS icd_name,
-       a.relationship_id AS current_relationship,
-       b.concept_id AS current_id,
-       b.concept_code AS current_code,
-       b.concept_name AS current_name,
-       b.domain_id AS current_domain,
-       b.vocabulary_id AS current_vocabulary,
-       'deprecated mapping' AS reason
-FROM concept_relationship_stage a
-  JOIN concept b
-    ON a.concept_code_2 = b.concept_code
-   AND b.vocabulary_id = a.vocabulary_id_2
-   AND a.relationship_id IN ('Maps to', 'Maps to value')
-   AND b.invalid_reason IN ('D', 'U')
-  JOIN concept_stage c
-    ON a.concept_code_1 = c.concept_code
-    AND c.concept_class_id NOT IN ('ICD10 Chapter','ICD10 SubChapter')
-AND a.invalid_reason IS NULL
-UNION
--- 'non-standard mapping'
-SELECT c.concept_code,
-       c.concept_name,
-       a.relationship_id AS current_relationship,
-       b.concept_id AS current_id,
-       b.concept_code AS current_code,
-       b.concept_name AS current_name,
-       b.domain_id AS current_domain,
-       b.vocabulary_id AS current_vocabulary,
-       'non-standard mapping' AS reason
-FROM concept_relationship_stage a
-  JOIN concept b
-    ON a.concept_code_2 = b.concept_code
-   AND b.vocabulary_id = a.vocabulary_id_2
-   AND a.relationship_id IN ('Maps to', 'Maps to value')
-   AND b.invalid_reason IS NULL
-   AND b.standard_concept IS NULL
-  JOIN concept_stage c
-    ON a.concept_code_1 = c.concept_code
-    AND c.concept_class_id NOT IN ('ICD10 Chapter','ICD10 SubChapter')
-AND a.invalid_reason is null
-UNION
--- 'without mapping'
-SELECT a.concept_code,
-       a.concept_name,
-       NULL,
-       NULL,
-       NULL,
-       NULL,
-       NULL,
-       NULL,
-       'without mapping' AS reason
-FROM concept_stage a
-  LEFT JOIN concept_relationship_stage r
-         ON a.concept_code = concept_code_1
-        AND r.relationship_id IN ('Maps to')
-        AND r.invalid_reason IS NULL
-  LEFT JOIN concept b
-         ON b.concept_code = concept_code_2
-        AND b.vocabulary_id = vocabulary_id_2
-WHERE a.vocabulary_id = 'CIM10'
-AND   a.invalid_reason IS NULL
-AND   b.concept_id IS NULL
-AND a.concept_class_id NOT IN ('ICD10 Chapter','ICD10 SubChapter')),
---brothers of depracted concepts: for cases when source concept has 1-to-many mapping and one of the target concepts is dead, we should see all other target concepts to create an accurate mapping
-miss_map_brother AS ( SELECT
-       a.icd_code,
-       a.icd_name,
-       c.relationship_id,
-       b.concept_id AS current_id,
-       b.concept_code AS current_code,
-       b.concept_name AS current_name,
-       b.domain_id AS current_domain,
-       b.vocabulary_id AS current_vocabulary,
-       b.concept_id,
-       b.concept_code,
-        b.concept_name,
-       b.domain_id,
-       b.vocabulary_id,
-       'brother of deprecated mapping' AS reason
-FROM miss_map a
-JOIN concept_relationship_stage c ON c.concept_code_1 = a.icd_code
-JOIN concept b
-    ON c.concept_code_2 = b.concept_code
-   AND b.vocabulary_id = c.vocabulary_id_2
-   AND c.relationship_id IN ('Maps to', 'Maps to value')
-   AND b.invalid_reason IS NULL),
--- concepts which mapping can be replaced through 'Maps to' relationship
-t1 AS (SELECT d.concept_code AS icd_code,
-              d.concept_name AS icd_name,
-              d.domain_id AS icd_domain,
-              j.concept_id AS repl_by_id,
-              j.concept_code AS repl_by_code,
-              j.concept_name AS repl_by_name,
-              j.domain_id AS repl_by_domain,
-              j.vocabulary_id AS repl_by_vocabulary,
-              NULL
-       FROM concept_relationship_stage a
-         JOIN concept b
-           ON a.concept_code_2 = b.concept_code
-          AND b.vocabulary_id = a.vocabulary_id_2
-          AND a.relationship_id = 'Maps to'
-          AND b.invalid_reason IN ('D', 'U')
-          JOIN concept_stage d
-    ON a.concept_code_1 = d.concept_code
-         JOIN concept_relationship r2 ON b.concept_id = r2.concept_id_1
-         JOIN concept j
-           ON j.concept_id = r2.concept_id_2
-          AND j.vocabulary_id = 'SNOMED'-- place for target vocabulary
-          AND j.standard_concept = 'S'
-          AND r2.relationship_id = 'Maps to'
-       WHERE a.concept_code_1 IN (SELECT icd_code FROM miss_map)),
--- concepts which mapping can be replaced through 'Concept replaces' relationship
-t2 AS (SELECT d.concept_code AS icd_code,
-              d.concept_name AS icd_name,
-              d.domain_id AS icd_domain,
-              j.concept_id AS repl_by_id,
-              j.concept_code AS repl_by_code,
-              j.concept_name AS repl_by_name,
-              j.domain_id AS repl_by_domain,
-              j.vocabulary_id AS repl_by_vocabulary,
-              NULL
-       FROM concept_relationship_stage a
-         JOIN concept b
-           ON a.concept_code_2 = b.concept_code
-          AND b.vocabulary_id = a.vocabulary_id_2
-          AND a.relationship_id = 'Maps to'
-          AND b.invalid_reason IN ('D', 'U')
-        JOIN concept_stage d
-    ON a.concept_code_1 = d.concept_code
-         JOIN concept_relationship r2 ON b.concept_id = r2.concept_id_1
-         JOIN concept j
-           ON j.concept_id = r2.concept_id_2
-          AND j.vocabulary_id = 'SNOMED' -- place for target vocabulary
-          AND j.standard_concept = 'S'
-          AND r2.relationship_id = 'Concept replaced by'
-       WHERE a.concept_code_1 IN (SELECT icd_code FROM miss_map)),
--- all concepts which can be remapped autimatically (however, should be reviewed)
-t3 AS (SELECT * FROM t1 UNION SELECT * FROM t2),
--- look-up table with all concepts with deprecated mapping + automatically remapped
-t4 AS (SELECT miss_map.icd_code,
-              miss_map.icd_name,
-              miss_map.current_relationship,
-              miss_map.current_id,
-              miss_map.current_code,
-              miss_map.current_name,
-              miss_map.current_domain,
-              miss_map.current_vocabulary,
-              t3.repl_by_id,
-              t3.repl_by_code,
-              t3.repl_by_name,
-              t3.repl_by_domain,
-              t3.repl_by_vocabulary,
-              miss_map.reason
-       FROM miss_map
-         LEFT JOIN t3 ON miss_map.icd_code = t3.icd_code
-      UNION
-         SELECT * FROM miss_map_brother),
--- improve_map - automatically detected mapping improvements. Look carefully! Target vocabulary could have the same names of concepts with different domain_ids. Also, ICD10 chapter means a lot and should be taken into account for choosing appropriate mapping
-improve_map
-AS
-(SELECT DISTINCT
-a.concept_code AS icd_code,
-       a.concept_name AS icd_name,
-       r.relationship_id AS current_relationship,
-       d.concept_id AS current_id,
-       d.concept_code AS current_code,
-       d.concept_name AS current_name,
-       d.domain_id AS current_domain,
-       d.vocabulary_id AS current_vocabulary,
-       c.concept_id AS repl_by_id,
-       code AS repl_by_code,
-       str AS repl_by_name,
-       c.domain_id AS repl_by_domain,
-       c.vocabulary_id AS repl_by_vocabulary,
-       'improve_map' AS reason
-FROM concept a
-JOIN concept_relationship r ON r.concept_id_1 = a.concept_id AND a.vocabulary_id = 'CIM10'
-JOIN concept d ON d.concept_id = r.concept_id_2 AND r.invalid_reason IS NULL AND d.standard_concept = 'S' AND r.relationship_id IN ('Maps to', 'Maps to value')
-  JOIN sources.mrconso
-    ON lower (a.concept_name) = lower (str)
-   AND sab = 'SNOMEDCT_US'
-   AND suppress = 'N'
-   AND tty = 'PT'
-  JOIN concept c
-    ON c.concept_code = code
-   AND c.vocabulary_id = 'SNOMED'
-   AND c.standard_concept = 'S'
-   AND c.concept_class_id IN ('Procedure', 'Context-dependent', 'Clinical Finding', 'Event', 'Social Context', 'Observable Entity')),
-t5 as (
-SELECT  * FROM improve_map
-WHERE icd_code IN (SELECT icd_code
-                   FROM improve_map
-                   WHERE repl_by_code != current_code)
-AND icd_code NOT IN (SELECT icd_code FROM improve_map WHERE icd_name ~ '\s+and\s+' GROUP BY icd_code HAVING COUNT(icd_code)=1)
-                   ),
-t6 AS (
-SELECT DISTINCT
-a.concept_code AS icd_code,
-       a.concept_name AS icd_name,
-       r.relationship_id AS current_relationship,
-       d.concept_id AS current_id,
-       d.concept_code AS current_code,
-       d.concept_name AS current_name,
-       d.domain_id AS current_domain,
-       d.vocabulary_id AS current_vocabulary,
-       c.concept_id AS repl_by_id,
-       c.concept_code AS repl_by_code,
-       c.concept_name AS repl_by_name,
-       c.domain_id AS repl_by_domain,
-       c.vocabulary_id AS repl_by_vocabulary,
-       'improve_map' AS reason
-FROM concept a
-JOIN concept_relationship r ON r.concept_id_1 = a.concept_id and a.vocabulary_id = 'CIM10'
-JOIN concept d ON d.concept_id = r.concept_id_2 AND r.invalid_reason IS NULL AND d.standard_concept = 'S' AND r.relationship_id IN ('Maps to', 'Maps to value')
-  JOIN concept_synonym cs ON lower (a.concept_name) = lower (cs.concept_synonym_name) AND a.vocabulary_id = 'CIM10'
-  JOIN concept c
-    ON cs.concept_id = c.concept_id
-   AND c.vocabulary_id = 'SNOMED'
-   AND c.standard_concept = 'S'
-   AND c.concept_class_id IN ('Procedure', 'Context-dependent', 'Clinical Finding', 'Event', 'Social Context', 'Observable Entity')
-AND c.concept_id NOT IN (SELECT descendant_concept_id FROM devv5.concept_ancestor WHERE ancestor_concept_id = 40485423 )), -- concept Unilateral clinical finding has weak hierarchy
-p_map AS (
-    SELECT * FROM t5
-  UNION
---exclude the cases 1 to 1 mapping with the current_id = repl_by_id, if there's multiple mapping and current_id = repl_by_id, the additional mapping serves as a hiearchy connector, so these are included into the comparison
-    SELECT * FROM t6 WHERE icd_code NOT IN (SELECT icd_code FROM (
-SELECT *, COUNT(1) over (partition BY icd_code) AS cnt FROM t6) a WHERE a.cnt =1 AND current_id = repl_by_id))
-SELECT * FROM p_map
-UNION
-SELECT * FROM t4
-ORDER BY icd_code;
+and cs.concept_class_id NOT IN ('ICD10 Chapter','ICD10 SubChapter', 'ICD10 Hierarchy');

ICD10/manual_work/crm_changes.sql

@@ -108,5 +108,4 @@ INSERT INTO concept_relationship_manual(concept_code_1, concept_code_2, vocabula
                        vocabulary_id_2,
                        relationship_id FROM concept_relationship_manual)
     )
-;
-
+;

ICD10CM/manual_work/crm_changes.sql

@@ -108,4 +108,4 @@ INSERT INTO concept_relationship_manual(concept_code_1, concept_code_2, vocabula
                        vocabulary_id_2,
                        relationship_id FROM concept_relationship_manual)
     )
-;
+;

ICD10CM/manual_work/mapping_refresh.sql

@@ -16,6 +16,9 @@
 * Authors: Irina Zherko, Dmitry Dymshyts, Polina Talapova, Daryna Ivakhnenko
 * Date: 2024
 **************************************************************************/
+--icd10cm_refresh 02_2024 backup
+CREATE TABLE icd10cm_refresh_02_2024_backup as SELECT * FROM icd10cm_refresh;
+CREATE TABLE icd10cm_refresh_07_2024_backup as SELECT * FROM icd10cm_refresh;
 
 -- Create table icd10cm_refresh
 DROP TABLE icd10cm_refresh;
@@ -121,3 +124,5 @@ and crs.relationship_id in ('Maps to', 'Maps to value')
 WHERE crs.concept_code_2 is null
 and cs.invalid_reason is null
 and cs.concept_class_id NOT IN ('ICD10 Chapter','ICD10 SubChapter', 'ICD10 Hierarchy');
+
+SELECT * FROM icd10cm_refresh;

ICD10CN/manual_work/crm_changes.sql

@@ -17,6 +17,8 @@
 * Date: 2021
 **************************************************************************/
 --1. Update the concept_relationship_manual table
+CREATE TABLE concept_relationship_manual_bu as (SELECT * FROM concept_relationship_manual);
+INSERT INTO concept_relationship_manual (SELECT * FROM concept_relationship_manual_bu);
 TRUNCATE TABLE dev_ICD10CN.concept_relationship_manual;
 INSERT INTO concept_relationship_manual (concept_code_1, concept_code_2, vocabulary_id_1, vocabulary_id_2, relationship_id, valid_start_date, valid_end_date, invalid_reason)
 SELECT DISTINCT
@@ -108,6 +110,4 @@ INSERT INTO concept_relationship_manual(concept_code_1, concept_code_2, vocabula
                        vocabulary_id_2,
                        relationship_id FROM concept_relationship_manual)
     )
-;
-
-
+;