Skip to content

Commit

Permalink
Merge pull request #210 from alan-francis/main
Browse files Browse the repository at this point in the history 
Update to version 0.44.0
  • Loading branch information
@ratheesh-kr
ratheesh-kr authored Jul 23, 2024
2 parents a2fd820 + 70d0b94 commit 01936ea
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 17 deletions.
8 changes: 4 additions & 4 deletions src/ahc-hrsn-elt/screening/csv.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -672,10 +672,10 @@ export class AdminDemographicCsvFileIngestSource<
)}
${tr.mandatoryValueInAllRows("FIRST_NAME")}
${tr.onlyAllowAlphabetsInAllRows("FIRST_NAME")}
${tr.onlyAllowAlphabetsInAllRows("MIDDLE_NAME")}
${adar.car.onlyAllowAlphabetsInAllRows("FIRST_NAME")}
${adar.car.onlyAllowAlphabetsInAllRows("MIDDLE_NAME")}
${tr.mandatoryValueInAllRows("LAST_NAME")}
${tr.onlyAllowAlphabetsInAllRows("LAST_NAME")}
${adar.car.onlyAllowAlphabetsInAllRows("LAST_NAME")}
${tr.mandatoryValueInAllRows("ADMINISTRATIVE_SEX_CODE")}
${tr.mandatoryValueInAllRows("ADMINISTRATIVE_SEX_CODE_SYSTEM")}
${adar.onlyAllowValidAdministrativeSexCodeInAllRows("ADMINISTRATIVE_SEX_CODE")}
Expand All @@ -689,7 +689,7 @@ export class AdminDemographicCsvFileIngestSource<
${adar.car.onlyAllowValidFieldCombinationsInAllRows("SEX_AT_BIRTH_CODE","SEX_AT_BIRTH_CODE_DESCRIPTION")}
${adar.car.onlyAllowValidFieldCombinationsInAllRows("SEX_AT_BIRTH_CODE_DESCRIPTION","SEX_AT_BIRTH_CODE")}
${tr.mandatoryValueInAllRows("PAT_BIRTH_DATE")}
${tr.onlyAllowValidDateTimeInAllRows("PAT_BIRTH_DATE")}
${adar.onlyAllowValidPatBirthDateDateInAllTableRows("PAT_BIRTH_DATE")}
${tr.mandatoryValueInAllRows("CITY")}
${tr.mandatoryValueInAllRows("STATE")}
${tr.onlyAllowedValuesInAllRows("STATE", "'NY', 'ny', 'New York','new york', 'NEW YORK'")}
Expand Down
96 changes: 87 additions & 9 deletions src/ahc-hrsn-elt/screening/governance.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { SQLa_orch_duckdb as ddbo } from "./deps.ts";

const escapedSqlLiteral = (literal: string) => literal.replaceAll("'", "''");

export class GroupCsvStructureRules<
TableName extends string,
ColumnName extends string,
Expand Down Expand Up @@ -39,6 +41,7 @@ export class GroupCsvStructureRules<
`NULL`,
"table_name",
`'CSV file ' || table_name_suffix || '${groupName} not found under the group (${groupName})'`,
`'Please make sure there are 3 files within the group ${groupName}'`,
)
}
`;
Expand All @@ -53,35 +56,83 @@ export class CommonAssuranceRules<
ColumnName extends string,
> extends ddbo.DuckDbOrchTableAssuranceRules<TableName, ColumnName> {
// if there are any custom business logic rules put them here and if they can
// be further generalized we can move them into the upstream SQLa library
// be further geenralized we can move them into the upstream SQLa library
matchPatternWithRemediation(
columnName: ColumnName,
pattern: string,
remediation: string,
) {
const cteName = "pattern";
const escapedHumanMsgFragment = escapedSqlLiteral(pattern);
const patternSql =
`CAST("${columnName}" AS VARCHAR) NOT SIMILAR TO '${pattern}'`;
return this.govn.SQL`
WITH ${cteName} AS (
SELECT '${columnName}' AS issue_column,
"${columnName}" AS invalid_value,
src_file_row_number AS issue_row
FROM "${this.tableName}"
WHERE ${patternSql}
)
${
this.insertRowValueIssueCtePartial(
cteName,
"Pattern Mismatch",
"issue_row",
"issue_column",
"invalid_value",
`'Value ' || invalid_value || ' in ' || issue_column || ' does not match the pattern ${escapedHumanMsgFragment}'`,
`'${remediation}'`,
)
}`;
}

onlyAllowAlphabetsInAllRows(columnName: ColumnName) {
return this.matchPatternWithRemediation(
columnName,
"^[A-Za-z]+$",
columnName + "should be alphabetical",
);
}

// any rules defined here will be available as car.rule() in the
onlyAllowValidZipInAllRows(columnName: ColumnName) {
return this.tableRules.patternValueInAllRows(
return this.matchPatternWithRemediation(
columnName,
"^\\d{5}(\\d{4})?$",
"Zip code should be having 5 or 9 digits which are all numbers",
);
}

onlyAllowAlphabetsAndNumbersInAllRows(columnName: ColumnName) {
return this.tableRules.patternValueInAllRows(columnName, "^[a-zA-Z0-9]+$");
return this.matchPatternWithRemediation(
columnName,
"^[a-zA-Z0-9]+$",
columnName + " should contain only alphabets and numbers",
);
}

onlyAllowAlphabetsWithSpacesInAllRows(columnName: ColumnName) {
return this.tableRules.patternValueInAllRows(columnName, "^[a-zA-Z\\s]+$");
return this.matchPatternWithRemediation(
columnName,
"^[a-zA-Z\\s]+$",
columnName + " should contain only alphabets with spaces",
);
}

onlyAllowAlphabetsAndNumbersWithSpaceInAllRows(columnName: ColumnName) {
return this.tableRules.patternValueInAllRows(
return this.matchPatternWithRemediation(
columnName,
"^[a-zA-Z0-9\\s]+$",
columnName + " should contain only alphabets and numbers with spaces",
);
}

onlyAllowValidMedicaidCinFormatInAllRows(columnName: ColumnName) {
return this.tableRules.patternValueInAllRows(
return this.matchPatternWithRemediation(
columnName,
"^[A-Za-z]{2}\\d{5}[A-Za-z]$",
columnName + " should follow the format 2 letters, 5 numbers, 1 letter",
);
}

Expand All @@ -107,7 +158,7 @@ export class CommonAssuranceRules<
"issue_column",
"invalid_value",
`'Invalid value "' || invalid_value || '" found in ' || issue_column`,
`'Invalid string of numbers found'`,
`'Make sure the column ' || issue_column || 'has not only numbers.'`,
)
}`;
}
Expand Down Expand Up @@ -1204,7 +1255,7 @@ export class ScreeningAssuranceRules<
"issue_column",
"invalid_value",
`'Invalid timestamp "' || invalid_value || '" found in ' || issue_column`,
`'Please be sure to provide both a valid date and time.'`,
`'Please follow the format YYYY-MM-DDTHH:MM:SS.000Z.'`,
)}`;
}

Expand Down Expand Up @@ -1262,7 +1313,7 @@ export class ScreeningAssuranceRules<
"issue_column",
"invalid_pat_value",
`'${patMrnIdcolumnName} ("' || invalid_pat_value || '") that does not match the ${facilityIdcolumnName} ("' || invalid_facility_value || '") across the files was found in "' || issue_table_name || '".'`,
`'Validate ${patMrnIdcolumnName} that maches the ${facilityIdcolumnName} across the files'`,
`'Validate ${patMrnIdcolumnName} that matches the ${facilityIdcolumnName} across the files'`,
)
}`;
}
Expand Down Expand Up @@ -2048,6 +2099,33 @@ export class AdminDemographicAssuranceRules<
}`;
}
// if there are any admin-demographic-specific business logic rules put them here;

onlyAllowValidPatBirthDateDateInAllTableRows(
columnName: ColumnName,
) {
// Construct the SQL query using tagged template literals
const cteName = "valid_date_in_all_rows";
// deno-fmt-ignore
return this.govn.SQL`
WITH ${cteName} AS (
SELECT '${columnName}' AS issue_column,
"${columnName}" AS invalid_value,
src_file_row_number AS issue_row
FROM "${this.tableName}"
WHERE "${columnName}" IS NOT NULL
AND (TRY_CAST("${columnName}" AS DATE) IS NULL
OR "${columnName}"::TEXT ~ '^\d{4}-\\d{2}-\\d{2}$')
)
${this.insertRowValueIssueCtePartial(
cteName,
"Invalid Date",
"issue_row",
"issue_column",
"invalid_value",
`'Invalid date "' || invalid_value || '" found in ' || issue_column`,
`'Please follow the format YYYY-MM-DD for a valid ${columnName} .'`,
)}`;
}
}

/**
Expand Down
8 changes: 4 additions & 4 deletions src/ahc-hrsn-elt/screening/orchestrate.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import * as csv from "./csv.ts";
import * as excel from "./excel.ts";
import * as gov from "./governance.ts";

export const ORCHESTRATE_VERSION = "0.43.0";
export const ORCHESTRATE_VERSION = "0.44.0";

export interface FhirRecord {
PAT_MRN_ID: string;
Expand Down Expand Up @@ -1209,7 +1209,7 @@ export class OrchEngine {
),
'system', CONCAT('${baseUrl}','facility/',adt.FACILITY_ID),
'value', qat.PAT_MRN_ID,
'assigner', json_object('reference', 'Organization/' || qat.FACILITY_ID)
'assigner', json_object('reference', 'Organization/' || sha256(qat.FACILITY_ID))
),
CASE
WHEN MEDICAID_CIN != '' THEN
Expand Down Expand Up @@ -1310,7 +1310,7 @@ export class OrchEngine {
'reference', CONCAT('Patient/',scr.FACILITY_ID,'-',scr.PAT_MRN_ID)
),
'dateTime',(SELECT MAX(scr.RECORDED_TIME) FROM screening scr WHERE adt.FACILITY_ID = scr.FACILITY_ID),
'organization', json_array(json_object('reference', 'Organization/' || qat.FACILITY_ID))
'organization', json_array(json_object('reference', 'Organization/' || sha256(qat.FACILITY_ID)))
)
Expand Down Expand Up @@ -1488,8 +1488,8 @@ export class OrchEngine {
'coding', json_array(json_object(CASE WHEN SCREENING_CODE_SYSTEM_NAME IS NOT NULL THEN 'system' ELSE NULL END,SCREENING_CODE_SYSTEM_NAME,CASE WHEN scr.SCREENING_CODE IS NOT NULL THEN 'code' ELSE NULL END,scr.SCREENING_CODE,CASE WHEN SCREENING_CODE_DESCRIPTION IS NOT NULL THEN 'display' ELSE NULL END,SCREENING_CODE_DESCRIPTION))
),
'subject', json_object('reference',CONCAT('Patient/',scr.FACILITY_ID,'-',scr.PAT_MRN_ID)),
CASE WHEN ENCOUNTER_ID IS NOT NULL THEN 'encounter' ELSE NULL END, json_object('reference',CONCAT('Encounter/',ENCOUNTER_ID)),
'effectiveDateTime', MAX(RECORDED_TIME),
'encounter', CASE WHEN ENCOUNTER_ID IS NOT NULL THEN json_object('reference',CONCAT('Encounter/',sha256(ENCOUNTER_ID))) ELSE json_object('reference',CONCAT('Encounter/',sha256(CONCAT('encounter-',scr.FACILITY_ID,'-',scr.PAT_MRN_ID)))) END,
'issued', MAX(RECORDED_TIME),
'hasMember', (SELECT json_group_array(JSON_OBJECT(
'reference', CASE WHEN sub1.ENCOUNTER_ID IS NOT NULL THEN CONCAT('Observation/',sha256(CONCAT('observationResponseQuestion-',sub1.ENCOUNTER_ID,'-',md5(sub1.RECORDED_TIME),'-',sub1.QUESTION_SLNO))) ELSE CONCAT('Observation/',sha256(CONCAT('observationResponseQuestion-',sub1.PAT_MRN_ID,'-',sub1.FACILITY_ID,'-',md5(sub1.RECORDED_TIME),'-',sub1.QUESTION_SLNO))) END
Expand Down

0 comments on commit 01936ea

Please sign in to comment.