Skip to content

Commit

Permalink
Add backend filtering on conditions (#2938)
Browse files Browse the repository at this point in the history 
* first pass, add condition filtering

* update where statement in getTotalEcrCount to fix pag and update/add to related tests

* fix EcrTable test

* add tests for generateFilterConditionsStatement and generateWhereStatement

* clean log statements, placeholders

* fix getTotalEcrCount so it counts ecrs, not ecr and unique rr conditions

* add subquery so that library table still displays all reportable conditions, not just the one filtered on
  • Loading branch information
@angelathe
angelathe authored Nov 21, 2024
1 parent f91839d commit 25613dc
Show file tree
Hide file tree
Showing 5 changed files with 127 additions and 15 deletions.
58 changes: 54 additions & 4 deletions containers/ecr-viewer/src/app/api/services/listEcrDataService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export type EcrDisplay = {
* @param sortColumn - The column to sort by
* @param sortDirection - The direction to sort by
* @param searchTerm - The search term to use
* @param filterConditions - The condition(s) to filter on
* @returns A promise resolving to a list of eCR metadata
*/
export async function listEcrData(
Expand All @@ -44,11 +45,12 @@ export async function listEcrData(
sortColumn: string,
sortDirection: string,
searchTerm?: string,
filterConditions?: string[],
): Promise<EcrDisplay[]> {
const list = await database.manyOrNone<EcrMetadataModel>(
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id where $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id WHERE $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
{
whereClause: generateSearchStatement(searchTerm),
whereClause: generateWhereStatement(searchTerm, filterConditions),
startIndex,
itemsPerPage,
sortStatement: generateSortStatement(sortColumn, sortDirection),
Expand Down Expand Up @@ -93,18 +95,40 @@ export const processMetadata = (
/**
* Retrieves the total number of eCRs stored in the ecr_data table.
* @param searchTerm - The search term used to filter the count query
* @param filterConditions - The array of reportable conditions used to filter the count query
* @returns A promise resolving to the total number of eCRs.
*/
export const getTotalEcrCount = async (
searchTerm?: string,
filterConditions?: string[],
): Promise<number> => {
let number = await database.one(
"SELECT count(*) FROM ecr_data as ed WHERE $[whereClause]",
{ whereClause: generateSearchStatement(searchTerm) },
"SELECT count(DISTINCT ed.eICR_ID) FROM ecr_data as ed LEFT JOIN ecr_rr_conditions erc on ed.eICR_ID = erc.eICR_ID WHERE $[whereClause]",
{ whereClause: generateWhereStatement(searchTerm, filterConditions) },
);
return number.count;
};

/**
* A custom type format for where statement
* @param searchTerm - Optional search term used to filter
* @param filterConditions - Optional array of reportable conditions used to filter
* @returns custom type format object for use by pg-promise
*/
export const generateWhereStatement = (
searchTerm?: string,
filterConditions?: string[],
) => ({
rawType: true,
toPostgres: () => {
const statementSearch = generateSearchStatement(searchTerm).toPostgres();
const statementConditions =
generateFilterConditionsStatement(filterConditions).toPostgres();

return `(${statementSearch}) AND (${statementConditions})`;
},
});

/**
* A custom type format for search statement
* @param searchTerm - Optional search term used to filter
Expand All @@ -128,6 +152,32 @@ export const generateSearchStatement = (searchTerm?: string) => ({
},
});

/**
* A custom type format for statement filtering conditions
* @param filterConditions - Optional array of reportable conditions used to filter
* @returns custom type format object for use by pg-promise
*/
export const generateFilterConditionsStatement = (
filterConditions?: string[],
) => ({
rawType: true,
toPostgres: () => {
if (!filterConditions) {
return pgPromise.as.format("NULL IS NULL");
}

const whereStatement = filterConditions
.map((condition) => {
return pgPromise.as.format("erc_sub.condition ILIKE $[condition]", {
condition: `%${condition}%`,
});
})
.join(" OR ");
const subQuery = `SELECT DISTINCT ed_sub.eICR_ID FROM ecr_data ed_sub LEFT JOIN ecr_rr_conditions erc_sub ON ed_sub.eICR_ID = erc_sub.eICR_ID WHERE erc_sub.condition IS NOT NULL AND (${whereStatement})`;
return `ed.eICR_ID IN (${subQuery})`;
},
});

/**
* A custom type format for sort statement
* @param columnName - The column to sort by
Expand Down
73 changes: 63 additions & 10 deletions containers/ecr-viewer/src/app/api/tests/listEcrDataService.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import {
EcrMetadataModel,
getTotalEcrCount,
generateSearchStatement,
generateFilterConditionsStatement,
generateWhereStatement,
} from "@/app/api/services/listEcrDataService";
import { database } from "../services/db";
import {
Expand Down Expand Up @@ -111,7 +113,7 @@ describe("listEcrDataService", () => {
);
expect(database.manyOrNone).toHaveBeenCalledOnce();
expect(database.manyOrNone).toHaveBeenCalledWith(
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id where $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id WHERE $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
{
whereClause: expect.any(Object),
startIndex,
Expand Down Expand Up @@ -154,7 +156,7 @@ describe("listEcrDataService", () => {

expect(database.manyOrNone).toHaveBeenCalledOnce();
expect(database.manyOrNone).toHaveBeenCalledWith(
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id where $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id WHERE $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
{
whereClause: expect.any(Object),
startIndex,
Expand Down Expand Up @@ -207,7 +209,7 @@ describe("listEcrDataService", () => {
);
expect(database.manyOrNone).toHaveBeenCalledOnce();
expect(database.manyOrNone).toHaveBeenCalledWith(
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id where $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id WHERE $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
{
whereClause: expect.any(Object),
startIndex,
Expand Down Expand Up @@ -245,7 +247,7 @@ describe("listEcrDataService", () => {
);
expect(database.manyOrNone).toHaveBeenCalledOnce();
expect(database.manyOrNone).toHaveBeenCalledWith(
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id where $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id WHERE $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
{
whereClause: expect.any(Object),
startIndex,
Expand All @@ -271,7 +273,7 @@ describe("listEcrDataService", () => {
);
expect(database.manyOrNone).toHaveBeenCalledOnce();
expect(database.manyOrNone).toHaveBeenCalledWith(
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id where $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
"SELECT ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date, ed.report_date, ARRAY_AGG(DISTINCT erc.condition) AS conditions, ARRAY_AGG(DISTINCT ers.rule_summary) AS rule_summaries FROM ecr_data ed LEFT JOIN ecr_rr_conditions erc ON ed.eICR_ID = erc.eICR_ID LEFT JOIN ecr_rr_rule_summaries ers ON erc.uuid = ers.ecr_rr_conditions_id WHERE $[whereClause] GROUP BY ed.eICR_ID, ed.patient_name_first, ed.patient_name_last, ed.patient_birth_date, ed.date_created, ed.report_date $[sortStatement] OFFSET $[startIndex] ROWS FETCH NEXT $[itemsPerPage] ROWS ONLY",
{
whereClause: expect.any(Object),
startIndex,
Expand All @@ -290,18 +292,18 @@ describe("listEcrDataService", () => {
await getTotalEcrCount();
expect(database.one).toHaveBeenCalledOnce();
expect(database.one).toHaveBeenCalledWith(
"SELECT count(*) FROM ecr_data as ed WHERE $[whereClause]",
"SELECT count(DISTINCT ed.eICR_ID) FROM ecr_data as ed LEFT JOIN ecr_rr_conditions erc on ed.eICR_ID = erc.eICR_ID WHERE $[whereClause]",
{ whereClause: expect.any(Object) },
);
});
it("should use search term in count query", async () => {
database.one<{ count: number }> = jest.fn(() =>
Promise.resolve({ count: 0 }),
);
await getTotalEcrCount("blah");
await getTotalEcrCount("blah", undefined);
expect(database.one).toHaveBeenCalledOnce();
expect(database.one).toHaveBeenCalledWith(
"SELECT count(*) FROM ecr_data as ed WHERE $[whereClause]",
"SELECT count(DISTINCT ed.eICR_ID) FROM ecr_data as ed LEFT JOIN ecr_rr_conditions erc on ed.eICR_ID = erc.eICR_ID WHERE $[whereClause]",
{
whereClause: expect.any(Object),
},
Expand All @@ -311,10 +313,24 @@ describe("listEcrDataService", () => {
database.one<{ count: number }> = jest.fn(() =>
Promise.resolve({ count: 0 }),
);
await getTotalEcrCount("O'Riley");
await getTotalEcrCount("O'Riley", undefined);
expect(database.one).toHaveBeenCalledOnce();
expect(database.one).toHaveBeenCalledWith(
"SELECT count(*) FROM ecr_data as ed WHERE $[whereClause]",
"SELECT count(DISTINCT ed.eICR_ID) FROM ecr_data as ed LEFT JOIN ecr_rr_conditions erc on ed.eICR_ID = erc.eICR_ID WHERE $[whereClause]",
{
whereClause: expect.any(Object),
},
);
});
it("should use filter conditions in count query", async () => {
database.one<{ count: number }> = jest.fn(() =>
Promise.resolve({ count: 0 }),
);
await getTotalEcrCount("", ["Anthrax (disorder)"]);
expect(database.one).toHaveBeenCalledOnce();

expect(database.one).toHaveBeenCalledWith(
"SELECT count(DISTINCT ed.eICR_ID) FROM ecr_data as ed LEFT JOIN ecr_rr_conditions erc on ed.eICR_ID = erc.eICR_ID WHERE $[whereClause]",
{
whereClause: expect.any(Object),
},
Expand All @@ -339,4 +355,41 @@ describe("listEcrDataService", () => {
);
});
});

describe("generate filter conditions statement", () => {
it("should add conditions in the filter statement", () => {
expect(
generateFilterConditionsStatement(["Anthrax (disorder)"]).toPostgres(),
).toEqual(
"ed.eICR_ID IN (SELECT DISTINCT ed_sub.eICR_ID FROM ecr_data ed_sub LEFT JOIN ecr_rr_conditions erc_sub ON ed_sub.eICR_ID = erc_sub.eICR_ID WHERE erc_sub.condition IS NOT NULL AND (erc_sub.condition ILIKE '%Anthrax (disorder)%'))",
);
});
it("should only generate true statements when no conditions are provided to filter on", () => {
expect(generateFilterConditionsStatement(undefined).toPostgres()).toEqual(
"NULL IS NULL",
);
});
});

describe("generate where statement", () => {
it("should generate where statement using search and filter conditions statements", () => {
expect(
generateWhereStatement("blah", ["Anthrax (disorder)"]).toPostgres(),
).toEqual(
"(ed.patient_name_first ILIKE '%blah%' OR ed.patient_name_last ILIKE '%blah%') AND (ed.eICR_ID IN (SELECT DISTINCT ed_sub.eICR_ID FROM ecr_data ed_sub LEFT JOIN ecr_rr_conditions erc_sub ON ed_sub.eICR_ID = erc_sub.eICR_ID WHERE erc_sub.condition IS NOT NULL AND (erc_sub.condition ILIKE '%Anthrax (disorder)%')))",
);
});
it("should generate where statement using search statement (no filter conditions provided", () => {
expect(generateWhereStatement("blah", undefined).toPostgres()).toEqual(
"(ed.patient_name_first ILIKE '%blah%' OR ed.patient_name_last ILIKE '%blah%') AND (NULL IS NULL)",
);
});
it("should generate where statement using filter conditions statement (no search provided)", () => {
expect(
generateWhereStatement("", ["Anthrax (disorder)"]).toPostgres(),
).toEqual(
"(NULL IS NULL OR NULL IS NULL) AND (ed.eICR_ID IN (SELECT DISTINCT ed_sub.eICR_ID FROM ecr_data ed_sub LEFT JOIN ecr_rr_conditions erc_sub ON ed_sub.eICR_ID = erc_sub.eICR_ID WHERE erc_sub.condition IS NOT NULL AND (erc_sub.condition ILIKE '%Anthrax (disorder)%')))",
);
});
});
});
4 changes: 4 additions & 0 deletions containers/ecr-viewer/src/app/components/EcrTable.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { EcrTableClient } from "@/app/components/EcrTableClient";
* @param props.sortColumn - The column to sort by
* @param props.sortDirection - The direction to sort by
* @param props.searchTerm - The search term used to list data
* @param props.filterConditions - (Optional) The reportable condition(s) used to filter the data
* @returns - eCR Table element
*/
const EcrTable = async ({
Expand All @@ -19,12 +20,14 @@ const EcrTable = async ({
sortColumn,
sortDirection,
searchTerm,
filterConditions,
}: {
currentPage: number;
itemsPerPage: number;
sortColumn: string;
sortDirection: string;
searchTerm?: string;
filterConditions?: string[];
}) => {
const startIndex = (currentPage - 1) * itemsPerPage;

Expand All @@ -34,6 +37,7 @@ const EcrTable = async ({
sortColumn,
sortDirection,
searchTerm,
filterConditions,
);

return (
Expand Down
5 changes: 4 additions & 1 deletion containers/ecr-viewer/src/app/page.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ const HomePage = async ({
const sortColumn = (searchParams?.columnId as string) || "date_created";
const sortDirection = (searchParams?.direction as string) || "DESC";
const searchTerm = searchParams?.search as string | undefined;
// Placeholder for given array of conditions to filter on, remove in #2751
const filterConditions = undefined; // Ex. ["Anthrax (disorder)", "Measles (disorder)"];

const isNonIntegratedViewer =
process.env.NEXT_PUBLIC_NON_INTEGRATED_VIEWER === "true";
let totalCount: number = 0;
if (isNonIntegratedViewer) {
totalCount = await getTotalEcrCount(searchTerm);
totalCount = await getTotalEcrCount(searchTerm, filterConditions);
}

return isNonIntegratedViewer ? (
Expand All @@ -48,6 +50,7 @@ const HomePage = async ({
sortColumn={sortColumn}
sortDirection={sortDirection}
searchTerm={searchTerm}
filterConditions={filterConditions}
/>
</EcrPaginationWrapper>
</main>
Expand Down
2 changes: 2 additions & 0 deletions containers/ecr-viewer/src/app/tests/components/EcrTable.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ describe("EcrTable", () => {
sortColumn: "date_created",
sortDirection: "DESC",
searchTerm: "blah",
filterConditions: ["Anthrax (disorder)"],
}),
);

Expand All @@ -82,6 +83,7 @@ describe("EcrTable", () => {
"date_created",
"DESC",
"blah",
["Anthrax (disorder)"],
);
});
});

0 comments on commit 25613dc

Please sign in to comment.