diff --git a/deno.lock b/deno.lock index 9326dc0a..ed47509b 100644 --- a/deno.lock +++ b/deno.lock @@ -1468,6 +1468,77 @@ "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/render/graph.ts": "d8de5eae4c60322156d43a0c587b5cb8fc8f8b608093cfd4f380902f4049dec1", "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/render/mod.ts": "a3389f40211e752d77c9877a7b55477dafdb831020c227bcb71a89a1ffb73e01", "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/render/quality-system.ts": "7ec5b0c30eb799d18fa75627de93de490de96166652a298da0ccf648cf09a79f", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/deps.ts": "4827a3829b1c4ab324560d7e6a46549b3b0c8278562e67b91857a95ead722dbc", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/duckdb/mod.ts": "caf953151427f9cc8667a821c2704a820db19dde3ec41ea84844728c3ffc9497", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/duckdb/shell.ts": "a3e17459cb009552e2352f93acfa057a1c9128ee8774a6cff1ee985699261ae1", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/notebook/chain-of-responsibility.ts": "0b9d38cb753edbe0bac538c210f62985a56da46c591fdf306e6016cd54a3dd64", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/notebook/command.ts": "0c2df89b19ef0d60fb4e65257453f1b79ef8640692b4c73f827bf2b38540a938", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/notebook/deps.ts": "a746118f276b968d4c808478ba3d1bbd51f989f28f0e358d7462e0c7771f3407", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/pre-process/psql/include.ts": "df0ae6d18072dd1fc00a35ee5232475eceac28f85e77fb455fb1a9a2e4218fca", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/pre-process/psql/meta-command.ts": "50801350208a17fcb52527636ce13e7717e1882b1f1d108631d73bcba4336d69", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/pre-process/psql/mod.ts": "24d392a2d35691f175a8b2120de8c6ffb0a3d0dadad8768d71ef515a2261a125", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/pre-process/psql/set.ts": "4b1292268db8b7d74947919842cc54461f69eb1cfbbe8faae05a1b300ef3a744", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/quality-system/governance.ts": "899f9129b65790a49a4caa0d0cd38bddd69a7d519ba2f150c095e4e95f041d92", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/quality-system/mod.ts": "4485719eb6bae9e9163fdbd3af0e1c28185775aa10f2300e042651a998a4aeb4", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/array.ts": "d657bd5c0d99a4f666d740653721f7fbee754ce63b6f0177387c0cf84b00303c", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/flexible-text.ts": "21fb1297fed736b4d80ae6ef9b48acdbc9cfec9575cfb7d9e3aedc858820fe56", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/graph.ts": "091bebe26e9810994d791702fedd6df0caeb7fe3ad8232f4b2d71eec52c297b1", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/record.ts": "403be181c0ffc473a131dd606b544a380b1d7f792c85fd57657ea6a557d8bc6e", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/safety.ts": "6a221abc2c355287d354d5d9e3e5652aab7857dba72224fc13588726d4467697", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/sys-info.ts": "98622658e654865fbe1a86bfc77136b4de99041c1fe41fbab833227f93dd42ce", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/whitespace.ts": "303e037f9cf984848933002b6b1ccb5218f030ab6b4661dcc69352733f4f06ff", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/zod-aide.ts": "d1e389936d3b77581ce655aeb5fee21436ed62161a5480bd42bac7e5b717c1ae", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/ingest/duckdb/assurance.ts": "213b47e018cf18e73fb2a4621d8eb1ad00318f005a64957e85105891531efef4", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/ingest/duckdb/deps.ts": "8574e3f63209de00d3d7f80ed075f6fda4ab5e1da6c158b46039585d4fd2c186", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/ingest/duckdb/mod.ts": "645e2d4f72109cd0ef3ac30c833b8a0eca96d9d07e3d952ded6819a80b8f70da", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/ingest/duckdb/notebook.ts": "73d3e318b31b3dbe715274a5a1e2589f96355c816439c34a7547d6061605ed76", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/sqlpage/component.ts": "8023e4f6889266c8777ae2ca6c43d2ee7459899e5d1779daa969efc8481b7cb6", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/sqlpage/content.ts": "de07dfd4f6408e03e72ea0156a3e531bc4226cbbb4577b5d26fa9a0270eda0d1", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/sqlpage/mod.ts": "e3dbfe9b78b6acb18572c9dc6a4d4db831279d37adbbd0cacc56a0638e3fab24", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/sqlpage/notebook.ts": "6cc4e3a3f7b116e92731c1a8b75a8b79e07bb1076a46d77a5a65b97c0449a106", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/typical/enum-table.ts": "82f91bb95bdaa1c4ce2a1a6a43e9ae459fe5fd19bf868331087699bd6623bc20", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/typical/mod.ts": "8852e67a308d5a784fac094994832d0a0d16f66512b32193d9c15ba781a464e9", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/typical/typical-cli.ts": "7df709acf258215c7e63f29ec351d766eb939a9a88e4f2ac9ddd7efe913b7e73", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/typical/typical.ts": "d2bd05b9dc1c74a4c0bd19b57744246625de939d09c02dd7830cfa97acff2d05", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/mod.ts": "0ea2e3e0b9a8449a029b26633017c59b274e0c3c55f6850c161c345ea6b1bc79", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/schema.ts": "3ae39ff73d6b493768bc06752eda54ac74b6f0fd504e77211ac20697ba41fe94", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/column.ts": "f1c65bae4c4ed3591b2d41ef085400fde6ad8049e8005a0814e414b4a79e76c3", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/constraint.ts": "e51e53eb15fd34152967cda6b28c36bfae19b1835bb5847e447270b7d3297c02", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/foreign-key.ts": "5ed0a3ddfe4e264369e776beac196cc56d05fc014bc234d9f5a857d32f629be1", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/index.ts": "7399da506094d17324f4b72cefc2a30ffb5e26ef79f152429947fc83a8c2b8dc", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/lint.ts": "8d95f5d9b13791f02ab2511148b227298520ea8bd71aeda5fc8cba3b6d60284c", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/mod.ts": "eb5e07971125b3cbc9cfdfe9af0bd942bd76f305db82ce4014d4c0584910681c", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/primary-key.ts": "882112cf17549784cba8563bcb4cdd65ab5340f7234f60bd32096c19cfa7fbe5", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/record.ts": "f93ae25eddf94a06255af1b5505712e8d3cbad7c0d6c0d360950839ffebdb0e9", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/select.ts": "43d38132ea1eb92582c28abc85d21d61cea0f981cac1290a8ca8444196951995", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/self-ref.ts": "d0360e77d2d340c55fb611c17461a6e50cda2fd283a140185239d3152c8d92b7", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/table/table.ts": "9f81bf927a02667123bcb9205411546e87e17add1d3ddca7f479ea6236289d00", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/type.ts": "2bd8ed9d5b5cd412fe95a4b67a7dd2eaf3938433f02f7df9529e130214188206", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/ddl/view.ts": "0d5573b707d1292a2229a61e28c246da8bfe74f0b9d3b19e1696f87492699965", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/deps.ts": "f822685ed0ab1e5b2fc76d87c81997555320d2496399684518dc4f52850ffbdb", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/diagram/plantuml-ie-notation.ts": "e5bcfe0585887523ae81a0445abebe15f5896236e1096d6320ed1baef2724788", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/dml/insert.ts": "3b3186d47c398f03f37008a3f590222312dc74911d174cc6ec7e593428ec5b04", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/dml/mod.ts": "8a86dd5a22f2a9690d5a54526caf3d647fa0a85b52da71bed89ae7a8aff734be", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/domain/domain.ts": "2d98ecd92d939e7642e476bcdab92016831853e646021f71ee1863723db20a32", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/domain/domains.ts": "c568e5e7ec2732916ae40a794612dd6204668551dc088af8c1791866956cc7ca", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/domain/mod.ts": "976337eb03d15107cef0b6bbf9f19451a9f479d416f8eaf4842d8a780936b3b3", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/dql/criteria.ts": "9d91520e78e21ac613fadca2d0d1b00094498aecc22a9fee522a749b3d1dc3d2", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/dql/mod.ts": "cabe5089c3d3756fb64fa4165fbcd8a842aa9af5eed2f4e925deb033d3e0abba", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/dql/select.ts": "bc033a1c0e60939a539740440b2693fd64061d276997ea81c05b2e7f447cb35a", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/dialect.ts": "a54fe72979d58f97f99280dc202b5b349606182eee73434a3641cae0eb17e345", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/directive.ts": "bf2244d94b61a521d7a00b2fa52c778bc8e217fbd1dd2a7109d723b347e2baa2", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/interpolate.ts": "f7e4ddebcb9d9ea0ac5ec8ffd59432e3b04571bf6a27820a8ffe7a258f66cf2a", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/js.ts": "b3027a36f19bed035b255b61788aef1747a3c74b264ea31c9eb2ad703648bfc8", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/mod.ts": "392ad0e5affe06d1a386735d85329c9c2b252b73e7ce197c45563572358837f0", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/namespace.ts": "6e15a3fd4af6c9f17af1d94ca9f36764fa7d3306e2b8fc63bb77863edd2d6c50", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/polygenix-notebook.ts": "5bc6cfb72743ba4501331fbb13b25a073902434863cdf58e596c9833120028f9", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/polygenix.ts": "ef8b9864ebdf3cc8b8e04540d57f8b50b395b07c329aaeb952e660aa5e52d5d1", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/quality-system.ts": "d8fce4ec7eb54cab850c9ceac5be08e21d743b4457eac2ee9d5e51aad35fb989", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/sql-notebook.ts": "94f612eaebc4ae1bbaeaf40c57d9e49991e376d559e7a285bf7e3a421fc3051b", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/emit/sql.ts": "894c5437020139584e5b2b2b397fc93e414d3a66096ba5224b06ae5749f12ca3", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/graph.ts": "d8de5eae4c60322156d43a0c587b5cb8fc8f8b608093cfd4f380902f4049dec1", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/mod.ts": "a3389f40211e752d77c9877a7b55477dafdb831020c227bcb71a89a1ffb73e01", + "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/quality-system.ts": "7ec5b0c30eb799d18fa75627de93de490de96166652a298da0ccf648cf09a79f", "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.6/deps.ts": "4827a3829b1c4ab324560d7e6a46549b3b0c8278562e67b91857a95ead722dbc", "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.6/lib/notebook/chain-of-responsibility.ts": "0b9d38cb753edbe0bac538c210f62985a56da46c591fdf306e6016cd54a3dd64", "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.6/lib/notebook/command.ts": "0c2df89b19ef0d60fb4e65257453f1b79ef8640692b4c73f827bf2b38540a938", diff --git a/src/ahc-hrsn-elt/screening/csv.ts b/src/ahc-hrsn-elt/screening/csv.ts index 61813346..9af7c7a8 100644 --- a/src/ahc-hrsn-elt/screening/csv.ts +++ b/src/ahc-hrsn-elt/screening/csv.ts @@ -25,6 +25,8 @@ export class ScreeningCsvFileIngestSource return { ingestSQL: async (issac) => await this.ingestSQL(issac, sar), assuranceSQL: async () => await this.assuranceSQL(sar), + exportResourceSQL: async (targetSchema) => + await this.exportResourceSQL(targetSchema), }; } @@ -57,6 +59,12 @@ export class ScreeningCsvFileIngestSource return govn.SQL` ${sar.tableRules.intValueInAllRows('SURVEY_ID')}` } + + // deno-lint-ignore require-await + async exportResourceSQL(targetSchema: string) { + const { govn: { SQL }, tableName } = this; + return SQL`CREATE TABLE ${targetSchema}.${tableName} AS SELECT * FROM ${tableName}`; + } } export function ingestCsvFilesSourcesSupplier( diff --git a/src/ahc-hrsn-elt/screening/deps.ts b/src/ahc-hrsn-elt/screening/deps.ts index b66b8168..54859ee0 100644 --- a/src/ahc-hrsn-elt/screening/deps.ts +++ b/src/ahc-hrsn-elt/screening/deps.ts @@ -8,13 +8,13 @@ export * as dax from "https://deno.land/x/dax@0.36.0/mod.ts"; export * as safeText from "npm:string-ts"; // local: ../../../../../netspective-labs/sql-aide -// remote: https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23 +// remote: https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24 -export * as array from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/universal/array.ts"; -export * as chainNB from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/notebook/chain-of-responsibility.ts"; -export * as duckdb_shell from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/duckdb/mod.ts"; -export * as ws from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/universal/whitespace.ts"; +export * as array from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/array.ts"; +export * as chainNB from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/notebook/chain-of-responsibility.ts"; +export * as duckdb_shell from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/duckdb/mod.ts"; +export * as ws from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/lib/universal/whitespace.ts"; -export * as SQLa from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/render/mod.ts"; -export * as SQLa_ingest_duckdb from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/pattern/ingest/duckdb/mod.ts"; -export * as SQLa_sqlpage from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/pattern/sqlpage/mod.ts"; +export * as SQLa from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/render/mod.ts"; +export * as SQLa_ingest_duckdb from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/ingest/duckdb/mod.ts"; +export * as SQLa_sqlpage from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.24/pattern/sqlpage/mod.ts"; diff --git a/src/ahc-hrsn-elt/screening/excel.ts b/src/ahc-hrsn-elt/screening/excel.ts index 99bdece1..ab5a5a77 100644 --- a/src/ahc-hrsn-elt/screening/excel.ts +++ b/src/ahc-hrsn-elt/screening/excel.ts @@ -36,8 +36,13 @@ export class ExcelSheetTodoIngestSource ${await issac.sessionEntryInsertDML()} ${await issac.issueInsertDML(`Excel workbook '${path.basename(this.uri)}' sheet '${this.sheetName}' has not been implemented yet.`, "TODO")}`, + assuranceSQL: () => this.govn.SQL`-- Sheet '${this.sheetName}' ingestion not implemented.`, + + exportResourceSQL: (targetSchema: string) => + this.govn.SQL` + -- Sheet '${this.sheetName}' exportResourceSQL(${targetSchema})`, }; } } @@ -72,6 +77,8 @@ export class ScreeningExcelSheetIngestSource return { ingestSQL: async (issac) => await this.ingestSQL(issac, sar), assuranceSQL: async () => await this.assuranceSQL(), + exportResourceSQL: async (targetSchema) => + await this.exportResourceSQL(targetSchema), }; } @@ -104,6 +111,14 @@ export class ScreeningExcelSheetIngestSource // deno-fmt-ignore return this.govn.SQL`-- Sheet '${this.sheetName}' has no assurance SQL in Excel workbook '${path.basename(this.uri)}'`; } + + // deno-lint-ignore require-await + async exportResourceSQL(targetSchema: string) { + const { govn } = this; + + // deno-fmt-ignore + return govn.SQL`-- Sheet '${this.sheetName}' exportResourceSQL(${targetSchema})`; + } } export function ingestExcelSourcesSupplier( diff --git a/src/ahc-hrsn-elt/screening/ingest.ts b/src/ahc-hrsn-elt/screening/ingest.ts index 7b0fb8d7..1c2da344 100644 --- a/src/ahc-hrsn-elt/screening/ingest.ts +++ b/src/ahc-hrsn-elt/screening/ingest.ts @@ -304,9 +304,15 @@ export class IngestEngine { ); await this.duckdb.execute(assurableSQL.join("\n"), isc.current.nbCellID); + return ingestResult; } - async emitResources(isc: IngestStepContext) { + async emitResources( + isc: IngestStepContext, + ingestResult: Awaited< + ReturnType + >, + ) { const { args: { resourceDb, session }, govn: { emitCtx: ctx } } = this; if (resourceDb) { try { @@ -327,6 +333,13 @@ export class IngestEngine { const beforeFinalize = session.sqlCatalogSqlText("before-finalize"); const afterFinalize = session.sqlCatalogSqlText("after-finalize"); + const rdbSchemaName = "resource_db"; + + const exportsSQL = await Promise.all( + ingestResult.map(async (sr) => + (await sr.workflow.exportResourceSQL(rdbSchemaName)).SQL(ctx) + ), + ); // `beforeFinalize` SQL includes state management SQL that log all the // state changes between this notebooks' cells; however, the "exit" state @@ -341,13 +354,13 @@ export class IngestEngine { -- emit all the SQLPage content ${((await spc.sqlCells()).map(sc => sc.SQL(ctx))).join(";\n\n")}; - ATTACH '${resourceDb}' AS resource_db (TYPE SQLITE); + ATTACH '${resourceDb}' AS ${rdbSchemaName} (TYPE SQLITE); - ${adminTables.map(t => `CREATE TABLE resource_db.${t.tableName} AS SELECT * FROM ${t.tableName}`).join(";\n ")}; + ${adminTables.map(t => `CREATE TABLE ${rdbSchemaName}.${t.tableName} AS SELECT * FROM ${t.tableName}`).join(";\n ")}; - -- {contentResult.map(cr => \`CREATE TABLE resource_db.\${cr.iaSqlSupplier.tableName} AS SELECT * FROM \${cr.tableName}\`).join(";")}; + ${exportsSQL.join(";\n ")}; - DETACH DATABASE resource_db; + DETACH DATABASE ${rdbSchemaName}; ${afterFinalize.length > 0 ? `${afterFinalize.join(";\n ")};` : "-- no after-finalize SQL provided"}`), isc.current.nbCellID); } } diff --git a/src/ahc-hrsn-elt/screening/sqlpage.ts b/src/ahc-hrsn-elt/screening/sqlpage.ts index c78b3411..2783b2c9 100644 --- a/src/ahc-hrsn-elt/screening/sqlpage.ts +++ b/src/ahc-hrsn-elt/screening/sqlpage.ts @@ -4,18 +4,26 @@ import { SQLa_sqlpage as sp, } from "./deps.ts"; -type SQLPageCell = chainNB.NotebookCell< +type SQLPageFile = chainNB.NotebookCell< SQLPageNotebook, chainNB.NotebookCellID >; const nbDescr = new chainNB.NotebookDescriptor< SQLPageNotebook, - SQLPageCell + SQLPageFile >(); -// see https://sql.ophir.dev/documentation.sql?component=debug#component - +/** + * Encapsulates [SQLPage](https://sql.ophir.dev/) content. SqlPageNotebook has + * methods with the name of each [SQLPage](https://sql.ophir.dev/) content that + * we want in the database. The MutationSqlNotebook sqlPageSeedDML method + * "reads" the cells in the SqlPageNotebook (each method's result) and + * generates SQL to insert the content of the page in the database in the format + * and table expected by [SQLPage](https://sql.ophir.dev/). + * NOTE: we break our PascalCase convention for the name of the class since SQLPage + * is a proper noun (product name). + */ export class SQLPageNotebook { readonly tc: ReturnType>; readonly comps = sp.typicalComponents(); @@ -26,19 +34,21 @@ export class SQLPageNotebook { @nbDescr.disregard() shell() { + // deno-fmt-ignore return this.govn.SQL` - SELECT - 'shell' as component, - 'book' as icon, - 'QCS Ingestion Center' as title, - '/' as link, - 'issues' as menu_item, - 'schema' as menu_item`; + ${this.comps.shell({ + title: "QCS Ingestion Center", + icon: "book", + link: "/", + menuItems: [{ caption: "issues" }, { caption: "schema" }] + })} + `; } "index.sql"() { // passing in `chainNB.NotebookCellID` allows us to restrict - // menu hrefs to this notebook's cell names (the pages in SQLPage) + // menu hrefs to this notebook's cell names (the pages in SQLPage) for type + // safety const { list, listItem: li } = sp.typicalComponents< chainNB.NotebookCellID, ddbi.IngestEmitContext @@ -48,18 +58,60 @@ export class SQLPageNotebook { return this.govn.SQL` ${this.shell()} ${list({ items: [ + li({ title: "Screenings", link: "screenings.sql" }), + li({ title: "Jon Doe Screening", link: "jondoe.sql" }), li({ title: "Ingestion Issues", link: "issues.sql" }), li({ title: "Ingestion State Schema", link: "schema.sql" }), ]})}`; } "issues.sql"() { + const { comps: { table }, govn: { SQL } } = this; + // deno-fmt-ignore - return this.govn.SQL` + return SQL` + ${this.shell()} + + ${table({ rows: [{SQL: () => `SELECT * FROM "device"`}] })} + + ${table({ rows: [ + { SQL: () => `SELECT * FROM "ingest_session"`}]})} + + ${table({ search: true, rows: [ + { SQL: () => `SELECT ingest_session_entry_id, ingest_src, ingest_table_name FROM "ingest_session_entry"`}]})} + + ${table({ search: true, rows: [ + { SQL: () => ` + SELECT issue_type, issue_message, invalid_value, remediation + FROM "ingest_session_issue"`}]})} + `; + } + + "screenings.sql"() { + const { comps: { table }, govn: { SQL } } = this; + + // deno-fmt-ignore + return SQL` + ${this.shell()} + ${table({ search: true, sort: true, rows: [ + { SQL: () => ` + SELECT * + FROM "ahc_hrsn_12_12_2023_valid"`}]})} + `; + } + + "jondoe.sql"() { + const { comps: { text, table }, govn: { SQL } } = this; + + // deno-fmt-ignore + return SQL` ${this.shell()} - SELECT 'table' as component; - SELECT issue_type, issue_message, invalid_value, remediation - FROM "ingest_session_issue"; + ${text({title: "Jon Doe (11223344)", content: {markdown: 'Test'}})} + ${table({ search: true, sort: true, rows: [ + { SQL: () => ` + SELECT pat_mrn_id, question, meas_value + FROM "ahc_hrsn_12_12_2023_valid" + WHERE pat_mrn_id = '11223344'`}]})} `; } diff --git a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.md b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.md index e2b057d0..afc8f9ce 100644 --- a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.md +++ b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.md @@ -238,40 +238,40 @@ SELECT session_entry_id, ingest_session_issue_id, issue_type, issue_message, inv {"session_entry_id":"7b979b68-7227-53fd-b689-e4fe153afb76","ingest_session_issue_id":"abf5c680-a135-5d89-b871-fa5b9b99aed6","issue_type":"Sheet Missing","issue_message":"Excel workbook sheet 'Screening' not found in 'synthetic-fail-excel-01.xlsx' (available: Sheet1)","invalid_value":"support/assurance/ahc-hrsn-elt/screening/synthetic-content/synthetic-fail-excel-01.xlsx"}, {"session_entry_id":"641dff51-97fd-56b3-8443-c1ed568a6d66","ingest_session_issue_id":"d70a4700-6b40-52fc-a7a2-69ef0d7f69ff","issue_type":"Sheet Missing","issue_message":"Excel workbook sheet 'QE_Admin_Data' not found in 'synthetic-fail-excel-01.xlsx' (available: Sheet1)","invalid_value":"support/assurance/ahc-hrsn-elt/screening/synthetic-content/synthetic-fail-excel-01.xlsx"}, {"session_entry_id":"47277588-99e8-59f5-8384-b24344a86073","ingest_session_issue_id":"58b22e99-5854-53bf-adbe-08e67df99b85","issue_type":"TODO","issue_message":"Excel workbook 'ahc-hrsn-valid-01.xlsx' sheet 'Admin_Demographic' has not been implemented yet.","invalid_value":"support/assurance/ahc-hrsn-elt/screening/synthetic-content/ahc-hrsn-valid-01.xlsx"}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"5c3046fa-bb76-4d07-b4d7-2cdfd85193a2","issue_type":"Missing Column","issue_message":"Required column FACILITY is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"4882c6b1-77f1-4cba-ba86-c26e062dab71","issue_type":"Missing Column","issue_message":"Required column FIRST_NAME is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"e6139de2-edd1-4516-a7c0-8828e6a6d4b3","issue_type":"Missing Column","issue_message":"Required column LAST_NAME is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"948d5ddb-caef-4683-af04-a9936974d326","issue_type":"Missing Column","issue_message":"Required column PAT_BIRTH_DATE is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"7f0fd156-d853-438e-a643-62156d1d8f90","issue_type":"Missing Column","issue_message":"Required column MEDICAID_CIN is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"81ac8b91-89c0-4ec2-89ec-fa5d8ee36bc3","issue_type":"Missing Column","issue_message":"Required column ENCOUNTER_ID is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"13a43ce9-da28-48a9-90aa-7a1ff58d7bab","issue_type":"Missing Column","issue_message":"Required column SURVEY is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"a638ff45-26b4-46d9-81b2-ce35f105277f","issue_type":"Missing Column","issue_message":"Required column SURVEY_ID is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"9c1b5ed8-c102-41c4-a8a7-1e9a9b5168a4","issue_type":"Missing Column","issue_message":"Required column NEED_INDICATED is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"f96e38c5-a3e0-483e-ba36-5d5934b9503f","issue_type":"Missing Column","issue_message":"Required column VISIT_PART_2_FLAG is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"74b39964-6432-456d-8471-338dcd6d42e3","issue_type":"Missing Column","issue_message":"Required column VISIT_OMH_FLAG is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, -{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"4000bd76-bd24-49e2-849a-839dabc51be8","issue_type":"Missing Column","issue_message":"Required column VISIT_OPWDD_FLAG is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"c4b960e1-fd30-46a5-bb55-ef8f63a923e0","issue_type":"Missing Column","issue_message":"Required column FACILITY is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"7b730162-4b11-489f-898d-5a58e3219854","issue_type":"Missing Column","issue_message":"Required column FIRST_NAME is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"192f0f5f-25c5-471f-adff-96f8b2076dd3","issue_type":"Missing Column","issue_message":"Required column LAST_NAME is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"5253ce90-35af-474a-81a4-6306b99450d0","issue_type":"Missing Column","issue_message":"Required column PAT_BIRTH_DATE is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"655c87f4-4de7-4ef5-9a94-ec7eae4be078","issue_type":"Missing Column","issue_message":"Required column MEDICAID_CIN is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"cecf1905-e361-4f9f-9a3e-8781a2697bb5","issue_type":"Missing Column","issue_message":"Required column ENCOUNTER_ID is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"d2487b2b-5cd4-425f-b155-842b1a8ede22","issue_type":"Missing Column","issue_message":"Required column SURVEY is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"19ab5051-0ac7-4848-a88a-a15cf2d94639","issue_type":"Missing Column","issue_message":"Required column SURVEY_ID is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"068ec416-5a3b-432e-96d6-674dc6d51bbc","issue_type":"Missing Column","issue_message":"Required column NEED_INDICATED is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"8f2e32fd-f70d-46ed-a13b-db04fd7ca81e","issue_type":"Missing Column","issue_message":"Required column VISIT_PART_2_FLAG is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"d35883ce-22ac-4d53-8f1d-0ef5295a5a0f","issue_type":"Missing Column","issue_message":"Required column VISIT_OMH_FLAG is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, +{"session_entry_id":"a26ce332-3ced-5623-861d-23a2ef78e4a9","ingest_session_issue_id":"047f752e-a980-4b7b-9e88-5dfddd0b816a","issue_type":"Missing Column","issue_message":"Required column VISIT_OPWDD_FLAG is missing in ahc_hrsn_valid_01_screening.","invalid_value":null}, {"session_entry_id":"bc0c03b5-d1ba-5301-850f-5e4c42c1bf09","ingest_session_issue_id":"ae477ba1-c7f1-5f34-847a-50bddb7130aa","issue_type":"TODO","issue_message":"Excel workbook 'ahc-hrsn-valid-01.xlsx' sheet 'QE_Admin_Data' has not been implemented yet.","invalid_value":"support/assurance/ahc-hrsn-elt/screening/synthetic-content/ahc-hrsn-valid-01.xlsx"}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"5cd2a955-9d69-42e9-aa54-03d3f2e1e1be","issue_type":"Missing Column","issue_message":"Required column PAT_MRN_ID is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"e10797f7-b3cf-42c5-9ffc-f8fa529c89df","issue_type":"Missing Column","issue_message":"Required column FACILITY is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"35190394-6745-4cf5-ac25-7637e9c1d08d","issue_type":"Missing Column","issue_message":"Required column FIRST_NAME is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"9e6b7969-254a-4e5a-a2b4-7844fb3bb0ec","issue_type":"Missing Column","issue_message":"Required column LAST_NAME is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"9a969bf1-b846-4f90-b8bc-995c5cbe0ca1","issue_type":"Missing Column","issue_message":"Required column PAT_BIRTH_DATE is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"20074e00-8db4-4ced-950d-28dfc6a384a6","issue_type":"Missing Column","issue_message":"Required column MEDICAID_CIN is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"7c82d85a-a562-4a4c-be1d-eb3644f85357","issue_type":"Missing Column","issue_message":"Required column ENCOUNTER_ID is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"1f2f3f60-99f7-489c-a320-87ec93cd091e","issue_type":"Missing Column","issue_message":"Required column SURVEY is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"426b3fe4-013b-4d2e-a808-412399b989f4","issue_type":"Missing Column","issue_message":"Required column SURVEY_ID is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"263b09ed-942a-4ab5-a387-7b65798bb7e1","issue_type":"Missing Column","issue_message":"Required column RECORDED_TIME is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"c52dbac0-2f05-461a-9f17-7460c1846953","issue_type":"Missing Column","issue_message":"Required column QUESTION is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"028f272c-4fb9-44b2-a253-ed1957e69daf","issue_type":"Missing Column","issue_message":"Required column MEAS_VALUE is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"7aecbcab-353a-4168-8661-21d361ca2c12","issue_type":"Missing Column","issue_message":"Required column QUESTION_CODE is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"1f05683a-c313-4d6c-ba3c-dab029472ba9","issue_type":"Missing Column","issue_message":"Required column QUESTION_CODE_SYSTEM_NAME is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"a736fbae-3a61-44f9-9a4c-ed4720b91762","issue_type":"Missing Column","issue_message":"Required column ANSWER_CODE is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"76da67c9-4093-44cf-b3c4-5aa5dbe243f1","issue_type":"Missing Column","issue_message":"Required column ANSWER_CODE_SYSTEM_NAME is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"bc1fc326-08ba-4d2b-a7c5-3115ffca70a0","issue_type":"Missing Column","issue_message":"Required column SDOH_DOMAIN is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"6acfa2f1-2f37-4bd9-bd69-166a0bf82363","issue_type":"Missing Column","issue_message":"Required column NEED_INDICATED is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"57d1b10a-da42-45ab-96c7-590470dc3f3e","issue_type":"Missing Column","issue_message":"Required column VISIT_PART_2_FLAG is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"8b063687-59c5-4d4e-ab4b-6313e6c3f08f","issue_type":"Missing Column","issue_message":"Required column VISIT_OMH_FLAG is missing in synthetic_fail.","invalid_value":null}, -{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"e2c6ea89-c5e0-4d20-96d1-a2f5cb64efa2","issue_type":"Missing Column","issue_message":"Required column VISIT_OPWDD_FLAG is missing in synthetic_fail.","invalid_value":null}] +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"798bde55-9ddc-4547-bdc1-17c9c7a31fc7","issue_type":"Missing Column","issue_message":"Required column PAT_MRN_ID is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"f86bf60a-5e26-4477-9ba1-54d33fa1ee3c","issue_type":"Missing Column","issue_message":"Required column FACILITY is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"0afbcfba-c4d1-481a-a9dc-964e35a863c3","issue_type":"Missing Column","issue_message":"Required column FIRST_NAME is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"bba7f884-5d02-4654-b115-378bf5d18cf8","issue_type":"Missing Column","issue_message":"Required column LAST_NAME is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"84a79d5e-ec17-49a6-aa05-32c42eccf353","issue_type":"Missing Column","issue_message":"Required column PAT_BIRTH_DATE is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"8e313eee-4a13-4a97-9e61-8659e2ae856d","issue_type":"Missing Column","issue_message":"Required column MEDICAID_CIN is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"14893e2f-48e4-4d56-86f9-7cebcebd8abe","issue_type":"Missing Column","issue_message":"Required column ENCOUNTER_ID is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"18419899-347f-46b9-abc5-26fd42577096","issue_type":"Missing Column","issue_message":"Required column SURVEY is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"a719c7d2-a392-4de4-adfd-060ef456cf3f","issue_type":"Missing Column","issue_message":"Required column SURVEY_ID is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"2400b7ae-df64-4b79-b31d-c3a422e6239b","issue_type":"Missing Column","issue_message":"Required column RECORDED_TIME is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"837ff885-b7e2-45d6-9393-bf554a868fd2","issue_type":"Missing Column","issue_message":"Required column QUESTION is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"89b0767b-c4f7-4c6f-9884-d9d2595d4297","issue_type":"Missing Column","issue_message":"Required column MEAS_VALUE is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"114a3068-c7bc-42bc-aa12-f5323dbdfdd8","issue_type":"Missing Column","issue_message":"Required column QUESTION_CODE is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"bc0acb68-2440-4fc2-ae5c-b921d1147cc5","issue_type":"Missing Column","issue_message":"Required column QUESTION_CODE_SYSTEM_NAME is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"19bf6957-a07e-4295-842f-37837b344276","issue_type":"Missing Column","issue_message":"Required column ANSWER_CODE is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"de407d20-bb55-4f18-b323-04bebec7f2ea","issue_type":"Missing Column","issue_message":"Required column ANSWER_CODE_SYSTEM_NAME is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"cc6efe66-880a-495e-86e1-dc5be91e0f08","issue_type":"Missing Column","issue_message":"Required column SDOH_DOMAIN is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"8245347a-da91-4584-a750-38527c9f7434","issue_type":"Missing Column","issue_message":"Required column NEED_INDICATED is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"886ae2d0-f0bf-4e4b-ae84-01f0d7429f19","issue_type":"Missing Column","issue_message":"Required column VISIT_PART_2_FLAG is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"71828bf0-6318-41f3-84fb-59a6b6934cef","issue_type":"Missing Column","issue_message":"Required column VISIT_OMH_FLAG is missing in synthetic_fail.","invalid_value":null}, +{"session_entry_id":"8aad9cfa-b1a2-5fb1-a6ab-613a79a7e839","ingest_session_issue_id":"8e1d44a6-593b-4a13-b0f0-6cddc32e1265","issue_type":"Missing Column","issue_message":"Required column VISIT_OPWDD_FLAG is missing in synthetic_fail.","invalid_value":null}] ``` @@ -301,11 +301,11 @@ INSERT INTO ingest_session_issue (ingest_session_issue_id, session_id, session_e ## emitResources ```sql - INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('8f460419-7b80-516d-8919-84520950f612', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'NONE', 'ENTER(prepareInit)', NULL, 'rsEE.beforeCell', (make_timestamp(2024, 1, 3, 19, 51, 54.220)), NULL); - INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('7bab389e-54af-5a13-a39f-079abdc73a48', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(prepareInit)', 'ENTER(init)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 3, 19, 51, 54.220)), NULL); - INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('168a34c7-d043-5ec4-a84a-c961f1a301ef', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(init)', 'ENTER(ingest)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 3, 19, 51, 54.220)), NULL); - INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('7ef8bdeb-fd56-5eb9-a09b-ef15ce18dc49', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(ingest)', 'ENTER(ensureContent)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 3, 19, 51, 54.220)), NULL); - INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('591191c7-f693-5957-8734-ac87151ca981', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(ensureContent)', 'ENTER(emitResources)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 3, 19, 51, 54.221)), NULL); + INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('8f460419-7b80-516d-8919-84520950f612', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'NONE', 'ENTER(prepareInit)', NULL, 'rsEE.beforeCell', (make_timestamp(2024, 1, 4, 10, 54, 13.837)), NULL); + INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('7bab389e-54af-5a13-a39f-079abdc73a48', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(prepareInit)', 'ENTER(init)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 4, 10, 54, 13.837)), NULL); + INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('168a34c7-d043-5ec4-a84a-c961f1a301ef', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(init)', 'ENTER(ingest)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 4, 10, 54, 13.837)), NULL); + INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('7ef8bdeb-fd56-5eb9-a09b-ef15ce18dc49', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(ingest)', 'ENTER(ensureContent)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 4, 10, 54, 13.837)), NULL); + INSERT INTO "ingest_session_state" ("ingest_session_state_id", "session_id", "session_entry_id", "from_state", "to_state", "transition_result", "transition_reason", "transitioned_at", "elaboration") VALUES ('591191c7-f693-5957-8734-ac87151ca981', '05e8feaa-0bed-5909-a817-39812494b361', NULL, 'EXIT(ensureContent)', 'ENTER(emitResources)', NULL, 'rsEE.afterCell', (make_timestamp(2024, 1, 4, 10, 54, 13.837)), NULL); -- emit all the SQLPage content CREATE TABLE IF NOT EXISTS "sqlpage_files" ( @@ -314,36 +314,52 @@ INSERT INTO ingest_session_issue (ingest_session_issue_id, session_id, session_e "last_modified" DATE ); -INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('index.sql', 'SELECT - ''shell'' as component, - ''book'' as icon, - ''QCS Ingestion Center'' as title, - ''/'' as link, - ''issues'' as menu_item, - ''schema'' as menu_item; +INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('index.sql', 'SELECT ''shell'' as component, ''QCS Ingestion Center'' as title, ''book'' as icon, ''/'' as link, ''issues'' as menu_item, ''schema'' as menu_item; + ; SELECT ''list'' as component; -SELECT ''Ingestion Issues'' as title, ''issues.sql'' as link; -SELECT ''Ingestion State Schema'' as title, ''schema.sql'' as link;', (CURRENT_TIMESTAMP)) ON CONFLICT(path) DO UPDATE SET contents = EXCLUDED.contents /* TODO: does not work in DuckDB , last_modified = (CURRENT_TIMESTAMP) */; - -INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('issues.sql', 'SELECT - ''shell'' as component, - ''book'' as icon, - ''QCS Ingestion Center'' as title, - ''/'' as link, - ''issues'' as menu_item, - ''schema'' as menu_item; +SELECT ''Screenings'' as title,''screenings.sql'' as link; +SELECT ''Jon Doe Screening'' as title,''jondoe.sql'' as link; +SELECT ''Ingestion Issues'' as title,''issues.sql'' as link; +SELECT ''Ingestion State Schema'' as title,''schema.sql'' as link;', (CURRENT_TIMESTAMP)) ON CONFLICT(path) DO UPDATE SET contents = EXCLUDED.contents /* TODO: does not work in DuckDB , last_modified = (CURRENT_TIMESTAMP) */; + +INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('issues.sql', 'SELECT ''shell'' as component, ''QCS Ingestion Center'' as title, ''book'' as icon, ''/'' as link, ''issues'' as menu_item, ''schema'' as menu_item; + ; + +SELECT ''table'' as component; +SELECT * FROM "device"; + SELECT ''table'' as component; -SELECT issue_type, issue_message, invalid_value, remediation - FROM "ingest_session_issue"; +SELECT * FROM "ingest_session"; + +SELECT ''table'' as component, ''true'' as search; +SELECT ingest_session_entry_id, ingest_src, ingest_table_name FROM "ingest_session_entry"; + +SELECT ''table'' as component, ''true'' as search; + + SELECT issue_type, issue_message, invalid_value, remediation + FROM "ingest_session_issue"; + ', (CURRENT_TIMESTAMP)) ON CONFLICT(path) DO UPDATE SET contents = EXCLUDED.contents /* TODO: does not work in DuckDB , last_modified = (CURRENT_TIMESTAMP) */; + +INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('screenings.sql', 'SELECT ''shell'' as component, ''QCS Ingestion Center'' as title, ''book'' as icon, ''/'' as link, ''issues'' as menu_item, ''schema'' as menu_item; + ; +SELECT ''table'' as component, ''true'' as search, ''true'' as sort; + + SELECT * + FROM "ahc_hrsn_12_12_2023_valid"; + ', (CURRENT_TIMESTAMP)) ON CONFLICT(path) DO UPDATE SET contents = EXCLUDED.contents /* TODO: does not work in DuckDB , last_modified = (CURRENT_TIMESTAMP) */; + +INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('jondoe.sql', 'SELECT ''shell'' as component, ''QCS Ingestion Center'' as title, ''book'' as icon, ''/'' as link, ''issues'' as menu_item, ''schema'' as menu_item; + ; +SELECT ''text'' as component, ''Jon Doe (11223344)'' as title, ''Test'' as contentsmd; +SELECT ''table'' as component, ''true'' as search, ''true'' as sort; + + SELECT pat_mrn_id, question, meas_value + FROM "ahc_hrsn_12_12_2023_valid" + WHERE pat_mrn_id = ''11223344''; ', (CURRENT_TIMESTAMP)) ON CONFLICT(path) DO UPDATE SET contents = EXCLUDED.contents /* TODO: does not work in DuckDB , last_modified = (CURRENT_TIMESTAMP) */; -INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('schema.sql', 'SELECT - ''shell'' as component, - ''book'' as icon, - ''QCS Ingestion Center'' as title, - ''/'' as link, - ''issues'' as menu_item, - ''schema'' as menu_item; +INSERT INTO "sqlpage_files" ("path", "contents", "last_modified") VALUES ('schema.sql', 'SELECT ''shell'' as component, ''QCS Ingestion Center'' as title, ''book'' as icon, ''/'' as link, ''issues'' as menu_item, ''schema'' as menu_item; + ; -- TODO: https://github.com/lovasoa/SQLpage/discussions/109#discussioncomment-7359513 -- see the above for how to fix for SQLPage but figure out to use the same SQL -- in and out of SQLPage (maybe do what Ophir said in discussion and create @@ -429,7 +445,7 @@ FROM CREATE TABLE resource_db.ingest_session_issue AS SELECT * FROM ingest_session_issue; CREATE TABLE resource_db.sqlpage_files AS SELECT * FROM sqlpage_files; - -- {contentResult.map(cr => `CREATE TABLE resource_db.${cr.iaSqlSupplier.tableName} AS SELECT * FROM ${cr.tableName}`).join(";")}; + CREATE TABLE resource_db.ahc_hrsn_12_12_2023_valid AS SELECT * FROM ahc_hrsn_12_12_2023_valid; DETACH DATABASE resource_db; -- no after-finalize SQL provided diff --git a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.xlsx b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.xlsx index dcb52b3f..a44600b6 100644 Binary files a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.xlsx and b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/diagnostics.xlsx differ diff --git a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/ingestion-center.duckdb b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/ingestion-center.duckdb index afc67ef7..4ca5b9f4 100644 Binary files a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/ingestion-center.duckdb and b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/ingestion-center.duckdb differ diff --git a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/resource.sqlite.db b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/resource.sqlite.db index 027c9905..c730c239 100644 Binary files a/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/resource.sqlite.db and b/support/assurance/ahc-hrsn-elt/screening/results-test-e2e/resource.sqlite.db differ