feat: Introduce SQLPage #6

README.md

@@ -54,6 +54,9 @@ that developers and data analysts do not need to procure or purchase any tools.
   Javascript runtime and put it in your `PATH`.
 - Download [DuckDB](https://duckdb.org/docs/installation) 0.9+ for ingestion
   operations and put it in your `PATH`.
+- Download [SQLPage](https://github.com/lovasoa/SQLpage/releases) SQL browser
+  and put it in your `PATH`. This is not strictly required but highly
+  recommended for easier diagnostics.
 
 The utilities mentioned above are cross-platform single-file binaries and will
 run on Windows, MacOS, or Linux. Please be sure to get the proper binaries for
@@ -88,6 +91,7 @@ $ cd 1115-hub
 $ deno task                               # list available tasks in `deno.jsonc`
 $ deno task doctor                        # see if dependencies are installed properly
 $ deno task ahc-hrsn-screening-test-e2e   # run the ingestion tasks as end-to-end test
+$ deno task ahc-hrsn-screening-test-serve # run the ingestion tasks as end-to-end test and serve with SQLPage
 $ deno task ahc-hrsn-screening-doc        # generate documentation for the library in support/docs/lib/ahc-hrsn-elt/screening
 ```
 

deno.jsonc

@@ -3,6 +3,8 @@
     "doctor": "deno run -A ./support/bin/doctor.ts",
     "ahc-hrsn-screening-test-e2e": "deno run -A ./src/ahc-hrsn-elt/screening/test-e2e.ts",
     "ahc-hrsn-screening-doc": "deno doc --html --name=\"AHC HRSN Screening\" --output=./support/docs/lib/ahc-hrsn-elt/screening/ ./src/ahc-hrsn-elt/screening/mod.ts",
-    "prepare-release": "deno task ahc-hrsn-screening-doc"
+    "prepare-release": "deno task ahc-hrsn-screening-doc",
+    "sqlpage": "cd support/assurance/ahc-hrsn-elt/screening/results-test-e2e && sqlpage.bin",
+    "ahc-hrsn-screening-test-serve": "deno task ahc-hrsn-screening-test-e2e && deno task sqlpage"
   }
-}
+}

src/ahc-hrsn-elt/screening/deps.ts

@@ -8,12 +8,13 @@ export * as dax from "https://deno.land/x/[email protected]/mod.ts";
 export * as safeText from "npm:string-ts";
 
 // local: ../../../../../netspective-labs/sql-aide
-// remote: https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20
+// remote: https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23
 
-export * as array from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20/lib/universal/array.ts";
-export * as chainNB from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20/lib/notebook/chain-of-responsibility.ts";
-export * as duckdb_shell from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20/lib/duckdb/mod.ts";
-export * as ws from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20/lib/universal/whitespace.ts";
+export * as array from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/universal/array.ts";
+export * as chainNB from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/notebook/chain-of-responsibility.ts";
+export * as duckdb_shell from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/duckdb/mod.ts";
+export * as ws from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/lib/universal/whitespace.ts";
 
-export * as SQLa from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20/render/mod.ts";
-export * as SQLa_ingest_duckdb from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.20/pattern/ingest/duckdb/mod.ts";
+export * as SQLa from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/render/mod.ts";
+export * as SQLa_ingest_duckdb from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/pattern/ingest/duckdb/mod.ts";
+export * as SQLa_sqlpage from "https://raw.githubusercontent.com/netspective-labs/sql-aide/v0.11.23/pattern/sqlpage/mod.ts";

src/ahc-hrsn-elt/screening/ingest.ts

@@ -6,6 +6,7 @@ import {
   SQLa_ingest_duckdb as ddbi,
   ws,
 } from "./deps.ts";
+import * as sp from "./sqlpage.ts";
 
 import {
   ingestCsvFilesSourcesSupplier,
@@ -306,20 +307,22 @@ export class IngestEngine {
   }
 
   async emitResources(isc: IngestStepContext) {
-    const { args: { resourceDb, session } } = this;
+    const { args: { resourceDb, session }, govn: { emitCtx: ctx } } = this;
     if (resourceDb) {
       try {
         Deno.removeSync(resourceDb);
       } catch (_err) {
         // ignore errors if file does not exist
       }
 
+      const spc = sp.SQLPageNotebook.create(this.govn);
       const adminTables = [
         this.govn.device,
         this.govn.ingestSession,
         this.govn.ingestSessionEntry,
         this.govn.ingestSessionState,
         this.govn.ingestSessionIssue,
+        spc.table,
       ];
 
       const beforeFinalize = session.sqlCatalogSqlText("before-finalize");
@@ -335,6 +338,9 @@ export class IngestEngine {
       await this.duckdb.execute(ws.unindentWhitespace(`
         ${beforeFinalize.length > 0 ? `${beforeFinalize.join(";\n        ")};` : "-- no before-finalize SQL provided"}
 
+        -- emit all the SQLPage content
+        ${((await spc.sqlCells()).map(sc => sc.SQL(ctx))).join(";\n\n")};
+        
         ATTACH '${resourceDb}' AS resource_db (TYPE SQLITE);
 
         ${adminTables.map(t => `CREATE TABLE resource_db.${t.tableName} AS SELECT * FROM ${t.tableName}`).join(";\n        ")};

src/ahc-hrsn-elt/screening/mod.ts

@@ -2,3 +2,4 @@ export * from "./csv.ts";
 export * from "./excel.ts";
 export * from "./governance.ts";
 export * from "./ingest.ts";
+export * from "./sqlpage.ts";

src/ahc-hrsn-elt/screening/sqlpage.ts

@@ -0,0 +1,81 @@
+import {
+  chainNB,
+  SQLa_ingest_duckdb as ddbi,
+  SQLa_sqlpage as sp,
+} from "./deps.ts";
+
+type SQLPageCell = chainNB.NotebookCell<
+  SQLPageNotebook,
+  chainNB.NotebookCellID<SQLPageNotebook>
+>;
+
+const nbDescr = new chainNB.NotebookDescriptor<
+  SQLPageNotebook,
+  SQLPageCell
+>();
+
+// see https://sql.ophir.dev/documentation.sql?component=debug#component
+
+export class SQLPageNotebook {
+  readonly tc: ReturnType<typeof sp.typicalContent<ddbi.IngestEmitContext>>;
+  readonly comps = sp.typicalComponents<string, ddbi.IngestEmitContext>();
+
+  constructor(readonly govn: ddbi.IngestGovernance) {
+    this.tc = sp.typicalContent(govn.SQL);
+  }
+
+  @nbDescr.disregard()
+  shell() {
+    return this.govn.SQL`
+      SELECT
+        'shell'                   as component,
+        'book'                    as icon,
+        'QCS Ingestion Center'    as title,
+        '/'                       as link,
+        'issues'                  as menu_item,
+        'schema'                  as menu_item`;
+  }
+
+  "index.sql"() {
+    // passing in `chainNB.NotebookCellID<SQLPageNotebook>` allows us to restrict
+    // menu hrefs to this notebook's cell names (the pages in SQLPage)
+    const { list, listItem: li } = sp.typicalComponents<
+      chainNB.NotebookCellID<SQLPageNotebook>,
+      ddbi.IngestEmitContext
+    >();
+
+    // deno-fmt-ignore
+    return this.govn.SQL`
+      ${this.shell()}
+      ${list({ items: [
+                li({ title: "Ingestion Issues", link: "issues.sql" }),
+                li({ title: "Ingestion State Schema", link: "schema.sql" }),
+               ]})}`;
+  }
+
+  "issues.sql"() {
+    // deno-fmt-ignore
+    return this.govn.SQL`
+      ${this.shell()}
+      SELECT 'table' as component;
+      SELECT issue_type, issue_message, invalid_value, remediation 
+        FROM "ingest_session_issue";
+    `;
+  }
+
+  "schema.sql"() {
+    return this.govn.SQL`
+      ${this.shell()}
+      ${this.tc.infoSchemaSQL()}
+    `;
+  }
+
+  static create(govn: ddbi.IngestGovernance) {
+    return sp.sqlPageNotebook(
+      SQLPageNotebook.prototype,
+      () => new SQLPageNotebook(govn),
+      () => govn.emitCtx,
+      nbDescr,
+    );
+  }
+}