feat: integrate diagnostics into models

netspective-labs · Jan 18, 2024 · 6a8c977 · 6a8c977
1 parent 0196124
commit 6a8c977
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 30 deletions.
diff --git a/pattern/orchestration/duckdb/notebook.ts b/pattern/orchestration/duckdb/notebook.ts
@@ -72,6 +72,12 @@ export class DuckDbShell {
   readonly diagnostics: ReturnType<
     DuckDbOrchGovernance["orchSessionExecCRF"]["insertDML"]
   >[] = [];
+  // RegEx to find `-- diagnostics-md-ignore-start "X"` / `-- diagnostics-md-ignore-finish "X"` pairs
+  readonly ignoreDiagsSqlMdRegExp = new RegExp(
+    /--\s*diagnostics-md-ignore-start\s+"(.+?)"[\s\S]*?--\s*diagnostics-md-ignore-finish\s+"\1"/,
+    "gm",
+  );
+
   constructor(
     readonly session: o.OrchSession<
       DuckDbOrchGovernance,
@@ -93,18 +99,24 @@ export class DuckDbShell {
     status: dax.CommandResult,
     stdoutFmt?: (stdout: string) => { fmt: string; content: string },
   ) {
-    const markdown: string[] = [`\`\`\`sql\n${sql}\n\`\`\`\n`];
+    const mdSQL = sql.replaceAll(
+      this.ignoreDiagsSqlMdRegExp,
+      (_, name) => `-- removed ${name} from diagnostics Markdown`,
+    );
+
+    // note that our code blocks start with four ```` because SQL might include Markdown with blocks too
+    const markdown: string[] = [`\`\`\`\`sql\n${mdSQL}\n\`\`\`\`\n`];
     if (status.stdout) {
       markdown.push("### stdout");
       const stdout = stdoutFmt?.(status.stdout) ??
         ({ fmt: "sh", content: status.stdout });
       markdown.push(
-        `\`\`\`${stdout.fmt}\n${stdout.content}\n\`\`\``,
+        `\`\`\`\`${stdout.fmt}\n${stdout.content}\n\`\`\`\``,
       );
     }
     if (status.stderr) {
       markdown.push("### stderr");
-      markdown.push(`\`\`\`sh\n${status.stderr}\n\`\`\``);
+      markdown.push(`\`\`\`\`sh\n${status.stderr}\n\`\`\`\``);
     }
     return markdown;
   }
@@ -203,36 +215,20 @@ export class DuckDbShell {
     };
   }
 
-  async emitDiagnostics(options: {
-    readonly emitJson?: ((json: string) => Promise<void>) | undefined;
-    readonly diagsMd?: {
-      readonly emit: (md: string) => Promise<void>;
-      readonly frontmatter?: Record<string, unknown>;
-    } | undefined;
-  }) {
-    const { emitJson, diagsMd } = options;
-
-    await emitJson?.(JSON.stringify(this.diagnostics, null, "  "));
-
-    if (diagsMd) {
-      const md: string[] = [
-        "---",
-        yaml.stringify(diagsMd.frontmatter ?? this.args) + "---",
-        "# Orchestration Diagnostics",
-      ];
-      for (const d of this.diagnostics) {
-        if (Array.isArray(d.insertable)) {
-          for (const i of d.insertable) {
-            md.push(`\n## ${i.exec_identity}`);
-            md.push(`${i.narrative_md}`);
-          }
-        } else {
-          md.push(`\n## ${d.insertable.exec_identity}`);
-          md.push(`${d.insertable.narrative_md}`);
+  diagnosticsMarkdown() {
+    const md: string[] = [];
+    for (const d of this.diagnostics) {
+      if (Array.isArray(d.insertable)) {
+        for (const i of d.insertable) {
+          md.push(`\n## ${i.exec_identity}`);
+          md.push(`${i.narrative_md}`);
         }
+      } else {
+        md.push(`\n## ${d.insertable.exec_identity}`);
+        md.push(`${d.insertable.narrative_md}`);
       }
-      await diagsMd.emit(md.join("\n"));
     }
+    return md.join("\n");
   }
 }
 

diff --git a/pattern/orchestration/governance.ts b/pattern/orchestration/governance.ts
@@ -90,6 +90,9 @@ export class OrchGovernance<EmitContext extends OrchEmitContext> {
     orch_started_at: this.gd.createdAt(),
     orch_finished_at: this.gd.dateTimeNullable(),
     elaboration: this.gd.jsonTextNullable(),
+    args_json: this.gd.jsonTextNullable(),
+    diagnostics_json: this.gd.jsonTextNullable(),
+    diagnostics_md: this.gd.textNullable(),
   }, {
     isIdempotent: true,
     populateQS: (t, c, _, tableName) => {
@@ -99,6 +102,12 @@ export class OrchGovernance<EmitContext extends OrchEmitContext> {
         `${tableName} primary key and internal label (UUID)`;
       c.elaboration.description =
         `JSON governance data (description, documentation, usage, etc. in JSON)`;
+      c.args_json.description =
+        `Sesison arguments in a machine-friendly (engine-dependent) JSON format`;
+      c.diagnostics_json.description =
+        `Diagnostics in a machine-friendly (engine-dependent) JSON format`;
+      c.diagnostics_md.description =
+        `Diagnostics in a human-friendly readable markdown format`;
     },
   });
 

diff --git a/pattern/orchestration/notebook.ts b/pattern/orchestration/notebook.ts
@@ -97,6 +97,11 @@ export class OrchSession<
         ...orchSessionCRF.insertDML({
           orch_session_id: sessionID,
           device_id: device.deviceID,
+          orch_started_at: this.govn.emitCtx.newCurrentTimestamp,
+          // orch_started_at and diagnostics_arg, diagnostics_json, diagnostics_md should be
+          // supplied after session is completed
+          diagnostics_md:
+            `Session ${sessionID} markdown diagnostics not provided (not completed?)`,
         }),
       };
     }