From 4d69bb08f3145c6a2923900f0152d60f7463bb67 Mon Sep 17 00:00:00 2001
From: "Shahid N. Shah" <shahid@shah.org>
Date: Sun, 15 Oct 2023 20:36:14 -0400
Subject: [PATCH] docs: update TODOs

---
 pattern/content-aide/README.md        | 20 ++++++++++++++++----
 pattern/content-aide/models.sqla.ts   | 13 +++++++++++++
 pattern/content-aide/notebook.sqla.ts |  3 +++
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/pattern/content-aide/README.md b/pattern/content-aide/README.md
index ad55e6be..71cc2d7f 100644
--- a/pattern/content-aide/README.md
+++ b/pattern/content-aide/README.md
@@ -65,7 +65,7 @@ $ sqlpkg install asg017/http     # https://github.com/asg017/sqlite-http/blob/ma
 $ sqlpkg install asg017/regex    # https://github.com/asg017/sqlite-regex/blob/main/docs.md
 ```
 
-Others to consider:
+Other SQLite extensions to consider:
 
 - `asg017/sqlite-md` - Markdown parser similar to asg017/html (at
   https://github.com/asg017/sqlite-md, not in SqlPkg yet, requested via ticket
@@ -95,6 +95,7 @@ $ ./cactl.ts
 See the contents with [SQLpage](https://github.com/lovasoa/SQLpage):
 
 ```bash
+eget lovasoa/SQLpage   # if you don't already have it downloaded
 DATABASE_URL=sqlite://./device-content.sqlite.db sqlpage.bin
 ```
 
@@ -112,11 +113,21 @@ $ ./cactl.ts sql allHtmlAnchors | sqlite3 device-content.sqlite.db --json
 
 ## Tasks
 
-- [ ] Track walk sessions, observability (parameters used, etc.)
-- [ ] Add args/option for running ./cactl.ts with a starting root path
+- [ ] Add SQL in notebook to easily remove and vacuum all sessions, walk paths,
+      and walk entries prior to a given date.
+  - [ ] Add SQL in notebook to easily remove all sessions, walk paths, and walk
+        entries except the ones referenced by the most recent session.
+- [ ] Add SQL in notebook to report on the differences between sessions (files
+      added, modified, removed)
+- [ ] Add args/option for running ./cactl.ts with a starting root path(s)
 - [ ] Figure out what to do about symlinks
 - [ ] Figure out what to do when fileio_read cannot read larger than 1,000,000
       bytes for hash, etc.
+- [ ] Add one or more SQLPage pages that will contain PlantUML or database
+      description markdown so that the documentation for the database is
+      contained within the DB itself.
+- [ ] Learn from
+      [Scraping JSON, HTML, and ZIP Files with Pure SQLite](https://observablehq.com/@asg017/scrape-json-html-zip-with-sqlite)
 - [ ] See [simon987/sist2](https://github.com/simon987/sist2) for other ideas
       like:
   - [ ] Extracts text and metadata from
@@ -126,7 +137,8 @@ $ ./cactl.ts sql allHtmlAnchors | sqlite3 device-content.sqlite.db --json
         attributes via
         [user scripts](https://github.com/simon987/sist2/blob/master/docs/scripting.md)
   - [ ] Recursive scan inside
-        [archive files](https://github.com/simon987/sist2#archive-files)
+        [archive files](https://github.com/simon987/sist2#archive-files) using
+        [SQLite Zip File support](https://sqlite.org/zipfile.html)
   - [ ] [Named-entity recognition](https://github.com/simon987/sist2#NER)
 
 ## ULID Primary Keys across multiple devices
diff --git a/pattern/content-aide/models.sqla.ts b/pattern/content-aide/models.sqla.ts
index 3b70d6bd..66a72a6b 100755
--- a/pattern/content-aide/models.sqla.ts
+++ b/pattern/content-aide/models.sqla.ts
@@ -32,6 +32,19 @@ export function models<EmitContext extends SQLa.SqlEmitContext>() {
     viewName: ViewName,
   ) => SQLa.viewDefinition<ViewName, EmitContext, DomainQS>(viewName);
 
+  // TODO: consider storing all notebook entries in the database so that
+  // once the database is created, all SQL is part of the database and may be
+  // executed like this from the CLI:
+  //    sqlite3 xyz.db "select sql from sqla_notebook where sqla_notebook_id = 'init'" | sqlite3 xyz.db
+  // TODO: you can pass in arguments using .parameter or `sql_parameters` table, like:
+  //    echo ".parameter set X Y; $(sqlite3 xyz.db \"SELECT sql FROM sqla_notebook WHERE sqla_notebook_id = 'init'\")" | sqlite3 xyz.db
+  // const sqlNotebook = gm.textPkTable("sqla_notebook", {
+  //   sqla_notebook_id: gk.textPrimaryKey(),
+  //   sql: gd.blobText(),
+  //   description: gd.textNullable(),
+  //   ...gm.housekeeping.columns,
+  // });
+
   /**
    * Immutable Devices table represents different machines, servers, or workstations.
    * Every device has a unique identifier (ULID) and contains fields for its name,
diff --git a/pattern/content-aide/notebook.sqla.ts b/pattern/content-aide/notebook.sqla.ts
index 26b53550..7f25672a 100755
--- a/pattern/content-aide/notebook.sqla.ts
+++ b/pattern/content-aide/notebook.sqla.ts
@@ -580,6 +580,9 @@ export function library<EmitContext extends SQLa.SqlEmitContext>(libOptions: {
         SELECT 'table' as component, 1 as search, 1 as sort;
         SELECT name, file_extn, description from mime_type;
       `,
+      // TODO: add one or more pages that will contain PlantUML or database
+      //       description markdown so that the documentation for the database
+      //       is contained within the DB itself.
     };
 
     const { sqlPageFiles } = models;