From a641316cde3e9574ee083751055648d3a741a04f Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 17:52:11 +0100
Subject: [PATCH 1/9] Extracting filesystem access better.

Created the 'data access broker' package.
So far it contains only the functions previously in the main command
package, but I think considerably more from the current pkg/workspace
package should be bound there in the near future as well.

The main command package carries an fs.FS around *much* more
consistently now.  This makes little practical difference in this diff,
because it's always still just `os.DirFS("/")`, but it makes the API
more futureproof and ready for pieces to be mocked and tested in units.

I skipped fs'izing some features (namely, quickstart) that are writing
to the filesystem heavily.  Golang still needs writability features
in the new fs packages.  Hopefully that comes out soon.

Added a new error code to describe data from newer versions.
However, I don't think those branches are actually reachable at all;
the libraries we're using from IPLD don't really disambiguate between
codec parse errors vs schema match errors... nor is it immediately
obvious how we'd disambiguate between {schema mismatch because of
something intended as a versioning hint} and {schema mismatch from
wildly different data}.  (Right now, everything will look like the
latter, even if it was something like a plot capsule with a string
looking like something as innocuous as "plot.v2".  Future work!)
---
 cmd/warpforge/catalog.go    | 39 +++++++++++++------
 cmd/warpforge/check.go      |  1 +
 cmd/warpforge/ferk.go       | 10 +++--
 cmd/warpforge/main_test.go  |  1 +
 cmd/warpforge/quickstart.go | 20 +++++-----
 cmd/warpforge/run.go        | 28 +++++++------
 cmd/warpforge/status.go     | 16 +++++---
 cmd/warpforge/util.go       | 52 ++-----------------------
 cmd/warpforge/watch.go      | 13 +++++--
 pkg/dab/doc.go              | 23 +++++++++++
 pkg/dab/module.go           | 78 +++++++++++++++++++++++++++++++++++++
 wfapi/error.go              | 18 +++++++++
 12 files changed, 205 insertions(+), 94 deletions(-)
 create mode 100644 pkg/dab/doc.go
 create mode 100644 pkg/dab/module.go

diff --git a/cmd/warpforge/catalog.go b/cmd/warpforge/catalog.go
index 3fb36a81..28e255c5 100644
--- a/cmd/warpforge/catalog.go
+++ b/cmd/warpforge/catalog.go
@@ -13,10 +13,12 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel"
+
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/pkg/logging"
 	"github.com/warpfork/warpforge/pkg/plotexec"
 	"github.com/warpfork/warpforge/wfapi"
-	"go.opentelemetry.io/otel"
 )
 
 const defaultCatalogUrl = "https://github.com/warpsys/mincatalog.git"
@@ -108,9 +110,10 @@ func cmdCatalogInit(c *cli.Context) error {
 		return fmt.Errorf("no catalog name provided")
 	}
 	catalogName := c.Args().First()
+	fsys := os.DirFS("/")
 
 	// open the workspace set and get the catalog path
-	wsSet, err := openWorkspaceSet()
+	wsSet, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
@@ -145,8 +148,10 @@ func cmdCatalogAdd(c *cli.Context) error {
 	catalogRefStr := c.Args().Get(1)
 	url := c.Args().Get(2)
 
+	fsys := os.DirFS("/")
+
 	// open the workspace set
-	wsSet, err := openWorkspaceSet()
+	wsSet, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
@@ -255,7 +260,9 @@ func cmdCatalogAdd(c *cli.Context) error {
 }
 
 func cmdCatalogLs(c *cli.Context) error {
-	wsSet, err := openWorkspaceSet()
+	fsys := os.DirFS("/")
+
+	wsSet, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
@@ -311,7 +318,9 @@ func gatherCatalogRefs(plot wfapi.Plot) []wfapi.CatalogRef {
 }
 
 func cmdCatalogBundle(c *cli.Context) error {
-	wsSet, err := openWorkspaceSet()
+	fsys := os.DirFS("/")
+
+	wsSet, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
@@ -321,7 +330,7 @@ func cmdCatalogBundle(c *cli.Context) error {
 		return fmt.Errorf("failed to get pwd: %s", err)
 	}
 
-	plot, err := plotFromFile(filepath.Join(pwd, PLOT_FILE_NAME))
+	plot, err := dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot))
 	if err != nil {
 		return err
 	}
@@ -337,7 +346,7 @@ func cmdCatalogBundle(c *cli.Context) error {
 		}
 
 		// we need to reopen the workspace set after creating the directory
-		wsSet, err = openWorkspaceSet()
+		wsSet, err = openWorkspaceSet(fsys)
 		if err != nil {
 			return err
 		}
@@ -397,7 +406,9 @@ func installDefaultRemoteCatalog(c *cli.Context, path string) error {
 }
 
 func cmdCatalogUpdate(c *cli.Context) error {
-	wss, err := openWorkspaceSet()
+	fsys := os.DirFS("/")
+
+	wss, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return fmt.Errorf("failed to open workspace set: %s", err)
 	}
@@ -480,8 +491,10 @@ func cmdCatalogRelease(c *cli.Context) error {
 	}
 	catalogName := c.String("name")
 
+	fsys := os.DirFS("/")
+
 	// open the workspace set
-	wsSet, err := openWorkspaceSet()
+	wsSet, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
@@ -499,7 +512,7 @@ func cmdCatalogRelease(c *cli.Context) error {
 	}
 
 	// get the module, release, and item values (in format `module:release:item`)
-	module, err := moduleFromFile("module.wf")
+	module, err := dab.ModuleFromFile(fsys, "module.wf")
 	if err != nil {
 		return err
 	}
@@ -507,7 +520,7 @@ func cmdCatalogRelease(c *cli.Context) error {
 	releaseName := c.Args().Get(0)
 
 	fmt.Printf("building replay for module = %q, release = %q, executing plot...\n", module.Name, releaseName)
-	plot, err := plotFromFile(PLOT_FILE_NAME)
+	plot, err := dab.PlotFromFile(fsys, dab.MagicFilename_Plot)
 	if err != nil {
 		return err
 	}
@@ -561,6 +574,8 @@ func cmdIngestGitTags(c *cli.Context) error {
 	url := c.Args().Get(1)
 	itemName := c.Args().Get(2)
 
+	fsys := os.DirFS("/")
+
 	// open the remote and list all references
 	remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
 		Name: "origin",
@@ -573,7 +588,7 @@ func cmdIngestGitTags(c *cli.Context) error {
 
 	// open the workspace set and catalog
 	catalogName := c.String("name")
-	wsSet, err := openWorkspaceSet()
+	wsSet, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/warpforge/check.go b/cmd/warpforge/check.go
index a58e5deb..7dbd9ad4 100644
--- a/cmd/warpforge/check.go
+++ b/cmd/warpforge/check.go
@@ -7,6 +7,7 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/codec/json"
 	"github.com/urfave/cli/v2"
+
 	"github.com/warpfork/warpforge/pkg/plotexec"
 	"github.com/warpfork/warpforge/wfapi"
 )
diff --git a/cmd/warpforge/ferk.go b/cmd/warpforge/ferk.go
index e8bc9fc2..f8a53f71 100644
--- a/cmd/warpforge/ferk.go
+++ b/cmd/warpforge/ferk.go
@@ -8,10 +8,12 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/codec/json"
 	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel"
+
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/pkg/logging"
 	"github.com/warpfork/warpforge/pkg/plotexec"
 	"github.com/warpfork/warpforge/wfapi"
-	"go.opentelemetry.io/otel"
 )
 
 var ferkCmdDef = cli.Command{
@@ -89,7 +91,9 @@ func cmdFerk(c *cli.Context) error {
 	ctx, span := tr.Start(ctx, c.Command.FullName())
 	defer span.End()
 
-	wss, err := openWorkspaceSet()
+	fsys := os.DirFS("/")
+
+	wss, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return err
 	}
@@ -97,7 +101,7 @@ func cmdFerk(c *cli.Context) error {
 	plot := wfapi.Plot{}
 	if c.String("plot") != "" {
 		// plot was provided, load from file
-		plot, err = plotFromFile(c.String("plot"))
+		plot, err = dab.PlotFromFile(fsys, c.String("plot"))
 		if err != nil {
 			return fmt.Errorf("error loading plot from file %q: %s", c.String("plot"), err)
 		}
diff --git a/cmd/warpforge/main_test.go b/cmd/warpforge/main_test.go
index defea2e0..e3d2b50f 100644
--- a/cmd/warpforge/main_test.go
+++ b/cmd/warpforge/main_test.go
@@ -11,6 +11,7 @@ import (
 	qt "github.com/frankban/quicktest"
 	"github.com/warpfork/go-testmark"
 	"github.com/warpfork/go-testmark/testexec"
+
 	"github.com/warpfork/warpforge/pkg/workspace"
 )
 
diff --git a/cmd/warpforge/quickstart.go b/cmd/warpforge/quickstart.go
index 933a789f..7ef3ad5b 100644
--- a/cmd/warpforge/quickstart.go
+++ b/cmd/warpforge/quickstart.go
@@ -7,6 +7,8 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/codec/json"
 	"github.com/urfave/cli/v2"
+
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/wfapi"
 )
 
@@ -59,13 +61,13 @@ func cmdQuickstart(c *cli.Context) error {
 		return fmt.Errorf("no module name provided")
 	}
 
-	_, err := os.Stat(MODULE_FILE_NAME)
+	_, err := os.Stat(dab.MagicFilename_Module)
 	if !os.IsNotExist(err) {
-		return fmt.Errorf("%s file already exists", MODULE_FILE_NAME)
+		return fmt.Errorf("%s file already exists", dab.MagicFilename_Module)
 	}
-	_, err = os.Stat(PLOT_FILE_NAME)
+	_, err = os.Stat(dab.MagicFilename_Plot)
 	if !os.IsNotExist(err) {
-		return fmt.Errorf("%s file already exists", PLOT_FILE_NAME)
+		return fmt.Errorf("%s file already exists", dab.MagicFilename_Plot)
 	}
 
 	moduleName := c.Args().First()
@@ -79,7 +81,7 @@ func cmdQuickstart(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to serialize module")
 	}
-	err = os.WriteFile(MODULE_FILE_NAME, moduleSerial, 0644)
+	err = os.WriteFile(dab.MagicFilename_Module, moduleSerial, 0644)
 	if err != nil {
 		return fmt.Errorf("failed to write module.json file: %s", err)
 	}
@@ -94,17 +96,17 @@ func cmdQuickstart(c *cli.Context) error {
 		return fmt.Errorf("failed to serialize plot")
 	}
 
-	err = os.WriteFile(PLOT_FILE_NAME, plotSerial, 0644)
+	err = os.WriteFile(dab.MagicFilename_Plot, plotSerial, 0644)
 	if err != nil {
-		return fmt.Errorf("failed to write %s: %s", PLOT_FILE_NAME, err)
+		return fmt.Errorf("failed to write %s: %s", dab.MagicFilename_Plot, err)
 	}
 
 	if !c.Bool("quiet") {
-		fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", MODULE_FILE_NAME, PLOT_FILE_NAME, moduleName)
+		fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", dab.MagicFilename_Module, dab.MagicFilename_Plot, moduleName)
 		fmt.Fprintf(c.App.Writer, "Ensure your catalogs are up to date by running `%s catalog update.`.\n", os.Args[0])
 		fmt.Fprintf(c.App.Writer, "You can check status of this module with `%s status`.\n", os.Args[0])
 		fmt.Fprintf(c.App.Writer, "You can run this module with `%s run`.\n", os.Args[0])
-		fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", PLOT_FILE_NAME)
+		fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", dab.MagicFilename_Plot)
 	}
 
 	return nil
diff --git a/cmd/warpforge/run.go b/cmd/warpforge/run.go
index 881625fe..3403bf1c 100644
--- a/cmd/warpforge/run.go
+++ b/cmd/warpforge/run.go
@@ -3,19 +3,21 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path/filepath"
 
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/codec/json"
 	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel"
+
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/pkg/formulaexec"
 	"github.com/warpfork/warpforge/pkg/logging"
 	"github.com/warpfork/warpforge/pkg/plotexec"
 	"github.com/warpfork/warpforge/pkg/workspace"
 	"github.com/warpfork/warpforge/wfapi"
-	"go.opentelemetry.io/otel"
 )
 
 var runCmdDef = cli.Command{
@@ -36,16 +38,16 @@ var runCmdDef = cli.Command{
 	},
 }
 
-func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) {
+func execModule(ctx context.Context, fsys fs.FS, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) {
 	result := wfapi.PlotResults{}
 
 	// parse the module, even though it is not currently used
-	_, err := moduleFromFile(fileName)
+	_, err := dab.ModuleFromFile(fsys, fileName)
 	if err != nil {
 		return result, err
 	}
 
-	plot, err := plotFromFile(filepath.Join(filepath.Dir(fileName), PLOT_FILE_NAME))
+	plot, err := dab.PlotFromFile(fsys, filepath.Join(filepath.Dir(fileName), dab.MagicFilename_Plot))
 	if err != nil {
 		return result, err
 	}
@@ -55,7 +57,7 @@ func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName strin
 		return result, err
 	}
 
-	wss, err := openWorkspaceSet()
+	wss, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return result, err
 	}
@@ -97,13 +99,15 @@ func cmdRun(c *cli.Context) error {
 		},
 	}
 
+	fsys := os.DirFS("/")
+
 	if !c.Args().Present() {
 		// execute the module in the current directory
 		pwd, err := os.Getwd()
 		if err != nil {
 			return fmt.Errorf("could not get current directory")
 		}
-		_, err = execModule(ctx, config, filepath.Join(pwd, MODULE_FILE_NAME))
+		_, err = execModule(ctx, fsys, config, filepath.Join(pwd, dab.MagicFilename_Module))
 		if err != nil {
 			return err
 		}
@@ -114,11 +118,11 @@ func cmdRun(c *cli.Context) error {
 				if err != nil {
 					return err
 				}
-				if filepath.Base(path) == MODULE_FILE_NAME {
+				if filepath.Base(path) == dab.MagicFilename_Module {
 					if c.Bool("verbose") {
 						logger.Debug("executing %q", path)
 					}
-					_, err = execModule(ctx, config, path)
+					_, err = execModule(ctx, fsys, config, path)
 					if err != nil {
 						return err
 					}
@@ -134,13 +138,13 @@ func cmdRun(c *cli.Context) error {
 			}
 			if info.IsDir() {
 				// directory provided, execute module if it exists
-				_, err := execModule(ctx, config, filepath.Join(fileName, "module.wf"))
+				_, err := execModule(ctx, fsys, config, filepath.Join(fileName, "module.wf"))
 				if err != nil {
 					return err
 				}
 			} else {
 				// formula or module file provided
-				f, err := ioutil.ReadFile(fileName)
+				f, err := fs.ReadFile(fsys, fileName)
 				if err != nil {
 					return err
 				}
@@ -169,7 +173,7 @@ func cmdRun(c *cli.Context) error {
 						return err
 					}
 				case "module":
-					_, err := execModule(ctx, config, fileName)
+					_, err := execModule(ctx, fsys, config, fileName)
 					if err != nil {
 						return err
 					}
diff --git a/cmd/warpforge/status.go b/cmd/warpforge/status.go
index 07cb2061..5dfea538 100644
--- a/cmd/warpforge/status.go
+++ b/cmd/warpforge/status.go
@@ -3,12 +3,15 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/fatih/color"
 	"github.com/urfave/cli/v2"
+
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/pkg/formulaexec"
 	"github.com/warpfork/warpforge/wfapi"
 )
@@ -25,6 +28,7 @@ func cmdStatus(c *cli.Context) error {
 	fmtWarning := color.New(color.FgHiRed, color.Bold)
 	verbose := c.Bool("verbose")
 
+	fsys := os.DirFS("/")
 	pwd, err := os.Getwd()
 	if err != nil {
 		return fmt.Errorf("could not get current directory")
@@ -85,9 +89,9 @@ func cmdStatus(c *cli.Context) error {
 	// check if pwd is a module, read module and set flag
 	isModule := false
 	var module wfapi.Module
-	if _, err := os.Stat(filepath.Join(pwd, MODULE_FILE_NAME)); err == nil {
+	if _, err := fs.Stat(fsys, filepath.Join(pwd, dab.MagicFilename_Module)); err == nil {
 		isModule = true
-		module, err = moduleFromFile(filepath.Join(pwd, MODULE_FILE_NAME))
+		module, err = dab.ModuleFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Module))
 		if err != nil {
 			return fmt.Errorf("failed to open module file: %s", err)
 		}
@@ -102,11 +106,11 @@ func cmdStatus(c *cli.Context) error {
 	// display module and plot info
 	var plot wfapi.Plot
 	hasPlot := false
-	_, err = os.Stat(filepath.Join(pwd, PLOT_FILE_NAME))
+	_, err = fs.Stat(fsys, filepath.Join(pwd, dab.MagicFilename_Plot))
 	if isModule && err == nil {
 		// module.wf and plot.wf exists, read the plot
 		hasPlot = true
-		plot, err = plotFromFile(filepath.Join(pwd, PLOT_FILE_NAME))
+		plot, err = dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot))
 		if err != nil {
 			return fmt.Errorf("failed to open plot file: %s", err)
 		}
@@ -119,7 +123,7 @@ func cmdStatus(c *cli.Context) error {
 			len(plot.Outputs.Keys))
 
 		// check for missing catalog refs
-		wss, err := openWorkspaceSet()
+		wss, err := openWorkspaceSet(fsys)
 		if err != nil {
 			return fmt.Errorf("failed to open workspace: %s", err)
 		}
@@ -163,7 +167,7 @@ func cmdStatus(c *cli.Context) error {
 
 	// display workspace info
 	fmt.Fprintf(c.App.Writer, "\nWorkspace:\n")
-	wss, err := openWorkspaceSet()
+	wss, err := openWorkspaceSet(fsys)
 	if err != nil {
 		return fmt.Errorf("failed to open workspace set: %s", err)
 	}
diff --git a/cmd/warpforge/util.go b/cmd/warpforge/util.go
index 9da219fd..cfb18076 100644
--- a/cmd/warpforge/util.go
+++ b/cmd/warpforge/util.go
@@ -2,22 +2,14 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/codec/json"
 	"github.com/warpfork/warpforge/pkg/workspace"
-	"github.com/warpfork/warpforge/wfapi"
 )
 
-// special file names for plot and module files
-// these are json files with special formatting for detection
-const PLOT_FILE_NAME = "plot.wf"
-const MODULE_FILE_NAME = "module.wf"
-
 // Returns the file type, which is the file name without extension
 // e.g., formula.wf -> formula, module.wf -> module, etc...
 func getFileType(name string) (string, error) {
@@ -49,53 +41,15 @@ func binPath(bin string) (string, error) {
 // stack: a workspace stack starting at the current working directory,
 // root workspace: the first marked root workspace in the stack, or the home workspace if none are marked,
 // home workspace: the workspace at the user's homedir
-func openWorkspaceSet() (workspace.WorkspaceSet, error) {
+func openWorkspaceSet(fsys fs.FS) (workspace.WorkspaceSet, error) {
 	pwd, err := os.Getwd()
 	if err != nil {
 		return workspace.WorkspaceSet{}, fmt.Errorf("failed to get working directory: %s", err)
 	}
 
-	wss, err := workspace.OpenWorkspaceSet(os.DirFS("/"), "", pwd[1:])
+	wss, err := workspace.OpenWorkspaceSet(fsys, "", pwd[1:])
 	if err != nil {
 		return workspace.WorkspaceSet{}, fmt.Errorf("failed to open workspace: %s", err)
 	}
 	return wss, nil
 }
-
-// takes a path to a plot file, returns a plot
-func plotFromFile(filename string) (wfapi.Plot, error) {
-	f, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return wfapi.Plot{}, err
-	}
-
-	plotCapsule := wfapi.PlotCapsule{}
-	_, err = ipld.Unmarshal(f, json.Decode, &plotCapsule, wfapi.TypeSystem.TypeByName("PlotCapsule"))
-	if err != nil {
-		return wfapi.Plot{}, err
-	}
-	if plotCapsule.Plot == nil {
-		return wfapi.Plot{}, fmt.Errorf("no v1 Plot in PlotCapsule")
-	}
-
-	return *plotCapsule.Plot, nil
-}
-
-// takes a path to a module file, returns a module
-func moduleFromFile(filename string) (wfapi.Module, error) {
-	f, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return wfapi.Module{}, err
-	}
-
-	moduleCapsule := wfapi.ModuleCapsule{}
-	_, err = ipld.Unmarshal(f, json.Decode, &moduleCapsule, wfapi.TypeSystem.TypeByName("ModuleCapsule"))
-	if err != nil {
-		return wfapi.Module{}, err
-	}
-	if moduleCapsule.Module == nil {
-		return wfapi.Module{}, fmt.Errorf("no v1 Module in ModuleCapsule")
-	}
-
-	return *moduleCapsule.Module, nil
-}
diff --git a/cmd/warpforge/watch.go b/cmd/warpforge/watch.go
index f90ceb67..94e3b75d 100644
--- a/cmd/warpforge/watch.go
+++ b/cmd/warpforge/watch.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -9,9 +10,11 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/storage/memory"
 	"github.com/urfave/cli/v2"
+	"go.opentelemetry.io/otel"
+
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/pkg/logging"
 	"github.com/warpfork/warpforge/wfapi"
-	"go.opentelemetry.io/otel"
 )
 
 var watchCmdDef = cli.Command{
@@ -38,10 +41,12 @@ func cmdWatch(c *cli.Context) error {
 	defer span.End()
 
 	path := c.Args().First()
+	fsys := os.DirFS("/")
 
 	// TODO: currently we read the module/plot from the provided path.
 	// instead, we should read it from the git cache dir
-	plot, err := plotFromFile(filepath.Join(path, PLOT_FILE_NAME))
+	// FIXME: though it's rare, this can be considerably divergent
+	plot, err := dab.PlotFromFile(fsys, filepath.Join(path, dab.MagicFilename_Plot))
 	if err != nil {
 		return err
 	}
@@ -94,7 +99,9 @@ func cmdWatch(c *cli.Context) error {
 			if ingestCache[path] != hash {
 				fmt.Println("path", path, "changed, new hash", hash)
 				ingestCache[path] = hash
-				_, err := execModule(ctx, config, filepath.Join(c.Args().First(), MODULE_FILE_NAME))
+				// FIXME: this is also reading off the working tree filesystem instead of out of the git index, which is wrong
+				// Perhaps ideally we'd like to give this thing a whole fsys that just keeps reading out of the git index.
+				_, err := execModule(ctx, fsys, config, filepath.Join(c.Args().First(), dab.MagicFilename_Module))
 				if err != nil {
 					fmt.Printf("exec failed: %s\n", err)
 				}
diff --git a/pkg/dab/doc.go b/pkg/dab/doc.go
new file mode 100644
index 00000000..72c95632
--- /dev/null
+++ b/pkg/dab/doc.go
@@ -0,0 +1,23 @@
+/*
+	Package dab -- short for Data Access Broker -- contains functions that help save and load data,
+	mostly to a local filesystem (but sometimes to a blind content-addressed objectstore, as well).
+
+	Most dab functions return objects from the wfapi package.
+	Some return a dab type, in which case that object is to help manage further access --
+	but eventually you should still reach wfapi data types.
+
+	Functions that deal with the filesystem may expect to be dealing with either
+	a workspace filesystem (e.g., conmingled with other user files),
+	or a catalog filesystem projection (a somewhat stricter situation).
+	Sometimes these are the same.
+	The function name should provide a hint about which situations it handles.
+
+	Sometimes, search features are provided for workspace filesystems,
+	since there is no other index of those contents aside from the filesystem itself.
+
+	Most of these functions return the "latest" version of their relevant API type.
+	At the moment, that's not saying much, because we haven't grown in such a way
+	that we support major varations of API object reversions -- but in the future,
+	this means these functions may do "migrational" transforms to the data on the fly.
+*/
+package dab
diff --git a/pkg/dab/module.go b/pkg/dab/module.go
new file mode 100644
index 00000000..3aee51d9
--- /dev/null
+++ b/pkg/dab/module.go
@@ -0,0 +1,78 @@
+package dab
+
+import (
+	"fmt"
+	"io/fs"
+
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/codec/json"
+
+	"github.com/warpfork/warpforge/wfapi"
+)
+
+const (
+	MagicFilename_Module = "module.wf"
+	MagicFilename_Plot   = "plot.wf"
+)
+
+// ModuleFromFile loads a wfapi.Module from filesystem path.
+//
+// In typical usage, the filename parameter will have the suffix of MagicFilename_Module.
+//
+// Errors:
+//
+// 	- warpforge-error-io -- for errors reading from fsys.
+// 	- warpforge-error-serialization -- for errors from try to parse the data as a Module.
+// 	- warpforge-error-datatoonew -- if encountering unknown data from a newer version of warpforge!
+//
+func ModuleFromFile(fsys fs.FS, filename string) (wfapi.Module, error) {
+	situation := "loading a module"
+
+	f, err := fs.ReadFile(fsys, filename)
+	if err != nil {
+		return wfapi.Module{}, wfapi.ErrorIo(situation, &filename, err)
+	}
+
+	moduleCapsule := wfapi.ModuleCapsule{}
+	_, err = ipld.Unmarshal(f, json.Decode, &moduleCapsule, wfapi.TypeSystem.TypeByName("ModuleCapsule"))
+	if err != nil {
+		return wfapi.Module{}, wfapi.ErrorSerialization(situation, err)
+	}
+	if moduleCapsule.Module == nil {
+		// ... this isn't really reachable.
+		return wfapi.Module{}, wfapi.ErrorDataTooNew(situation, fmt.Errorf("no v1 Module in ModuleCapsule"))
+	}
+
+	return *moduleCapsule.Module, nil
+}
+
+// PlotFromFile loads a wfapi.Plot from filesystem path.
+//
+// In typical usage, the filename parameter will have the suffix of MagicFilename_Plot.
+//
+// Errors:
+//
+// 	- warpforge-error-io -- for errors reading from fsys.
+// 	- warpforge-error-serialization -- for errors from try to parse the data as a Plot.
+// 	- warpforge-error-datatoonew -- if encountering unknown data from a newer version of warpforge!
+//
+func PlotFromFile(fsys fs.FS, filename string) (wfapi.Plot, error) {
+	situation := "loading a plot"
+
+	f, err := fs.ReadFile(fsys, filename)
+	if err != nil {
+		return wfapi.Plot{}, wfapi.ErrorIo(situation, &filename, err)
+	}
+
+	plotCapsule := wfapi.PlotCapsule{}
+	_, err = ipld.Unmarshal(f, json.Decode, &plotCapsule, wfapi.TypeSystem.TypeByName("PlotCapsule"))
+	if err != nil {
+		return wfapi.Plot{}, wfapi.ErrorSerialization(situation, err)
+	}
+	if plotCapsule.Plot == nil {
+		// ... this isn't really reachable.
+		return wfapi.Plot{}, wfapi.ErrorDataTooNew(situation, fmt.Errorf("no v1 Plot in PlotCapsule"))
+	}
+
+	return *plotCapsule.Plot, nil
+}
diff --git a/wfapi/error.go b/wfapi/error.go
index 958fa2f2..11145824 100644
--- a/wfapi/error.go
+++ b/wfapi/error.go
@@ -175,6 +175,24 @@ func ErrorSerialization(context string, cause error) Error {
 	}
 }
 
+// ErrorDataTooNew is returned when some data was (partially) deserialized,
+// but only enough that we could recognize it as being a newer version of message
+// than this application supports.
+//
+// Errors:
+//
+//    - warpforge-error-datatoonew -- if some data is too new to parse completely.
+func ErrorDataTooNew(context string, cause error) Error {
+	return &ErrorVal{
+		CodeString: "warpforge-error-datatoonew",
+		Message:    fmt.Sprintf("while %s, encountered data from an unknown version: %s", context, cause),
+		Details: [][2]string{
+			{"context", context},
+		},
+		Cause: wrapErr(cause),
+	}
+}
+
 // ErrorWareUnpack is returned when the unpacking of a ware fails
 //
 // Errors:

From 45e45525a708db5437bea8a37c52a91498bc4afa Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 19:35:45 +0100
Subject: [PATCH 2/9] Fix some regressions from the conversions to using fs.FS.

Turns out fs.FS is very spikey about receiving a param that begins with
a slash.  Kind of unfortunate, honestly.  Would be a lot easier to use
if it just decided to normalize that out.  Ah well.

(Also turns out one of these was harder to notice than it should've
been because we were shrugging at an over-wide range of errors, rather
than only shrugging at a specifically acceptable not-exists error...
so that's a bug fixed, coincidentally.)

I hope this fixes all regressions.  It's not the cleanest changes now,
either... but this should continue to improve if we get writablity
features attached to the fs package in the future (which, again,
hopefully is very (very) soon now).
---
 cmd/warpforge/catalog.go |  6 ++++--
 cmd/warpforge/run.go     |  1 +
 cmd/warpforge/status.go  | 15 ++++++++++-----
 cmd/warpforge/util.go    |  2 +-
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/cmd/warpforge/catalog.go b/cmd/warpforge/catalog.go
index 28e255c5..525bb78c 100644
--- a/cmd/warpforge/catalog.go
+++ b/cmd/warpforge/catalog.go
@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -329,6 +330,7 @@ func cmdCatalogBundle(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to get pwd: %s", err)
 	}
+	pwd = pwd[1:] // Drop leading slash, for use with fs package.
 
 	plot, err := dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot))
 	if err != nil {
@@ -339,8 +341,8 @@ func cmdCatalogBundle(c *cli.Context) error {
 
 	catalogPath := filepath.Join(pwd, ".warpforge", "catalog")
 	// create a catalog if it does not exist
-	if _, err = os.Stat(catalogPath); os.IsNotExist(err) {
-		err = os.MkdirAll(catalogPath, 0755)
+	if _, err = fs.Stat(fsys, catalogPath); os.IsNotExist(err) {
+		err = os.MkdirAll("/"+catalogPath, 0755)
 		if err != nil {
 			return fmt.Errorf("failed to create catalog directory: %s", err)
 		}
diff --git a/cmd/warpforge/run.go b/cmd/warpforge/run.go
index 3403bf1c..a19cb21e 100644
--- a/cmd/warpforge/run.go
+++ b/cmd/warpforge/run.go
@@ -107,6 +107,7 @@ func cmdRun(c *cli.Context) error {
 		if err != nil {
 			return fmt.Errorf("could not get current directory")
 		}
+		pwd = pwd[1:] // Drop leading slash, for use with fs package.
 		_, err = execModule(ctx, fsys, config, filepath.Join(pwd, dab.MagicFilename_Module))
 		if err != nil {
 			return err
diff --git a/cmd/warpforge/status.go b/cmd/warpforge/status.go
index 5dfea538..78c6e27d 100644
--- a/cmd/warpforge/status.go
+++ b/cmd/warpforge/status.go
@@ -33,6 +33,7 @@ func cmdStatus(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("could not get current directory")
 	}
+	pwd = pwd[1:] // Drop leading slash, for use with fs package.
 
 	// display version
 	if verbose {
@@ -95,6 +96,10 @@ func cmdStatus(c *cli.Context) error {
 		if err != nil {
 			return fmt.Errorf("failed to open module file: %s", err)
 		}
+	} else if os.IsNotExist(err) {
+		// fine; it can just not exist.
+	} else {
+		return err
 	}
 
 	if isModule {
@@ -173,20 +178,20 @@ func cmdStatus(c *cli.Context) error {
 	}
 
 	// handle special case for pwd
-	fmt.Fprintf(c.App.Writer, "\t%s (pwd", pwd)
+	fmt.Fprintf(c.App.Writer, "\t/%s (pwd", pwd)
 	if isModule {
 		fmt.Fprintf(c.App.Writer, ", module")
 	}
 	// check if it's a workspace
-	if _, err := os.Stat(filepath.Join(pwd, ".warpforge")); !os.IsNotExist(err) {
+	if _, err := fs.Stat(fsys, filepath.Join(pwd, ".warpforge")); !os.IsNotExist(err) {
 		fmt.Fprintf(c.App.Writer, ", workspace")
 	}
 	// check if it's a root workspace
-	if _, err := os.Stat(filepath.Join(pwd, ".warpforge/root")); !os.IsNotExist(err) {
+	if _, err := fs.Stat(fsys, filepath.Join(pwd, ".warpforge/root")); !os.IsNotExist(err) {
 		fmt.Fprintf(c.App.Writer, ", root workspace")
 	}
 	// check if it's a git repo
-	if _, err := os.Stat(filepath.Join(pwd, ".git")); !os.IsNotExist(err) {
+	if _, err := fs.Stat(fsys, filepath.Join(pwd, ".git")); !os.IsNotExist(err) {
 		fmt.Fprintf(c.App.Writer, ", git repo")
 	}
 
@@ -197,7 +202,7 @@ func cmdStatus(c *cli.Context) error {
 		fs, subPath := ws.Path()
 		path := fmt.Sprintf("%s%s", fs, subPath)
 
-		if path == pwd {
+		if path == "/"+pwd {
 			// we handle pwd earlier, ignore
 			continue
 		}
diff --git a/cmd/warpforge/util.go b/cmd/warpforge/util.go
index cfb18076..68be19f6 100644
--- a/cmd/warpforge/util.go
+++ b/cmd/warpforge/util.go
@@ -42,7 +42,7 @@ func binPath(bin string) (string, error) {
 // root workspace: the first marked root workspace in the stack, or the home workspace if none are marked,
 // home workspace: the workspace at the user's homedir
 func openWorkspaceSet(fsys fs.FS) (workspace.WorkspaceSet, error) {
-	pwd, err := os.Getwd()
+	pwd, err := os.Getwd() // FIXME why are you doing this again?  you almost certainly already did it moments ago.
 	if err != nil {
 		return workspace.WorkspaceSet{}, fmt.Errorf("failed to get working directory: %s", err)
 	}

From f8c621070eedacc4cd98e3643541980b61c1755f Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 18:23:53 +0100
Subject: [PATCH 3/9] Begin outlining a 'workspace inspect' command.

---
 cmd/warpforge/main.go          |  7 ++++
 cmd/warpforge/winspect.go      | 77 ++++++++++++++++++++++++++++++++++
 pkg/dab/workspace.go           |  5 +++
 pkg/workspace/fsdetect.go      |  3 +-
 pkg/workspace/workspace_set.go |  2 +-
 5 files changed, 92 insertions(+), 2 deletions(-)
 create mode 100644 cmd/warpforge/winspect.go
 create mode 100644 pkg/dab/workspace.go

diff --git a/cmd/warpforge/main.go b/cmd/warpforge/main.go
index 4713019a..40952194 100644
--- a/cmd/warpforge/main.go
+++ b/cmd/warpforge/main.go
@@ -59,6 +59,13 @@ func makeApp(stdin io.Reader, stdout, stderr io.Writer) *cli.App {
 		&statusCmdDef,
 		&quickstartCmdDef,
 		&ferkCmdDef,
+		&cli.Command{
+			Name:  "workspace",
+			Usage: "Grouping for subcommands that inspect or affect a whole workspace.",
+			Subcommands: []*cli.Command{
+				&cmdDefWorkspaceInspect,
+			},
+		},
 	}
 	return app
 }
diff --git a/cmd/warpforge/winspect.go b/cmd/warpforge/winspect.go
new file mode 100644
index 00000000..5dfd3d7e
--- /dev/null
+++ b/cmd/warpforge/winspect.go
@@ -0,0 +1,77 @@
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/warpfork/warpforge/pkg/dab"
+
+	"github.com/urfave/cli/v2"
+)
+
+var cmdDefWorkspaceInspect = cli.Command{
+	Name:   "inspect",
+	Usage:  "Inspect and report upon the situation of the current workspace (how many modules are there, have we got a cached evaluation of them, etc).",
+	Action: cmdFnWorkspaceInspect,
+	// Aliases: []string{"winspect"}, // doesn't put them at the top level.  Womp.
+}
+
+func cmdFnWorkspaceInspect(c *cli.Context) error {
+	fsys := os.DirFS("/")
+
+	// First, find the workspace.
+	wss, err := openWorkspaceSet(fsys)
+	if err != nil {
+		return fmt.Errorf("failed to open workspace set: %s", err)
+	}
+
+	// Briefly report on the nearest workspace.
+	// (We could talk about the grandparents too, but 'wf status' already does that; here we want to focus more on contents than parentage.)
+	wsFs, wsPath := wss.Stack[0].Path()
+	fmt.Fprintf(c.App.Writer, "Workspace: %s%s\n", wsFs, wsPath)
+
+	// Search for modules within the workspace.
+	fs.WalkDir(wsFs, wsPath, func(path string, d fs.DirEntry, err error) error {
+		// fmt.Fprintf(c.App.Writer, "hi: %s%s\n", wsFs, path)
+
+		if err != nil {
+			return err
+		}
+
+		// Don't ever look into warpforge guts directories.
+		if d.Name() == dab.MagicFilename_Workspace {
+			return fs.SkipDir
+		}
+
+		// If this is a dir (beyond the root): look see if it contains a workspace marker.
+		// If it does, we might not want to report on it.
+		// TODO: a bool flag for this.
+		if d.IsDir() && len(path) > len(wsPath) {
+			_, e2 := fs.Stat(wsFs, filepath.Join(path, dab.MagicFilename_Workspace))
+			if e2 == nil || os.IsNotExist(e2) {
+				// carry on
+			} else {
+				return fs.SkipDir
+			}
+		}
+
+		// Peek for module file.
+		if d.Name() == dab.MagicFilename_Module {
+			modPathWithinWs := path[len(wsPath)+1 : len(path)-len(dab.MagicFilename_Module)] // leave the trailing slash on.  For disambig in case we support multiple module files per dir someday.
+			mod, err := dab.ModuleFromFile(wsFs, path)
+			modName := mod.Name
+			if err != nil {
+				modName = "!!Unknown!!"
+			}
+
+			// Tell me about it.
+			fmt.Fprintf(c.App.Writer, "Module found: %q -- at path %q\n", modName, modPathWithinWs)
+		}
+
+		return nil
+	})
+
+	return nil
+}
diff --git a/pkg/dab/workspace.go b/pkg/dab/workspace.go
new file mode 100644
index 00000000..d80267ef
--- /dev/null
+++ b/pkg/dab/workspace.go
@@ -0,0 +1,5 @@
+package dab
+
+const (
+	MagicFilename_Workspace = ".warpforge"
+)
diff --git a/pkg/workspace/fsdetect.go b/pkg/workspace/fsdetect.go
index 531cbf0d..17c767d2 100644
--- a/pkg/workspace/fsdetect.go
+++ b/pkg/workspace/fsdetect.go
@@ -6,11 +6,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/wfapi"
 )
 
 const (
-	magicWorkspaceDirname = ".warpforge"
+	magicWorkspaceDirname = dab.MagicFilename_Workspace
 )
 
 var homedir string
diff --git a/pkg/workspace/workspace_set.go b/pkg/workspace/workspace_set.go
index 00bf7021..64576873 100644
--- a/pkg/workspace/workspace_set.go
+++ b/pkg/workspace/workspace_set.go
@@ -14,7 +14,7 @@ import (
 type WorkspaceSet struct {
 	Home  *Workspace
 	Root  *Workspace
-	Stack []*Workspace
+	Stack []*Workspace // the 0'th index is the closest workspace; the next is its parent, and so on.
 }
 
 // Opens a full WorkspaceSet

From 8fdb3c6fbf90c7af14c31d37818cee8588299749 Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 19:47:24 +0100
Subject: [PATCH 4/9] Hoist out a ComputeStats func for plots.

Right now this is behavior only found in `wf status`, but I want to
use it again in other places as well (such as workspace-wide stats).
---
 cmd/warpforge/status.go    | 44 ++++++++++-------------------
 pkg/plotexec/plot_stats.go | 57 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 29 deletions(-)
 create mode 100644 pkg/plotexec/plot_stats.go

diff --git a/cmd/warpforge/status.go b/cmd/warpforge/status.go
index 78c6e27d..c44933d5 100644
--- a/cmd/warpforge/status.go
+++ b/cmd/warpforge/status.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/warpfork/warpforge/pkg/dab"
 	"github.com/warpfork/warpforge/pkg/formulaexec"
+	"github.com/warpfork/warpforge/pkg/plotexec"
 	"github.com/warpfork/warpforge/wfapi"
 )
 
@@ -127,42 +128,27 @@ func cmdStatus(c *cli.Context) error {
 			len(plot.Steps.Keys),
 			len(plot.Outputs.Keys))
 
-		// check for missing catalog refs
+		// Load up workspaces; we'll need to read them to check for missing catalog refs.
 		wss, err := openWorkspaceSet(fsys)
 		if err != nil {
 			return fmt.Errorf("failed to open workspace: %s", err)
 		}
-		catalogRefCount := 0
-		resolvedCatalogRefCount := 0
-		ingestCount := 0
-		mountCount := 0
-		for _, input := range plot.Inputs.Values {
-			if input.Basis().Mount != nil {
-				mountCount++
-			} else if input.Basis().Ingest != nil {
-				ingestCount++
-			} else if input.Basis().CatalogRef != nil {
-				catalogRefCount++
-				ware, _, err := wss.GetCatalogWare(*input.PlotInputSimple.CatalogRef)
-				if err != nil {
-					return fmt.Errorf("failed to lookup catalog ref: %s", err)
-				}
-				if ware == nil {
-					fmt.Fprintf(c.App.Writer, "\tMissing catalog item: %q.\n", input.Basis().CatalogRef.String())
-				} else if err == nil {
-					resolvedCatalogRefCount++
-				}
-			}
+
+		// Compute stats on the plot and report on them (especially any problematic ones).
+		plotStats, err := plotexec.ComputeStats(plot, wss)
+
+		fmt.Fprintf(c.App.Writer, "\tPlot contains %d catalog inputs. %d/%d catalog inputs resolved successfully.\n", plotStats.InputsUsingCatalog, plotStats.ResolvableCatalogInputs, plotStats.InputsUsingCatalog)
+		if plotStats.ResolvableCatalogInputs < plotStats.InputsUsingCatalog {
+			fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d unresolved catalog inputs!\n", (plotStats.InputsUsingCatalog - plotStats.ResolvableCatalogInputs))
 		}
-		fmt.Fprintf(c.App.Writer, "\tPlot contains %d catalog inputs. %d/%d catalog inputs resolved successfully.\n", catalogRefCount, resolvedCatalogRefCount, catalogRefCount)
-		if resolvedCatalogRefCount < catalogRefCount {
-			fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d unresolved catalog inputs!\n", (catalogRefCount - resolvedCatalogRefCount))
+		for k, _ := range plotStats.UnresolvedCatalogInputs {
+			fmt.Fprintf(c.App.Writer, "\tMissing catalog item: %q.\n", k.String())
 		}
-		if ingestCount > 0 {
-			fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d ingest inputs and is not hermetic!\n", ingestCount)
+		if plotStats.InputsUsingIngest > 0 {
+			fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d ingest inputs and is not hermetic!\n", plotStats.InputsUsingIngest)
 		}
-		if mountCount > 0 {
-			fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d mount inputs and is not hermetic!\n", mountCount)
+		if plotStats.InputsUsingMount > 0 {
+			fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d mount inputs and is not hermetic!\n", plotStats.InputsUsingMount)
 		}
 
 	} else if isModule {
diff --git a/pkg/plotexec/plot_stats.go b/pkg/plotexec/plot_stats.go
new file mode 100644
index 00000000..bc7f7148
--- /dev/null
+++ b/pkg/plotexec/plot_stats.go
@@ -0,0 +1,57 @@
+package plotexec
+
+import (
+	"github.com/warpfork/warpforge/pkg/workspace"
+	"github.com/warpfork/warpforge/wfapi"
+)
+
+// Might not match the package name -- funcs in this file certainly don't exec anything.
+
+type PlotStats struct {
+	InputsUsingCatalog      int
+	InputsUsingIngest       int
+	InputsUsingMount        int
+	ResolvableCatalogInputs int
+	ResolvedCatalogInputs   map[wfapi.CatalogRef]wfapi.WareID // might as well remember it if we already did all that work.
+	UnresolvedCatalogInputs map[wfapi.CatalogRef]struct{}
+}
+
+// ComputeStats counts up how many times a plot uses various features,
+// and also checks for reference resolvablity.
+func ComputeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) (PlotStats, error) {
+	v := PlotStats{
+		ResolvedCatalogInputs:   make(map[wfapi.CatalogRef]wfapi.WareID),
+		UnresolvedCatalogInputs: make(map[wfapi.CatalogRef]struct{}),
+	}
+	for _, input := range plot.Inputs.Values {
+		inputBasis := input.Basis() // unwrap if it's a complex filtered thing.
+		switch {
+		// This switch should be exhaustive on the possible members of PlotInputSimple.
+		case inputBasis.WareID != nil:
+			// not interesting :)
+		case inputBasis.Mount != nil:
+			v.InputsUsingMount++
+		case inputBasis.Literal != nil:
+			// not interesting :)
+		case inputBasis.Pipe != nil:
+			// not interesting :)
+		case inputBasis.CatalogRef != nil:
+			v.InputsUsingCatalog++
+			ware, _, err := wsSet.GetCatalogWare(*inputBasis.CatalogRef)
+			if err != nil {
+				return v, err // These mean catalog read failed entirely, so we're in deep water.
+			}
+			if ware == nil {
+				v.UnresolvedCatalogInputs[*inputBasis.CatalogRef] = struct{}{}
+			} else {
+				v.ResolvableCatalogInputs++
+				v.ResolvedCatalogInputs[*inputBasis.CatalogRef] = *ware
+			}
+		case inputBasis.Ingest != nil:
+			v.InputsUsingIngest++
+		default:
+			panic("unreachable")
+		}
+	}
+	return v, nil
+}

From 0f1695509c07bd9041028541335af91469957513 Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 19:55:05 +0100
Subject: [PATCH 5/9] ComputeStats on a plot needs to be recursive!

The actual code that evaluates and execs things is, so the estimator
most definitely must match that as well.

Previously, this would underestimate how many things a plot was
referencing, and could miss it if mounts or ingests were used in a
subplot or in a protoformula instead of being imported and piped in.

If we had been using this for a security boundary, I'd say this is
rather bad.  However, we've only been treating this data as advisory
for now anyway, so... perhaps no klaxons.  But yikes.
---
 pkg/plotexec/plot_stats.go | 40 ++++++++++++++++++++++++++++++++------
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/pkg/plotexec/plot_stats.go b/pkg/plotexec/plot_stats.go
index bc7f7148..808dd31a 100644
--- a/pkg/plotexec/plot_stats.go
+++ b/pkg/plotexec/plot_stats.go
@@ -23,23 +23,28 @@ func ComputeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) (PlotStats, err
 		ResolvedCatalogInputs:   make(map[wfapi.CatalogRef]wfapi.WareID),
 		UnresolvedCatalogInputs: make(map[wfapi.CatalogRef]struct{}),
 	}
-	for _, input := range plot.Inputs.Values {
+	return v, v.computeStats(plot, wsSet)
+}
+
+func (v *PlotStats) computeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) error {
+	accountForInput := func(input wfapi.PlotInput) error {
 		inputBasis := input.Basis() // unwrap if it's a complex filtered thing.
 		switch {
 		// This switch should be exhaustive on the possible members of PlotInputSimple.
 		case inputBasis.WareID != nil:
-			// not interesting :)
+			return nil // not interesting :)
 		case inputBasis.Mount != nil:
 			v.InputsUsingMount++
+			return nil
 		case inputBasis.Literal != nil:
-			// not interesting :)
+			return nil // not interesting :)
 		case inputBasis.Pipe != nil:
-			// not interesting :)
+			return nil // not interesting :)
 		case inputBasis.CatalogRef != nil:
 			v.InputsUsingCatalog++
 			ware, _, err := wsSet.GetCatalogWare(*inputBasis.CatalogRef)
 			if err != nil {
-				return v, err // These mean catalog read failed entirely, so we're in deep water.
+				return err // These mean catalog read failed entirely, so we're in deep water.
 			}
 			if ware == nil {
 				v.UnresolvedCatalogInputs[*inputBasis.CatalogRef] = struct{}{}
@@ -47,11 +52,34 @@ func ComputeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) (PlotStats, err
 				v.ResolvableCatalogInputs++
 				v.ResolvedCatalogInputs[*inputBasis.CatalogRef] = *ware
 			}
+			return nil
 		case inputBasis.Ingest != nil:
 			v.InputsUsingIngest++
+			return nil
+		default:
+			panic("unreachable")
+		}
+	}
+	for _, input := range plot.Inputs.Values {
+		if err := accountForInput(input); err != nil {
+			return err
+		}
+	}
+	for _, step := range plot.Steps.Values {
+		switch {
+		case step.Plot != nil:
+			if err := v.computeStats(*step.Plot, wsSet); err != nil {
+				return err
+			}
+		case step.Protoformula != nil:
+			for _, input := range step.Protoformula.Inputs.Values {
+				if err := accountForInput(input); err != nil {
+					return err
+				}
+			}
 		default:
 			panic("unreachable")
 		}
 	}
-	return v, nil
+	return nil
 }

From fd29e04eaa0802b2ff53baeaec68430b89657a7c Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 20:07:23 +0100
Subject: [PATCH 6/9] Appease error analyzer.

Had to extract the closure to a method.  It seems to have have found an
unsupported situation in the analyzer tool that currently triggers a
panic.  :(

(But getting rid of a closure: alright, fine anyway; that avoids
some very unnecessary allocs anyway, as it turns out.)
---
 pkg/plotexec/plot_stats.go | 82 ++++++++++++++++++++++----------------
 1 file changed, 47 insertions(+), 35 deletions(-)

diff --git a/pkg/plotexec/plot_stats.go b/pkg/plotexec/plot_stats.go
index 808dd31a..5c54580a 100644
--- a/pkg/plotexec/plot_stats.go
+++ b/pkg/plotexec/plot_stats.go
@@ -18,6 +18,17 @@ type PlotStats struct {
 
 // ComputeStats counts up how many times a plot uses various features,
 // and also checks for reference resolvablity.
+//
+// Any errors arising from this process have to do with failure to load
+// catalog info, and causes immediate abort which will result in
+// incomplete counts for all features.
+//
+// Errors:
+//
+// 	- warpforge-error-catalog-invalid -- like it says on the tin.
+// 	- warpforge-error-catalog-parse -- like it says on the tin.
+// 	- warpforge-error-io -- for IO errors while reading catalogs.
+//
 func ComputeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) (PlotStats, error) {
 	v := PlotStats{
 		ResolvedCatalogInputs:   make(map[wfapi.CatalogRef]wfapi.WareID),
@@ -27,41 +38,8 @@ func ComputeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) (PlotStats, err
 }
 
 func (v *PlotStats) computeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) error {
-	accountForInput := func(input wfapi.PlotInput) error {
-		inputBasis := input.Basis() // unwrap if it's a complex filtered thing.
-		switch {
-		// This switch should be exhaustive on the possible members of PlotInputSimple.
-		case inputBasis.WareID != nil:
-			return nil // not interesting :)
-		case inputBasis.Mount != nil:
-			v.InputsUsingMount++
-			return nil
-		case inputBasis.Literal != nil:
-			return nil // not interesting :)
-		case inputBasis.Pipe != nil:
-			return nil // not interesting :)
-		case inputBasis.CatalogRef != nil:
-			v.InputsUsingCatalog++
-			ware, _, err := wsSet.GetCatalogWare(*inputBasis.CatalogRef)
-			if err != nil {
-				return err // These mean catalog read failed entirely, so we're in deep water.
-			}
-			if ware == nil {
-				v.UnresolvedCatalogInputs[*inputBasis.CatalogRef] = struct{}{}
-			} else {
-				v.ResolvableCatalogInputs++
-				v.ResolvedCatalogInputs[*inputBasis.CatalogRef] = *ware
-			}
-			return nil
-		case inputBasis.Ingest != nil:
-			v.InputsUsingIngest++
-			return nil
-		default:
-			panic("unreachable")
-		}
-	}
 	for _, input := range plot.Inputs.Values {
-		if err := accountForInput(input); err != nil {
+		if err := v.accountForInput(input, wsSet); err != nil {
 			return err
 		}
 	}
@@ -73,7 +51,7 @@ func (v *PlotStats) computeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet)
 			}
 		case step.Protoformula != nil:
 			for _, input := range step.Protoformula.Inputs.Values {
-				if err := accountForInput(input); err != nil {
+				if err := v.accountForInput(input, wsSet); err != nil {
 					return err
 				}
 			}
@@ -83,3 +61,37 @@ func (v *PlotStats) computeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet)
 	}
 	return nil
 }
+
+func (v *PlotStats) accountForInput(input wfapi.PlotInput, wsSet workspace.WorkspaceSet) error {
+	inputBasis := input.Basis() // unwrap if it's a complex filtered thing.
+	switch {
+	// This switch should be exhaustive on the possible members of PlotInputSimple.
+	case inputBasis.WareID != nil:
+		return nil // not interesting :)
+	case inputBasis.Mount != nil:
+		v.InputsUsingMount++
+		return nil
+	case inputBasis.Literal != nil:
+		return nil // not interesting :)
+	case inputBasis.Pipe != nil:
+		return nil // not interesting :)
+	case inputBasis.CatalogRef != nil:
+		v.InputsUsingCatalog++
+		ware, _, err := wsSet.GetCatalogWare(*inputBasis.CatalogRef)
+		if err != nil {
+			return err // These mean catalog read failed entirely, so we're in deep water.
+		}
+		if ware == nil {
+			v.UnresolvedCatalogInputs[*inputBasis.CatalogRef] = struct{}{}
+		} else {
+			v.ResolvableCatalogInputs++
+			v.ResolvedCatalogInputs[*inputBasis.CatalogRef] = *ware
+		}
+		return nil
+	case inputBasis.Ingest != nil:
+		v.InputsUsingIngest++
+		return nil
+	default:
+		panic("unreachable")
+	}
+}

From 450b5e6544b2115a2c8b6752a066780f8fd552b8 Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 20:29:04 +0100
Subject: [PATCH 7/9] 'workspace inspect' command now generates many
 checkmarks.

Okay, not checkmarks yet.  I'm going to prettify this later.

But it does a bunch of interesting accounting.

And also, yes, having the ability to say "--gohard=false" is already
relevant in its performance impacts.  In the 29 modules currently
in the warpsys workspace... it takes 0m0.008s real wallclock time
to just enumerate them (makes sense; it's mostly walk, and just open
and parse on the module file itself to get the name)...

... and it takes a whopping 0m0.390s real wallclock seconds to collect
all these connectivity and health checks.  Like, it's *noticable*.

Mind, I suspect there's very low-hanging fruit for improving that.
I notice the system time is 0m0.273s of that (and 0m0.133s user time
from the same sample), so... I suspect even just throwing some caching
around the reads of the catalog would probably make a big difference.

Tracing might tell us more (but that work is going on parallel to this,
so, more on that later perhaps).
---
 cmd/warpforge/winspect.go | 58 +++++++++++++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 6 deletions(-)

diff --git a/cmd/warpforge/winspect.go b/cmd/warpforge/winspect.go
index 5dfd3d7e..d46693f7 100644
--- a/cmd/warpforge/winspect.go
+++ b/cmd/warpforge/winspect.go
@@ -6,9 +6,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/warpfork/warpforge/pkg/dab"
-
 	"github.com/urfave/cli/v2"
+
+	"github.com/warpfork/warpforge/pkg/dab"
+	"github.com/warpfork/warpforge/pkg/plotexec"
 )
 
 var cmdDefWorkspaceInspect = cli.Command{
@@ -16,6 +17,13 @@ var cmdDefWorkspaceInspect = cli.Command{
 	Usage:  "Inspect and report upon the situation of the current workspace (how many modules are there, have we got a cached evaluation of them, etc).",
 	Action: cmdFnWorkspaceInspect,
 	// Aliases: []string{"winspect"}, // doesn't put them at the top level.  Womp.
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "gohard",
+			Usage: "whether to spend effort checking the health of modules found; if false, just list them.",
+			Value: true,
+		},
+	},
 }
 
 func cmdFnWorkspaceInspect(c *cli.Context) error {
@@ -33,7 +41,7 @@ func cmdFnWorkspaceInspect(c *cli.Context) error {
 	fmt.Fprintf(c.App.Writer, "Workspace: %s%s\n", wsFs, wsPath)
 
 	// Search for modules within the workspace.
-	fs.WalkDir(wsFs, wsPath, func(path string, d fs.DirEntry, err error) error {
+	return fs.WalkDir(wsFs, wsPath, func(path string, d fs.DirEntry, err error) error {
 		// fmt.Fprintf(c.App.Writer, "hi: %s%s\n", wsFs, path)
 
 		if err != nil {
@@ -66,12 +74,50 @@ func cmdFnWorkspaceInspect(c *cli.Context) error {
 				modName = "!!Unknown!!"
 			}
 
+			everythingParses := false
+			importsResolve := false
+			noticeIngestUsage := false
+			noticeMountUsage := false
+			havePacksCached := false // maybe should have a variant for "or we have a replay we're hopeful about"?
+			haveRunrecord := false
+			haveHappyExit := false
+			if c.Bool("gohard") {
+				if err != nil {
+					goto _checksDone
+				}
+				plot, err := dab.PlotFromFile(wsFs, filepath.Join(filepath.Dir(path), dab.MagicFilename_Plot))
+				if err != nil {
+					goto _checksDone
+				}
+				everythingParses = true
+				plotStats, err := plotexec.ComputeStats(plot, wss)
+				if err != nil {
+					return err // if it's hardcore catalog errors, rather than just unresolvables, I'm out
+				}
+				if plotStats.ResolvableCatalogInputs == plotStats.InputsUsingCatalog {
+					importsResolve = true
+				}
+				if plotStats.InputsUsingIngest > 0 {
+					noticeIngestUsage = true
+				}
+				if plotStats.InputsUsingMount > 0 {
+					noticeMountUsage = true
+				}
+				// TODO: havePacksCached is not supported right now :(
+				// TODO: haveRunrecord needs to both do resolve, and go peek at memos, and yet (obviously) not actually run.
+				// TODO: haveHappyExit needs the above.
+			}
+		_checksDone:
+
 			// Tell me about it.
-			fmt.Fprintf(c.App.Writer, "Module found: %q -- at path %q\n", modName, modPathWithinWs)
+			fmt.Fprintf(c.App.Writer, "Module found: %q -- at path %q", modName, modPathWithinWs)
+			if c.Bool("gohard") {
+				fmt.Fprintf(c.App.Writer, " -- %v %v %v %v %v %v %v",
+					everythingParses, importsResolve, noticeIngestUsage, noticeMountUsage, havePacksCached, haveRunrecord, haveHappyExit)
+			}
+			fmt.Fprintf(c.App.Writer, "\n")
 		}
 
 		return nil
 	})
-
-	return nil
 }

From 5b9f07d53a15b293c920cacebbcbc763f192567d Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 20:48:33 +0100
Subject: [PATCH 8/9] 'workspace inspect' now has valence opinions on these
 checkboxes.

Characters placeholder.  I'm being entirely silly here.

This should get replaced by a JSON API or something, and a filter
function that emits ANSI colors and such.

But!  The demo is neat.  If you pipe it to "column -t", it DTRT,
and when using it on the warpsys repo?  Hej, suddenly all the example
modules stand out... because they use ingests, and get flagged with
cautionary markers!  Nice.
---
 cmd/warpforge/winspect.go | 105 +++++++++++++++++++++++++++++++++-----
 1 file changed, 93 insertions(+), 12 deletions(-)

diff --git a/cmd/warpforge/winspect.go b/cmd/warpforge/winspect.go
index d46693f7..02116a8e 100644
--- a/cmd/warpforge/winspect.go
+++ b/cmd/warpforge/winspect.go
@@ -74,34 +74,43 @@ func cmdFnWorkspaceInspect(c *cli.Context) error {
 				modName = "!!Unknown!!"
 			}
 
-			everythingParses := false
-			importsResolve := false
-			noticeIngestUsage := false
-			noticeMountUsage := false
-			havePacksCached := false // maybe should have a variant for "or we have a replay we're hopeful about"?
-			haveRunrecord := false
-			haveHappyExit := false
+			// 0 = idk; 1 = yes; 2 = no.  (0 generally doesn't get rendered.)
+			everythingParses := 0
+			importsResolve := 0
+			noticeIngestUsage := 0
+			noticeMountUsage := 0
+			havePacksCached := 0 // maybe should have a variant for "or we have a replay we're hopeful about"?
+			haveRunrecord := 0
+			haveHappyExit := 0
 			if c.Bool("gohard") {
 				if err != nil {
+					everythingParses = 2
 					goto _checksDone
 				}
 				plot, err := dab.PlotFromFile(wsFs, filepath.Join(filepath.Dir(path), dab.MagicFilename_Plot))
 				if err != nil {
+					everythingParses = 2
 					goto _checksDone
 				}
-				everythingParses = true
+				everythingParses = 1
 				plotStats, err := plotexec.ComputeStats(plot, wss)
 				if err != nil {
 					return err // if it's hardcore catalog errors, rather than just unresolvables, I'm out
 				}
 				if plotStats.ResolvableCatalogInputs == plotStats.InputsUsingCatalog {
-					importsResolve = true
+					importsResolve = 1
+				} else {
+					importsResolve = 2
 				}
 				if plotStats.InputsUsingIngest > 0 {
-					noticeIngestUsage = true
+					noticeIngestUsage = 1
+				} else {
+					noticeIngestUsage = 2
 				}
 				if plotStats.InputsUsingMount > 0 {
-					noticeMountUsage = true
+					noticeMountUsage = 1
+				} else {
+					noticeMountUsage = 2
 				}
 				// TODO: havePacksCached is not supported right now :(
 				// TODO: haveRunrecord needs to both do resolve, and go peek at memos, and yet (obviously) not actually run.
@@ -113,7 +122,14 @@ func cmdFnWorkspaceInspect(c *cli.Context) error {
 			fmt.Fprintf(c.App.Writer, "Module found: %q -- at path %q", modName, modPathWithinWs)
 			if c.Bool("gohard") {
 				fmt.Fprintf(c.App.Writer, " -- %v %v %v %v %v %v %v",
-					everythingParses, importsResolve, noticeIngestUsage, noticeMountUsage, havePacksCached, haveRunrecord, haveHappyExit)
+					glyphCheckOrKlaxon(everythingParses),
+					glyphCheckOrX(importsResolve),
+					glyphCautionary(noticeIngestUsage),
+					glyphCautionary(noticeMountUsage),
+					glyphCheckOrYellow(havePacksCached),
+					glyphCheckOrNada(haveRunrecord),
+					glyphCheckOrKlaxon(haveHappyExit),
+				)
 			}
 			fmt.Fprintf(c.App.Writer, "\n")
 		}
@@ -121,3 +137,68 @@ func cmdFnWorkspaceInspect(c *cli.Context) error {
 		return nil
 	})
 }
+
+func glyphCheckOrX(state int) string {
+	switch state {
+	case 0:
+		return " "
+	case 1:
+		return "✔"
+	case 2:
+		return "✘"
+	default:
+		panic("unreachable")
+	}
+}
+
+func glyphCheckOrKlaxon(state int) string {
+	switch state {
+	case 0:
+		return " "
+	case 1:
+		return "✔"
+	case 2:
+		return "!"
+	default:
+		panic("unreachable")
+	}
+}
+
+func glyphCheckOrYellow(state int) string {
+	switch state {
+	case 0:
+		return " "
+	case 1:
+		return "✔"
+	case 2:
+		return "å"
+	default:
+		panic("unreachable")
+	}
+}
+
+func glyphCheckOrNada(state int) string {
+	switch state {
+	case 0:
+		return " "
+	case 1:
+		return "✔"
+	case 2:
+		return "_"
+	default:
+		panic("unreachable")
+	}
+}
+
+func glyphCautionary(state int) string {
+	switch state {
+	case 0:
+		return " "
+	case 1:
+		return "⚠"
+	case 2:
+		return "_"
+	default:
+		panic("unreachable")
+	}
+}

From 0fbe91ac1dae698734bf62f77fe9423b71bf8ba8 Mon Sep 17 00:00:00 2001
From: Eric Myhre <hash@exultant.us>
Date: Fri, 26 Aug 2022 21:04:50 +0100
Subject: [PATCH 9/9] 'workspace inspect' attempts to align output now.

---
 cmd/warpforge/winspect.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cmd/warpforge/winspect.go b/cmd/warpforge/winspect.go
index 02116a8e..a4f392a5 100644
--- a/cmd/warpforge/winspect.go
+++ b/cmd/warpforge/winspect.go
@@ -119,7 +119,8 @@ func cmdFnWorkspaceInspect(c *cli.Context) error {
 		_checksDone:
 
 			// Tell me about it.
-			fmt.Fprintf(c.App.Writer, "Module found: %q -- at path %q", modName, modPathWithinWs)
+			// FUTURE: perhaps a workspace configuration option for defaults for these padding sizes.
+			fmt.Fprintf(c.App.Writer, "Module found: %-40q -- at path %-26q", modName, modPathWithinWs)
 			if c.Bool("gohard") {
 				fmt.Fprintf(c.App.Writer, " -- %v %v %v %v %v %v %v",
 					glyphCheckOrKlaxon(everythingParses),