Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extracting filesystem accesses; 'workspace inspect' command #13

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
Draft

Extracting filesystem accesses; 'workspace inspect' command #13

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a641316
Extracting filesystem access better.
warpfork Aug 26, 2022
45e4552
Fix some regressions from the conversions to using fs.FS.
warpfork Aug 26, 2022
f8c6210
Begin outlining a 'workspace inspect' command.
warpfork Aug 26, 2022
8fdb3c6
Hoist out a ComputeStats func for plots.
warpfork Aug 26, 2022
0f16955
ComputeStats on a plot needs to be recursive!
warpfork Aug 26, 2022
fd29e04
Appease error analyzer.
warpfork Aug 26, 2022
450b5e6
'workspace inspect' command now generates many checkmarks.
warpfork Aug 26, 2022
5b9f07d
'workspace inspect' now has valence opinions on these checkboxes.
warpfork Aug 26, 2022
0fbe91a
'workspace inspect' attempts to align output now.
warpfork Aug 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 31 additions & 14 deletions cmd/warpforge/catalog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"bytes"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
Expand All @@ -13,10 +14,12 @@ import (
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/pkg/logging"
"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/wfapi"
"go.opentelemetry.io/otel"
)

const defaultCatalogUrl = "https://github.com/warpsys/mincatalog.git"
Expand Down Expand Up @@ -108,9 +111,10 @@ func cmdCatalogInit(c *cli.Context) error {
return fmt.Errorf("no catalog name provided")
}
catalogName := c.Args().First()
fsys := os.DirFS("/")

// open the workspace set and get the catalog path
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -145,8 +149,10 @@ func cmdCatalogAdd(c *cli.Context) error {
catalogRefStr := c.Args().Get(1)
url := c.Args().Get(2)

fsys := os.DirFS("/")

// open the workspace set
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -255,7 +261,9 @@ func cmdCatalogAdd(c *cli.Context) error {
}

func cmdCatalogLs(c *cli.Context) error {
wsSet, err := openWorkspaceSet()
fsys := os.DirFS("/")

wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -311,7 +319,9 @@ func gatherCatalogRefs(plot wfapi.Plot) []wfapi.CatalogRef {
}

func cmdCatalogBundle(c *cli.Context) error {
wsSet, err := openWorkspaceSet()
fsys := os.DirFS("/")

wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand All @@ -320,8 +330,9 @@ func cmdCatalogBundle(c *cli.Context) error {
if err != nil {
return fmt.Errorf("failed to get pwd: %s", err)
}
pwd = pwd[1:] // Drop leading slash, for use with fs package.

plot, err := plotFromFile(filepath.Join(pwd, PLOT_FILE_NAME))
plot, err := dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot))
if err != nil {
return err
}
Expand All @@ -330,14 +341,14 @@ func cmdCatalogBundle(c *cli.Context) error {

catalogPath := filepath.Join(pwd, ".warpforge", "catalog")
// create a catalog if it does not exist
if _, err = os.Stat(catalogPath); os.IsNotExist(err) {
err = os.MkdirAll(catalogPath, 0755)
if _, err = fs.Stat(fsys, catalogPath); os.IsNotExist(err) {
err = os.MkdirAll("/"+catalogPath, 0755)
if err != nil {
return fmt.Errorf("failed to create catalog directory: %s", err)
}

// we need to reopen the workspace set after creating the directory
wsSet, err = openWorkspaceSet()
wsSet, err = openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down Expand Up @@ -397,7 +408,9 @@ func installDefaultRemoteCatalog(c *cli.Context, path string) error {
}

func cmdCatalogUpdate(c *cli.Context) error {
wss, err := openWorkspaceSet()
fsys := os.DirFS("/")

wss, err := openWorkspaceSet(fsys)
if err != nil {
return fmt.Errorf("failed to open workspace set: %s", err)
}
Expand Down Expand Up @@ -480,8 +493,10 @@ func cmdCatalogRelease(c *cli.Context) error {
}
catalogName := c.String("name")

fsys := os.DirFS("/")

// open the workspace set
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand All @@ -499,15 +514,15 @@ func cmdCatalogRelease(c *cli.Context) error {
}

// get the module, release, and item values (in format `module:release:item`)
module, err := moduleFromFile("module.wf")
module, err := dab.ModuleFromFile(fsys, "module.wf")
if err != nil {
return err
}

releaseName := c.Args().Get(0)

fmt.Printf("building replay for module = %q, release = %q, executing plot...\n", module.Name, releaseName)
plot, err := plotFromFile(PLOT_FILE_NAME)
plot, err := dab.PlotFromFile(fsys, dab.MagicFilename_Plot)
if err != nil {
return err
}
Expand Down Expand Up @@ -561,6 +576,8 @@ func cmdIngestGitTags(c *cli.Context) error {
url := c.Args().Get(1)
itemName := c.Args().Get(2)

fsys := os.DirFS("/")

// open the remote and list all references
remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin",
Expand All @@ -573,7 +590,7 @@ func cmdIngestGitTags(c *cli.Context) error {

// open the workspace set and catalog
catalogName := c.String("name")
wsSet, err := openWorkspaceSet()
wsSet, err := openWorkspaceSet(fsys)
if err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions cmd/warpforge/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"

"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/wfapi"
)
Expand Down
10 changes: 7 additions & 3 deletions cmd/warpforge/ferk.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/pkg/logging"
"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/wfapi"
"go.opentelemetry.io/otel"
)

var ferkCmdDef = cli.Command{
Expand Down Expand Up @@ -89,15 +91,17 @@ func cmdFerk(c *cli.Context) error {
ctx, span := tr.Start(ctx, c.Command.FullName())
defer span.End()

wss, err := openWorkspaceSet()
fsys := os.DirFS("/")

wss, err := openWorkspaceSet(fsys)
if err != nil {
return err
}

plot := wfapi.Plot{}
if c.String("plot") != "" {
// plot was provided, load from file
plot, err = plotFromFile(c.String("plot"))
plot, err = dab.PlotFromFile(fsys, c.String("plot"))
if err != nil {
return fmt.Errorf("error loading plot from file %q: %s", c.String("plot"), err)
}
Expand Down
7 changes: 7 additions & 0 deletions cmd/warpforge/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ func makeApp(stdin io.Reader, stdout, stderr io.Writer) *cli.App {
&statusCmdDef,
&quickstartCmdDef,
&ferkCmdDef,
&cli.Command{
Name: "workspace",
Usage: "Grouping for subcommands that inspect or affect a whole workspace.",
Subcommands: []*cli.Command{
&cmdDefWorkspaceInspect,
},
},
}
return app
}
Expand Down
1 change: 1 addition & 0 deletions cmd/warpforge/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
qt "github.com/frankban/quicktest"
"github.com/warpfork/go-testmark"
"github.com/warpfork/go-testmark/testexec"

"github.com/warpfork/warpforge/pkg/workspace"
)

Expand Down
20 changes: 11 additions & 9 deletions cmd/warpforge/quickstart.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/wfapi"
)

Expand Down Expand Up @@ -59,13 +61,13 @@ func cmdQuickstart(c *cli.Context) error {
return fmt.Errorf("no module name provided")
}

_, err := os.Stat(MODULE_FILE_NAME)
_, err := os.Stat(dab.MagicFilename_Module)
if !os.IsNotExist(err) {
return fmt.Errorf("%s file already exists", MODULE_FILE_NAME)
return fmt.Errorf("%s file already exists", dab.MagicFilename_Module)
}
_, err = os.Stat(PLOT_FILE_NAME)
_, err = os.Stat(dab.MagicFilename_Plot)
if !os.IsNotExist(err) {
return fmt.Errorf("%s file already exists", PLOT_FILE_NAME)
return fmt.Errorf("%s file already exists", dab.MagicFilename_Plot)
}

moduleName := c.Args().First()
Expand All @@ -79,7 +81,7 @@ func cmdQuickstart(c *cli.Context) error {
if err != nil {
return fmt.Errorf("failed to serialize module")
}
err = os.WriteFile(MODULE_FILE_NAME, moduleSerial, 0644)
err = os.WriteFile(dab.MagicFilename_Module, moduleSerial, 0644)
if err != nil {
return fmt.Errorf("failed to write module.json file: %s", err)
}
Expand All @@ -94,17 +96,17 @@ func cmdQuickstart(c *cli.Context) error {
return fmt.Errorf("failed to serialize plot")
}

err = os.WriteFile(PLOT_FILE_NAME, plotSerial, 0644)
err = os.WriteFile(dab.MagicFilename_Plot, plotSerial, 0644)
if err != nil {
return fmt.Errorf("failed to write %s: %s", PLOT_FILE_NAME, err)
return fmt.Errorf("failed to write %s: %s", dab.MagicFilename_Plot, err)
}

if !c.Bool("quiet") {
fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", MODULE_FILE_NAME, PLOT_FILE_NAME, moduleName)
fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", dab.MagicFilename_Module, dab.MagicFilename_Plot, moduleName)
fmt.Fprintf(c.App.Writer, "Ensure your catalogs are up to date by running `%s catalog update.`.\n", os.Args[0])
fmt.Fprintf(c.App.Writer, "You can check status of this module with `%s status`.\n", os.Args[0])
fmt.Fprintf(c.App.Writer, "You can run this module with `%s run`.\n", os.Args[0])
fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", PLOT_FILE_NAME)
fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", dab.MagicFilename_Plot)
}

return nil
Expand Down
29 changes: 17 additions & 12 deletions cmd/warpforge/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,21 @@ package main
import (
"context"
"fmt"
"io/ioutil"
"io/fs"
"os"
"path/filepath"

"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/json"
"github.com/urfave/cli/v2"
"go.opentelemetry.io/otel"

"github.com/warpfork/warpforge/pkg/dab"
"github.com/warpfork/warpforge/pkg/formulaexec"
"github.com/warpfork/warpforge/pkg/logging"
"github.com/warpfork/warpforge/pkg/plotexec"
"github.com/warpfork/warpforge/pkg/workspace"
"github.com/warpfork/warpforge/wfapi"
"go.opentelemetry.io/otel"
)

var runCmdDef = cli.Command{
Expand All @@ -36,16 +38,16 @@ var runCmdDef = cli.Command{
},
}

func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) {
func execModule(ctx context.Context, fsys fs.FS, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) {
result := wfapi.PlotResults{}

// parse the module, even though it is not currently used
_, err := moduleFromFile(fileName)
_, err := dab.ModuleFromFile(fsys, fileName)
if err != nil {
return result, err
}

plot, err := plotFromFile(filepath.Join(filepath.Dir(fileName), PLOT_FILE_NAME))
plot, err := dab.PlotFromFile(fsys, filepath.Join(filepath.Dir(fileName), dab.MagicFilename_Plot))
if err != nil {
return result, err
}
Expand All @@ -55,7 +57,7 @@ func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName strin
return result, err
}

wss, err := openWorkspaceSet()
wss, err := openWorkspaceSet(fsys)
if err != nil {
return result, err
}
Expand Down Expand Up @@ -97,13 +99,16 @@ func cmdRun(c *cli.Context) error {
},
}

fsys := os.DirFS("/")

if !c.Args().Present() {
// execute the module in the current directory
pwd, err := os.Getwd()
if err != nil {
return fmt.Errorf("could not get current directory")
}
_, err = execModule(ctx, config, filepath.Join(pwd, MODULE_FILE_NAME))
pwd = pwd[1:] // Drop leading slash, for use with fs package.
_, err = execModule(ctx, fsys, config, filepath.Join(pwd, dab.MagicFilename_Module))
if err != nil {
return err
}
Expand All @@ -114,11 +119,11 @@ func cmdRun(c *cli.Context) error {
if err != nil {
return err
}
if filepath.Base(path) == MODULE_FILE_NAME {
if filepath.Base(path) == dab.MagicFilename_Module {
if c.Bool("verbose") {
logger.Debug("executing %q", path)
}
_, err = execModule(ctx, config, path)
_, err = execModule(ctx, fsys, config, path)
if err != nil {
return err
}
Expand All @@ -134,13 +139,13 @@ func cmdRun(c *cli.Context) error {
}
if info.IsDir() {
// directory provided, execute module if it exists
_, err := execModule(ctx, config, filepath.Join(fileName, "module.wf"))
_, err := execModule(ctx, fsys, config, filepath.Join(fileName, "module.wf"))
if err != nil {
return err
}
} else {
// formula or module file provided
f, err := ioutil.ReadFile(fileName)
f, err := fs.ReadFile(fsys, fileName)
if err != nil {
return err
}
Expand Down Expand Up @@ -169,7 +174,7 @@ func cmdRun(c *cli.Context) error {
return err
}
case "module":
_, err := execModule(ctx, config, fileName)
_, err := execModule(ctx, fsys, config, fileName)
if err != nil {
return err
}
Expand Down
Loading