diff --git a/cmd/graph_plan/main.go b/cmd/graph_plan/main.go index 83c2040..30366de 100644 --- a/cmd/graph_plan/main.go +++ b/cmd/graph_plan/main.go @@ -1,84 +1,14 @@ package graph_plan import ( - "fmt" - "html/template" - "io/fs" "log" - "os" "path/filepath" - "strings" - "github.com/bmeg/sifter/evaluate" + "github.com/bmeg/sifter/graphplan" "github.com/bmeg/sifter/playbook" - "github.com/bmeg/sifter/task" "github.com/spf13/cobra" ) -type ObjectConvertStep struct { - Name string - Input string - Class string - Schema string -} - -type GraphBuildStep struct { - Name string - Outdir string - Objects []ObjectConvertStep -} - -var graphScript string = ` - -name: {{.Name}} -class: sifter - -outdir: {{.Outdir}} - -config: -{{range .Objects}} - {{.Name}}: {{.Input}} - {{.Name}}Schema: {{.Schema}} -{{end}} - -inputs: -{{range .Objects}} - {{.Name}}: - jsonLoad: - input: "{{ "{{config." }}{{.Name}}{{"}}"}}" -{{end}} - -pipelines: -{{range .Objects}} - {{.Name}}-graph: - - from: {{.Name}} - - graphBuild: - schema: "{{ "{{config."}}{{.Name}}Schema{{ "}}" }}" - title: {{.Class}} -{{end}} -` - -func contains(n string, c []string) bool { - for _, c := range c { - if n == c { - return true - } - } - return false -} - -func uniqueName(name string, used []string) string { - if !contains(name, used) { - return name - } - for i := 1; ; i++ { - f := fmt.Sprintf("%s_%d", name, i) - if !contains(f, used) { - return f - } - } -} - var outScriptDir = "" var outDataDir = "./" @@ -89,99 +19,35 @@ var Cmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - baseDir, _ := filepath.Abs(args[0]) + scriptPath, _ := filepath.Abs(args[0]) - if outScriptDir != "" { - baseDir, _ = filepath.Abs(outScriptDir) - } else if len(args) > 1 { - return fmt.Errorf("for multiple input directories, based dir must be defined") - } - - _ = baseDir + /* + if outScriptDir != "" { + baseDir, _ = filepath.Abs(outScriptDir) + } else if len(args) > 1 { + return fmt.Errorf("for multiple input directories, based dir must be defined") + } - outDataDir, _ = filepath.Abs(outDataDir) + _ = baseDir + */ outScriptDir, _ = filepath.Abs(outScriptDir) - //outScriptDir, _ = filepath.Rel(baseDir, outScriptDir) - - userInputs := map[string]string{} - - for _, dir := range args { - startDir, _ := filepath.Abs(dir) - filepath.Walk(startDir, - func(path string, info fs.FileInfo, err error) error { - if strings.HasSuffix(path, ".yaml") { - log.Printf("Scanning: %s", path) - pb := playbook.Playbook{} - if sifterErr := playbook.ParseFile(path, &pb); sifterErr == nil { - if len(pb.Pipelines) > 0 || len(pb.Inputs) > 0 { - - localInputs, err := pb.PrepConfig(userInputs, baseDir) - if err == nil { - scriptDir := filepath.Dir(path) - task := task.NewTask(pb.Name, scriptDir, baseDir, pb.GetDefaultOutDir(), localInputs) - - curDataDir, err := filepath.Rel(outScriptDir, outDataDir) - if err != nil { - log.Printf("Path error: %s", err) - } - - gb := GraphBuildStep{Name: pb.Name, Objects: []ObjectConvertStep{}, Outdir: curDataDir} - - for pname, p := range pb.Pipelines { - emitName := "" - for _, s := range p { - if s.Emit != nil { - emitName = s.Emit.Name - } - } - if emitName != "" { - for _, s := range p { - if s.ObjectValidate != nil { - schema, _ := evaluate.ExpressionString(s.ObjectValidate.Schema, task.GetConfig(), map[string]any{}) - outdir := pb.GetDefaultOutDir() - outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName) - - outpath := filepath.Join(outdir, outname) - outpath, _ = filepath.Rel(outScriptDir, outpath) - - schemaPath, _ := filepath.Rel(outScriptDir, schema) - - _ = schemaPath - - objCreate := ObjectConvertStep{Name: pname, Input: outpath, Class: s.ObjectValidate.Title, Schema: schemaPath} - gb.Objects = append(gb.Objects, objCreate) + outDataDir, _ = filepath.Abs(outDataDir) - } - } - } - } + outDataDir, _ = filepath.Rel(outScriptDir, outDataDir) - if len(gb.Objects) > 0 { - log.Printf("Found %d objects", len(gb.Objects)) - tmpl, err := template.New("graphscript").Parse(graphScript) - if err != nil { - panic(err) - } + pb := playbook.Playbook{} - outfile, err := os.Create(filepath.Join(outScriptDir, fmt.Sprintf("%s.yaml", pb.Name))) - if err != nil { - fmt.Printf("Error: %s\n", err) - } - err = tmpl.Execute(outfile, gb) - outfile.Close() - if err != nil { - fmt.Printf("Error: %s\n", err) - } - } - } - } - } else { - //log.Printf("Error: %s", sifterErr) - } - } - return nil - }) + if sifterErr := playbook.ParseFile(scriptPath, &pb); sifterErr == nil { + if len(pb.Pipelines) > 0 || len(pb.Inputs) > 0 { + err := graphplan.NewGraphBuild( + &pb, outScriptDir, outDataDir, + ) + if err != nil { + log.Printf("Error: %s\n", err) + } + } } + return nil }, } diff --git a/cmd/root.go b/cmd/root.go index d0c7474..99a0e6e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,6 +6,7 @@ import ( "github.com/bmeg/sifter/cmd/graph_plan" "github.com/bmeg/sifter/cmd/inspect" "github.com/bmeg/sifter/cmd/run" + "github.com/bmeg/sifter/cmd/scan" "github.com/spf13/cobra" ) @@ -20,6 +21,7 @@ func init() { RootCmd.AddCommand(run.Cmd) RootCmd.AddCommand(inspect.Cmd) RootCmd.AddCommand(graph_plan.Cmd) + RootCmd.AddCommand(scan.Cmd) } var genBashCompletionCmd = &cobra.Command{ diff --git a/cmd/scan/main.go b/cmd/scan/main.go new file mode 100644 index 0000000..2dee020 --- /dev/null +++ b/cmd/scan/main.go @@ -0,0 +1,214 @@ +package scan + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/bmeg/sifter/playbook" + "github.com/bmeg/sifter/task" + "github.com/spf13/cobra" +) + +var jsonOut = false +var objectsOnly = false +var baseDir = "" + +type ScanEntry struct { + ObjectType string `json:"objectType"` + SifterFile string `json:"sifterFile"` + Outfile string `json:"outFile"` +} + +var ObjectCommand = &cobra.Command{ + Use: "objects", + Short: "Scan for outputs", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + scanDir := args[0] + + outputs := []ScanEntry{} + + ScanSifter(scanDir, func(pb *playbook.Playbook) { + for pname, p := range pb.Pipelines { + emitName := "" + for _, s := range p { + if s.Emit != nil { + emitName = s.Emit.Name + } + } + if emitName != "" { + for _, s := range p { + outdir := pb.GetDefaultOutDir() + outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName) + outpath := filepath.Join(outdir, outname) + o := ScanEntry{SifterFile: pb.GetPath(), Outfile: outpath} + if s.ObjectValidate != nil { + //outpath, _ = filepath.Rel(baseDir, outpath) + //fmt.Printf("%s\t%s\n", s.ObjectValidate.Title, outpath) + o.ObjectType = s.ObjectValidate.Title + } + if objectsOnly { + if o.ObjectType != "" { + outputs = append(outputs, o) + } + } else { + outputs = append(outputs, o) + } + } + } + } + }) + + if jsonOut { + j := json.NewEncoder(os.Stdout) + j.SetIndent("", " ") + j.Encode(outputs) + } else { + for _, i := range outputs { + fmt.Printf("%s\t%s\n", i.ObjectType, i.Outfile) + } + } + + return nil + + }, +} + +type ScriptEntry struct { + Path string `json:"path"` + Inputs []string `json:"inputs"` + Outputs []string `json:"outputs"` +} + +func removeDuplicates(s []string) []string { + t := map[string]bool{} + + for _, i := range s { + t[i] = true + } + out := []string{} + for k := range t { + out = append(out, k) + } + return out +} + +func relPathArray(basedir string, paths []string) []string { + out := []string{} + for _, i := range paths { + if o, err := filepath.Rel(baseDir, i); err == nil { + out = append(out, o) + } + } + return out +} + +var ScriptCommand = &cobra.Command{ + Use: "scripts", + Short: "Scan for scripts", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + scanDir := args[0] + + scripts := []ScriptEntry{} + + if baseDir == "" { + baseDir, _ = os.Getwd() + } + baseDir, _ = filepath.Abs(baseDir) + //fmt.Printf("basedir: %s\n", baseDir) + + userInputs := map[string]string{} + + ScanSifter(scanDir, func(pb *playbook.Playbook) { + path := pb.GetPath() + scriptDir := filepath.Dir(path) + + config, _ := pb.PrepConfig(userInputs, baseDir) + + task := task.NewTask(pb.Name, scriptDir, baseDir, pb.GetDefaultOutDir(), config) + sourcePath, _ := filepath.Abs(path) + + cmdPath, _ := filepath.Rel(baseDir, sourcePath) + + inputs := []string{} + outputs := []string{} + for _, p := range pb.GetConfigFields() { + if p.IsDir() || p.IsFile() { + inputs = append(inputs, config[p.Name]) + } + } + //inputs = append(inputs, sourcePath) + + sinks, _ := pb.GetOutputs(task) + for _, v := range sinks { + outputs = append(outputs, v...) + } + + emitters, _ := pb.GetEmitters(task) + for _, v := range emitters { + outputs = append(outputs, v) + } + + //for _, e := range pb.Inputs { + //} + + s := ScriptEntry{ + Path: cmdPath, + Outputs: relPathArray(baseDir, removeDuplicates(outputs)), + Inputs: relPathArray(baseDir, removeDuplicates(inputs)), + } + scripts = append(scripts, s) + }) + + if jsonOut { + e := json.NewEncoder(os.Stdout) + e.SetIndent("", " ") + e.Encode(scripts) + } else { + for _, i := range scripts { + fmt.Printf("%s\n", i) + } + } + + return nil + }, +} + +// Cmd is the declaration of the command line +var Cmd = &cobra.Command{ + Use: "scan", +} + +func init() { + Cmd.AddCommand(ObjectCommand) + Cmd.AddCommand(ScriptCommand) + + objFlags := ObjectCommand.Flags() + objFlags.BoolVarP(&objectsOnly, "objects", "s", objectsOnly, "Objects Only") + objFlags.BoolVarP(&jsonOut, "json", "j", jsonOut, "Output JSON") + + scriptFlags := ScriptCommand.Flags() + scriptFlags.StringVarP(&baseDir, "base", "b", baseDir, "Base Dir") + scriptFlags.BoolVarP(&jsonOut, "json", "j", jsonOut, "Output JSON") + +} + +func ScanSifter(baseDir string, userFunc func(*playbook.Playbook)) { + filepath.Walk(baseDir, + func(path string, info fs.FileInfo, err error) error { + if strings.HasSuffix(path, ".yaml") { + pb := playbook.Playbook{} + if parseErr := playbook.ParseFile(path, &pb); parseErr == nil { + userFunc(&pb) + } + } + return nil + }) +} diff --git a/graphplan/build_template.go b/graphplan/build_template.go new file mode 100644 index 0000000..5c5e2a1 --- /dev/null +++ b/graphplan/build_template.go @@ -0,0 +1,137 @@ +package graphplan + +import ( + "fmt" + "log" + "os" + "path/filepath" + "text/template" + + "github.com/bmeg/sifter/evaluate" + "github.com/bmeg/sifter/playbook" + "github.com/bmeg/sifter/task" +) + +type ObjectConvertStep struct { + Name string + Input string + Class string + Schema string +} + +type GraphBuildStep struct { + Name string + Outdir string + Objects []ObjectConvertStep +} + +var graphScript string = ` + +name: {{.Name}} +class: sifter + +outdir: {{.Outdir}} + +config: +{{range .Objects}} + {{.Name}}: {{.Input}} + {{.Name}}Schema: {{.Schema}} +{{end}} + +inputs: +{{range .Objects}} + {{.Name}}: + jsonLoad: + input: "{{ "{{config." }}{{.Name}}{{"}}"}}" +{{end}} + +pipelines: +{{range .Objects}} + {{.Name}}-graph: + - from: {{.Name}} + - graphBuild: + schema: "{{ "{{config."}}{{.Name}}Schema{{ "}}" }}" + title: {{.Class}} +{{end}} +` + +func contains(n string, c []string) bool { + for _, c := range c { + if n == c { + return true + } + } + return false +} + +func uniqueName(name string, used []string) string { + if !contains(name, used) { + return name + } + for i := 1; ; i++ { + f := fmt.Sprintf("%s_%d", name, i) + if !contains(f, used) { + return f + } + } +} + +func NewGraphBuild(pb *playbook.Playbook, scriptOutDir, dataDir string) error { + userInputs := map[string]string{} + localInputs, _ := pb.PrepConfig(userInputs, filepath.Dir(pb.GetPath())) + + task := task.NewTask(pb.Name, filepath.Dir(pb.GetPath()), filepath.Dir(pb.GetPath()), pb.GetDefaultOutDir(), localInputs) + + convertName := fmt.Sprintf("%s-graph", pb.Name) + + gb := GraphBuildStep{Name: convertName, Objects: []ObjectConvertStep{}, Outdir: dataDir} + + for pname, p := range pb.Pipelines { + emitName := "" + for _, s := range p { + if s.Emit != nil { + emitName = s.Emit.Name + } + } + if emitName != "" { + for _, s := range p { + if s.ObjectValidate != nil { + schema, _ := evaluate.ExpressionString(s.ObjectValidate.Schema, task.GetConfig(), map[string]any{}) + outdir := pb.GetDefaultOutDir() + outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName) + + outpath := filepath.Join(outdir, outname) + outpath, _ = filepath.Rel(scriptOutDir, outpath) + + schemaPath, _ := filepath.Rel(scriptOutDir, schema) + + _ = schemaPath + + objCreate := ObjectConvertStep{Name: pname, Input: outpath, Class: s.ObjectValidate.Title, Schema: schemaPath} + gb.Objects = append(gb.Objects, objCreate) + + } + } + } + } + + if len(gb.Objects) > 0 { + log.Printf("Found %d objects", len(gb.Objects)) + tmpl, err := template.New("graphscript").Parse(graphScript) + if err != nil { + panic(err) + } + + outfile, err := os.Create(filepath.Join(scriptOutDir, fmt.Sprintf("%s.yaml", pb.Name))) + if err != nil { + fmt.Printf("Error: %s\n", err) + } + + err = tmpl.Execute(outfile, gb) + outfile.Close() + if err != nil { + fmt.Printf("Error: %s\n", err) + } + } + return nil +}