Logging update

cmd/graphplan/main.go

@@ -1,16 +1,18 @@
 package graphplan
 
 import (
-	"log"
 	"path/filepath"
 
 	"github.com/bmeg/sifter/graphplan"
+	"github.com/bmeg/sifter/logger"
 	"github.com/bmeg/sifter/playbook"
 	"github.com/spf13/cobra"
 )
 
 var outScriptDir = ""
 var outDataDir = "./"
+var objectExclude = []string{}
+var verbose bool = false
 
 // Cmd is the declaration of the command line
 var Cmd = &cobra.Command{
@@ -19,6 +21,10 @@ var Cmd = &cobra.Command{
 	Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 
+		if verbose {
+			logger.Init(true, false)
+		}
+
 		scriptPath, _ := filepath.Abs(args[0])
 
 		/*
@@ -40,10 +46,10 @@ var Cmd = &cobra.Command{
 		if sifterErr := playbook.ParseFile(scriptPath, &pb); sifterErr == nil {
 			if len(pb.Pipelines) > 0 || len(pb.Inputs) > 0 {
 				err := graphplan.NewGraphBuild(
-					&pb, outScriptDir, outDataDir,
+					&pb, outScriptDir, outDataDir, objectExclude,
 				)
 				if err != nil {
-					log.Printf("Error: %s\n", err)
+					logger.Error("Parse Error", "error", err)
 				}
 			}
 		}
@@ -54,6 +60,8 @@ var Cmd = &cobra.Command{
 
 func init() {
 	flags := Cmd.Flags()
+	flags.BoolVarP(&verbose, "verbose", "v", verbose, "Verbose logging")
 	flags.StringVarP(&outScriptDir, "dir", "C", outScriptDir, "Change Directory for script base")
 	flags.StringVarP(&outDataDir, "out", "o", outDataDir, "Change output Directory")
+	flags.StringArrayVarP(&objectExclude, "exclude", "x", objectExclude, "Object Exclude")
 }

cmd/inspect/main.go

@@ -3,16 +3,15 @@ package inspect
 import (
 	"encoding/json"
 	"fmt"
-	"log"
 	"path/filepath"
 
+	"github.com/bmeg/sifter/logger"
 	"github.com/bmeg/sifter/playbook"
 	"github.com/bmeg/sifter/task"
 	"github.com/spf13/cobra"
 )
 
 var outDir string = ""
-var inputFile string = ""
 var cmdInputs map[string]string
 
 // Cmd is the declaration of the command line
@@ -31,12 +30,12 @@ var Cmd = &cobra.Command{
 
 		pb := playbook.Playbook{}
 		if err := playbook.ParseFile(playFile, &pb); err != nil {
-			log.Printf("%s", err)
+			logger.Info("%s", err)
 			return err
 		}
 		var err error
 		inputs, err = pb.PrepConfig(inputs, "./")
-		log.Printf("inputs: %s", inputs)
+		logger.Info("inputs: %s", inputs)
 		if err != nil {
 			return err
 		}
@@ -45,7 +44,7 @@ var Cmd = &cobra.Command{
 			outDir = pb.GetDefaultOutDir()
 		}
 
-		log.Printf("outdir: %s", outDir)
+		logger.Info("outdir: %s", outDir)
 
 		p, _ := filepath.Abs(playFile)
 		baseDir := filepath.Dir(p)

cmd/run/main.go

@@ -1,37 +1,35 @@
 package run
 
 import (
-	"io/ioutil"
-	"log"
+	"io"
 	"os"
 
+	"github.com/bmeg/sifter/logger"
 	"github.com/bmeg/sifter/playbook"
 
 	"github.com/spf13/cobra"
 )
 
-var workDir string = "./"
 var outDir string = ""
-var resume string = ""
-var graph string = ""
 var inputFile string = ""
-var toStdout bool
-var keep bool
+var verbose bool = false
 var cmdInputs map[string]string
 
-var proxy = ""
-var port = 8888
-
 // Cmd is the declaration of the command line
 var Cmd = &cobra.Command{
 	Use:   "run <script>",
 	Short: "Run sifter script",
 	Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
+
+		if verbose {
+			logger.Init(true, false)
+		}
+
 		inputs := map[string]string{}
 		if inputFile != "" {
 			if err := playbook.ParseStringFile(inputFile, &inputs); err != nil {
-				log.Printf("%s", err)
+				logger.Error("%s", err)
 				return err
 			}
 		}
@@ -40,9 +38,9 @@ var Cmd = &cobra.Command{
 		}
 		for _, playFile := range args {
 			if playFile == "-" {
-				yaml, err := ioutil.ReadAll(os.Stdin)
+				yaml, err := io.ReadAll(os.Stdin)
 				if err != nil {
-					log.Printf("%s", err)
+					logger.Error("%s", err)
 					return err
 				}
 				pb := playbook.Playbook{}
@@ -63,15 +61,7 @@ var Cmd = &cobra.Command{
 
 func init() {
 	flags := Cmd.Flags()
-	flags.StringVarP(&workDir, "workdir", "w", workDir, "Workdir")
-	flags.BoolVarP(&toStdout, "stdout", "s", toStdout, "To STDOUT")
-	flags.BoolVarP(&keep, "keep", "k", keep, "Keep Working Directory")
-	flags.StringVarP(&outDir, "out", "o", outDir, "Output Dir")
-	flags.StringVarP(&resume, "resume", "r", resume, "Resume Directory")
-	flags.StringVarP(&graph, "graph", "g", graph, "Output to graph")
-
-	flags.StringVar(&proxy, "proxy", proxy, "Proxy site")
-	flags.IntVar(&port, "port", port, "Proxy Port")
+	flags.BoolVarP(&verbose, "verbose", "v", verbose, "Verbose logging")
 	flags.StringToStringVarP(&cmdInputs, "inputs", "i", cmdInputs, "Input variables")
 	flags.StringVarP(&inputFile, "inputfile", "f", inputFile, "Input variables file")
 }

cmd/run/run.go

@@ -1,27 +1,24 @@
 package run
 
 import (
-	"fmt"
-	"log"
 	"os"
 	"path/filepath"
 
+	"github.com/bmeg/sifter/logger"
 	"github.com/bmeg/sifter/playbook"
 	"github.com/bmeg/sifter/task"
 )
 
 func ExecuteFile(playFile string, workDir string, outDir string, inputs map[string]string) error {
-	log.Printf("Starting: %s\n", playFile)
-	log.Default().SetPrefix(fmt.Sprintf("%s: ", playFile))
+	logger.Info("Starting", "playFile", playFile)
 	pb := playbook.Playbook{}
 	if err := playbook.ParseFile(playFile, &pb); err != nil {
-		log.Printf("%s", err)
+		logger.Error("%s", err)
 		return err
 	}
 	a, _ := filepath.Abs(playFile)
 	baseDir := filepath.Dir(a)
-	log.Printf("basedir: %s", baseDir)
-	log.Printf("playbook: %#v", pb)
+	logger.Debug("parsed file", "baseDir", baseDir, "playbook", pb)
 	return Execute(pb, baseDir, workDir, outDir, inputs)
 }
 
@@ -39,7 +36,7 @@ func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string
 	if err != nil {
 		return err
 	}
-	log.Printf("Outdir: %s", outDir)
+	logger.Debug("Running", "outDir", outDir)
 
 	t := task.NewTask(pb.Name, baseDir, workDir, outDir, nInputs)
 	err = pb.Execute(t)

evaluate/docker_python.go

@@ -1,7 +1,6 @@
 package evaluate
 
 import (
-	"log"
 	"os"
 	"time"
 
@@ -10,6 +9,7 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/bmeg/sifter/logger"
 	grpc "google.golang.org/grpc"
 )
 
@@ -49,7 +49,7 @@ func StartDockerExecutor(dockerImage string) (Runner, error) {
 		return nil, err
 	}
 	id := strings.Trim(string(out), " \r\n\t")
-	log.Printf("Started Container: %s", id)
+	logger.Debug("Started Container: %s", id)
 	cmd = exec.Command("docker", "port", id)
 	out, err = cmd.Output()
 	if err != nil {
@@ -62,15 +62,15 @@ func StartDockerExecutor(dockerImage string) (Runner, error) {
 	var conn *grpc.ClientConn
 
 	for i := 0; i < 5; i++ {
-		log.Printf("Contacting: %s", serverAddr)
+		logger.Debug("Contacting: %s", serverAddr)
 		conn, err = grpc.Dial(serverAddr, grpc.WithInsecure(), grpc.WithBlock())
 		if err == nil {
 			break
 		}
 		time.Sleep(1 * time.Second)
 	}
 	if err != nil {
-		log.Printf("Returning err: %s", err)
+		logger.Error("Returning err: %s", err)
 		return nil, err
 	}
 	client := NewExecutorClient(conn)
@@ -86,7 +86,7 @@ func (run *DockerRunner) Call(in *Input) (*Result, error) {
 }
 
 func (run *DockerRunner) Close() {
-	log.Printf("Closing docker %s", run.containerID)
+	logger.Debug("Closing docker %s", run.containerID)
 	run.conn.Close()
 	cmd := exec.Command("docker", "kill", run.containerID)
 	cmd.Run()

evaluate/gpython.go

@@ -2,8 +2,8 @@ package evaluate
 
 import (
 	"fmt"
-	"log"
 
+	"github.com/bmeg/sifter/logger"
 	"github.com/go-python/gpython/py"
 	_ "github.com/go-python/gpython/stdlib" // Load modiles in gpython environment
 )
@@ -80,7 +80,7 @@ func PyObject(i interface{}) py.Object {
 	} else if i == nil {
 		return py.None
 	} else {
-		log.Printf("gpython conversion Not found: %T", i)
+		logger.Error("gpython conversion Not found: %T", i)
 	}
 	return nil
 }
@@ -127,7 +127,7 @@ func FromPyObject(i py.Object) interface{} {
 	} else if i == py.None {
 		return nil
 	} else {
-		log.Printf("gpython conversion Not found: %T", i)
+		logger.Error("gpython conversion Not found: %T", i)
 	}
 	return nil
 }
@@ -138,7 +138,7 @@ type PyCode struct {
 
 func PyCompile(codeStr string) (*PyCode, error) {
 
-	log.Printf("Gpython compile: %s", codeStr)
+	logger.Debug("Gpython compile: %s", codeStr)
 
 	opts := py.ContextOpts{SysArgs: []string{}, SysPaths: []string{}}
 	ctx := py.NewContext(opts)
@@ -166,9 +166,9 @@ func (p *PyCode) Evaluate(method string, inputs ...map[string]interface{}) (map[
 	out, err := py.Call(fun, in, nil)
 	if err != nil {
 		py.TracebackDump(err)
-		log.Printf("Error Inputs: %#v", inputs)
-		log.Printf("Error Inputs: %#v", in)
-		log.Printf("Map Error: %s", err)
+		logger.Error("Error Inputs: %#v", inputs)
+		logger.Error("Error Inputs: %#v", in)
+		logger.Error("Map Error: %s", err)
 		return nil, err
 	}
 	o := FromPyObject(out)
@@ -188,9 +188,9 @@ func (p *PyCode) EvaluateArray(method string, inputs ...map[string]interface{})
 	out, err := py.Call(fun, in, nil)
 	if err != nil {
 		py.TracebackDump(err)
-		log.Printf("Error Inputs: %#v", inputs)
-		log.Printf("Error Inputs: %#v", in)
-		log.Printf("Map Error: %s", err)
+		logger.Error("Error Inputs: %#v", inputs)
+		logger.Error("Error Inputs: %#v", in)
+		logger.Error("Map Error: %s", err)
 		return nil, err
 	}
 	o := FromPyObject(out)
@@ -210,7 +210,7 @@ func (p *PyCode) EvaluateBool(method string, inputs ...map[string]interface{}) (
 	out, err := py.Call(fun, in, nil)
 	if err != nil {
 		py.TracebackDump(err)
-		log.Printf("Map Error: %s", err)
+		logger.Error("Map Error: %s", err)
 		return false, err
 	}
 	o := FromPyObject(out)

evaluate/python.go

@@ -9,12 +9,12 @@ import (
 	//"bytes"
 	"context"
 	"encoding/json"
-	"io/ioutil"
 	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
 
+	"github.com/bmeg/sifter/logger"
 	grpc "google.golang.org/grpc"
 )
 
@@ -66,7 +66,7 @@ func (d PythonProcessor) EvaluateArray(inputs ...map[string]interface{}) ([]any,
 func (d PythonProcessor) EvaluateBool(inputs ...map[string]interface{}) (bool, error) {
 	i, err := json.Marshal(inputs)
 	if err != nil {
-		log.Printf("Serialization Error: %s", err)
+		logger.Error("Serialization Error: %s", err)
 		return false, err
 	}
 	out, err := d.runner.Call(&Input{Data: string(i), Code: d.fNum})
@@ -106,7 +106,7 @@ type LocalRunner struct {
 
 func StartLocalExecutor(workdir string) (Runner, error) {
 
-	d, err := ioutil.TempDir(workdir, "sifterexec_") //TODO: use directory from user config
+	d, err := os.MkdirTemp(workdir, "sifterexec_") //TODO: use directory from user config
 	if err != nil {
 		return nil, err
 	}
@@ -133,7 +133,7 @@ func StartLocalExecutor(workdir string) (Runner, error) {
 	stdout, _ := cmd.StdoutPipe()
 	err = cmd.Start()
 	if err != nil {
-		log.Printf("Error starting python: %s", err)
+		logger.Error("Error starting python: %s", err)
 		return nil, err
 	}
 
@@ -152,7 +152,7 @@ func StartLocalExecutor(workdir string) (Runner, error) {
 					d := buf[:n]
 					out = append(out, d...)
 				}
-				log.Printf("Read %d (%s) Buffer: %s", n, ierr, string(out))
+				logger.Debug("Read %d (%s) Buffer: %s", n, ierr, string(out))
 				if strings.Contains(string(out), "\n") {
 					t := strings.Split(string(out), "\n")
 					log.Printf("Return port: %s", out)
@@ -164,9 +164,9 @@ func StartLocalExecutor(workdir string) (Runner, error) {
 			}
 			if ierr != nil {
 				if ierr == io.EOF {
-					log.Printf("Executor closed")
+					logger.Debug("Executor closed")
 					if cmd.ProcessState.ExitCode() != 0 {
-						log.Printf("Executor error: %d", cmd.ProcessState.ExitCode())
+						logger.Error("Executor error: %d", cmd.ProcessState.ExitCode())
 					}
 					ierr = nil
 				}