Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sifter scanning commands [WIP] #62

Merged
merged 14 commits into from
Jan 30, 2024
Prev Previous commit
Next Next commit
Fixing lint issues
kellrott committed Jan 30, 2024
commit bf7c7877d25923bf07985460e57203f9ad84b1e9
2 changes: 1 addition & 1 deletion cmd/graph_plan/main.go → cmd/graphplan/main.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package graph_plan
package graphplan

import (
"log"
4 changes: 2 additions & 2 deletions cmd/root.go
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ package cmd
import (
"os"

"github.com/bmeg/sifter/cmd/graph_plan"
"github.com/bmeg/sifter/cmd/graphplan"
"github.com/bmeg/sifter/cmd/inspect"
"github.com/bmeg/sifter/cmd/run"
"github.com/bmeg/sifter/cmd/scan"
@@ -20,7 +20,7 @@ var RootCmd = &cobra.Command{
func init() {
RootCmd.AddCommand(run.Cmd)
RootCmd.AddCommand(inspect.Cmd)
RootCmd.AddCommand(graph_plan.Cmd)
RootCmd.AddCommand(graphplan.Cmd)
RootCmd.AddCommand(scan.Cmd)
}

2 changes: 1 addition & 1 deletion cmd/run/run.go
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ func ExecuteFile(playFile string, workDir string, outDir string, inputs map[stri
a, _ := filepath.Abs(playFile)
baseDir := filepath.Dir(a)
log.Printf("basedir: %s", baseDir)
log.Printf("playbook: %s", pb)
log.Printf("playbook: %#v", pb)
return Execute(pb, baseDir, workDir, outDir, inputs)
}

12 changes: 6 additions & 6 deletions cmd/scan/main.go
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ var jsonOut = false
var objectsOnly = false
var baseDir = ""

type ScanEntry struct {
type Entry struct {
ObjectType string `json:"objectType"`
SifterFile string `json:"sifterFile"`
Outfile string `json:"outFile"`
@@ -31,9 +31,9 @@ var ObjectCommand = &cobra.Command{

scanDir := args[0]

outputs := []ScanEntry{}
outputs := []Entry{}

ScanSifter(scanDir, func(pb *playbook.Playbook) {
PathWalker(scanDir, func(pb *playbook.Playbook) {
for pname, p := range pb.Pipelines {
emitName := ""
for _, s := range p {
@@ -46,7 +46,7 @@ var ObjectCommand = &cobra.Command{
outdir := pb.GetDefaultOutDir()
outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName)
outpath := filepath.Join(outdir, outname)
o := ScanEntry{SifterFile: pb.GetPath(), Outfile: outpath}
o := Entry{SifterFile: pb.GetPath(), Outfile: outpath}
if s.ObjectValidate != nil {
//outpath, _ = filepath.Rel(baseDir, outpath)
//fmt.Printf("%s\t%s\n", s.ObjectValidate.Title, outpath)
@@ -127,7 +127,7 @@ var ScriptCommand = &cobra.Command{

userInputs := map[string]string{}

ScanSifter(scanDir, func(pb *playbook.Playbook) {
PathWalker(scanDir, func(pb *playbook.Playbook) {
path := pb.GetPath()
scriptDir := filepath.Dir(path)

@@ -203,7 +203,7 @@ func init() {

}

func ScanSifter(baseDir string, userFunc func(*playbook.Playbook)) {
func PathWalker(baseDir string, userFunc func(*playbook.Playbook)) {
filepath.Walk(baseDir,
func(path string, info fs.FileInfo, err error) error {
if strings.HasSuffix(path, ".yaml") {
6 changes: 3 additions & 3 deletions task/task.go
Original file line number Diff line number Diff line change
@@ -107,10 +107,10 @@ func (m *Task) BaseDir() string {

func (m *Task) Emit(n string, e map[string]interface{}, useName bool) error {

new_name := m.GetName() + "." + n
newName := m.GetName() + "." + n
if useName {
temp := strings.Split(n, ".")
new_name = temp[len(temp)-1]
newName = temp[len(temp)-1]
}
return m.Emitter.Emit(new_name, e, useName)
return m.Emitter.Emit(newName, e, useName)
}
2 changes: 1 addition & 1 deletion transform/object_validate.go
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ import (
"github.com/bmeg/sifter/evaluate"
"github.com/bmeg/sifter/task"
"github.com/santhosh-tekuri/jsonschema/v5"
_ "github.com/santhosh-tekuri/jsonschema/v5/httploader"
_ "github.com/santhosh-tekuri/jsonschema/v5/httploader" // setup the httploader for the jsonschema checker
)

type ObjectValidateStep struct {

Unchanged files with check annotations Beta

go func() {
defer fhd.Close()
defer close(out)
tableColumns := map[string][]string{}

Check failure on line 62 in extractors/sqldump_step.go

GitHub Actions / lint

tableColumns declared but not used (typecheck)
data, _ := io.ReadAll(hd)
tokens := sqlparser.NewStringTokenizer(string(data))
for {