diff --git a/.goreleaser.yml b/.goreleaser.yml index af3639e..f77ad4c 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -3,12 +3,14 @@ before: hooks: # # you may remove this if you don't use vgo - # - go mod download + - go mod download # # you may remove this if you don't need go generate # - go generate ./... builds: - env: - CGO_ENABLED=0 + main: ./cmd/ycat/main.go + archive: replacements: darwin: Darwin diff --git a/README.md b/README.md index ccef7e8..db8b1fb 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,124 @@ -# yjq +# ycat -YAML wrapper for [jq](https://stedolan.github.io/jq/) commandline JSON processor +Comand line processor for YAML/JSON files using [Jsonnet](https://jsonnet.org/) ``` -Usage: yjq [JQ_ARG...] - yjq [options] [YAML_FILES...] -- [JQ_ARG...] +Usage: ycat [options|files...] Options: - --cmd string Name of the jq command (default "jq") - -h, --help Show usage and exit - -i, --yaml-input Convert YAML input to JSON - -o, --yaml-output Convert JSON output to YAML + -h, --help Show help and exit + -y, --yaml [files...] Read YAML values from file(s) + -j, --json [files...] Read JSON values from file(s) + -o, --out {json|j|yaml|y} Set output format + --to-json Output JSON one value per line (same as -o json, -oj) + -a, --array Merge output values into an array + -e, --eval Process values with Jsonnet + -m, --module = Load Jsonnet module into a local Jsonnet variable + --bind Bind input value to a local Jsonnet variable (default _) ``` +### Arguments + +All non-option arguments are files to read from. +Format is guessed from extension and fallsback to YAML. -## Usage +If filename is `-` values are read from stdin using the last specified format or YAML + +## Examples + +Concatenate files to a single YAML stream (type is selected from extension) ``` -yjq [JQ_ARG...] +$ ycat foo.yaml bar.yaml baz.json ``` -By default all arguments are forwarded to `jq` and input/output is YAML. For example: +Concatenate files to single JSON stream (one item per-line) + ``` -$ echo 'name: foo' | yjq '.name+="_bar"' -name: foo_bar +$ ycat --to-json foo.yaml bar.yaml baz.json +$ ycat -o j foo.yaml bar.yaml baz.json ``` -If there's a `--` argument, all arguments up to `--` are handled by `yjq` -and the rest are forwarded to `jq`. +Concatenate JSON values from `stdin` to a single YAML file + ``` -yjq [options] [YAML_FILES...] -- [JQ_ARG...] +$ ycat -j ``` -For example: + +Concatenate YAML from `a.txt`, `stdin`, `b.yaml` and JSON from `a.json`, + ``` -$ echo 'name: foo' | yjq -i -- '.name+="_bar"' -{ - "name": "foo_bar" -} +$ ycat -y a.txt - b.txt -j a.json ``` -### Options - - - `-i`, `--yaml-input` Convert YAML input from `stdin` to JSON - - `-o`, `--yaml-output` Convert JSON output to YAML - - `--cmd ` Specify alternative jq command (default "jq") +Concatenate to YAML array -If neither `-i|--yaml-input` nor `-o|--yaml-output` is specified both input and output is YAML. +``` +$ ycat -a a.json b.yaml +``` -### Arguments - -If `stdin` is not piped, `yjq` arguments are assumed to be YAML files to pipe to `jq` as JSON. If there are no arguments, interactive input from `stdin` is used (exit by sending `EOF` with `Ctrl-D`). +Concatenate YAML from `a.yaml` and `b.yaml` setting key `foo` to `bar` on each top level object -## YAML Input +``` +$ ycat a.yaml b.yaml -e '_+{foo: "bar"}' +``` -Multiple YAML values separated by `---` are passed to `jq` as separate values. -To combine them in an array use `jq`'s `-s|--slurp` option. +Add kubernetes namespace `foo` to all resources without namespace +``` +$ ycat -e '_ + { metadata +: { namespace: if "namespace" in super then super.namespace else "foo" }}' *.yaml +``` -## YAML Output +Process with [jq](http://stedolan.github.io/jq/) using a pipe -Results from `jq`'s output are separated by `---` in a single YAML value stream. +``` +$ ycat -oj a.yaml b.json | jq ... | ycat +``` ## Installation -Download an executable for your platform from github [releases]( https://github.com/alxarch/yjq/releases/latest). +Download an executable for your platform from github [releases]( https://github.com/alxarch/ycat/releases/latest). Alternatively, assuming `$GOPATH/bin` is in `$PATH` just ``` -go get github.com/alxarch/yjq +go get github.com/alxarch/ycat ``` -## Caveats -When input is YAML `jq` argument `-R|--raw-input` is not supported and gets dropped. -When output is YAML the following `jq` arguments ara not supported and get dropped: - - `-r|--raw-output` - - `-a|--ascii-output` - - `-C|--color-output` - - `-j|--join-output` - - `--tab` - - `--indent` - - `-c|--compact-output` +## YAML Input + +Multiple YAML values separated by `---\n` are processed separately. +Value reading stops at `...\n` or `EOF`. + +## JSON Input + +Multiple JSON values separated by whitespace are processed separately. +Value reading stops at `EOF`. + +## YAML Output + +Each result value is appended to the output with `---\n` separator. + +## JSON Output + +Each result value is appended into a new line of output. + +## Jsonnet + +[Jsonnet](https://jsonnet.org/) is a templating language from google that's really versatile in handling configuration files. Visit their site for more information. + +Each value is bound to a local variable named `_` inside the snippet by default. Use `--bind` to change the name. + +To use `Jsonnet` code from a file in the snippet use `-m =` and the exported value will be available as +a local variable in the snippet. ## TODO - - Add support for flow style YAML output, especially in long string values - - Intercept file arguments passed to `jq` and handle the ones with `.yaml,.yml` extension \ No newline at end of file + - Add support for pretty printed output + - Add support for reading .txt files + - Add support for reading files as base64 + - Add support for reading files as hex + - Add support for Jsonnet libraries + - Add support for sorting by JSONPath \ No newline at end of file diff --git a/arguments.go b/arguments.go new file mode 100644 index 0000000..4182268 --- /dev/null +++ b/arguments.go @@ -0,0 +1,217 @@ +package ycat + +import ( + "context" + "fmt" + "os" + "strings" +) + +// Usage for ycat cmd +const Usage = ` +ycat - command line YAML/JSON processor + +Usage: ycat [options|files...] + +Options: + -h, --help Show help and exit + -y, --yaml [files...] Read YAML values from file(s) + -j, --json [files...] Read JSON values from file(s) + -o, --out {json|j|yaml|y} Set output format + --to-json Output JSON one value per line (same as -o json, -oj) + -a, --array Merge output values into an array + -e, --eval Process values with Jsonnet + -m, --module = Load Jsonnet module into a local Jsonnet variable + --bind Bind input value to a local Jsonnet variable (default _) + +If no files are specified values are read from stdin. +Using "-" as a file path will read values from stdin. +Files without a format option will be parsed as YAML unless +they end in ".json". +` + +type Arguments struct { + Help bool + Output Output + Eval Eval + Array bool + Files []InputFile +} + +func (args *Arguments) Parse(argv []string) (err error) { + for err == nil && len(argv) > 0 { + a := argv[0] + argv = argv[1:] + if len(a) > 1 && a[0] == '-' { + switch c := a[1]; c { + case '-': + if len(a) == 2 { + // Special -- arg + for _, a := range argv { + args.addFile(a, Auto) + } + return + } + name, value := splitArgV(a[2:]) + argv, err = args.parseLong(name, value, argv) + default: + argv, err = args.parseShort(a[1:], argv) + } + } else { + args.addFile(a, Auto) + } + + } + return +} + +func splitArgV(s string) (string, string) { + if n := strings.IndexByte(s, '='); 0 <= n && n < len(s) { + return s[:n], s[n+1:] + } + return s, "" +} +func peekArg(args []string) (string, bool) { + if len(args) > 0 { + return args[0], true + } + return "", false +} + +func (args *Arguments) parseLong(name, value string, argv []string) ([]string, error) { + switch name { + case "max-stack": + return nil, nil + case "bind": + value, argv = shiftArgV(value, argv) + args.Eval.Bind = value + case "module": + value, argv = shiftArgV(value, argv) + name, file := splitArgV(value) + if file == "" { + return argv, fmt.Errorf("Invalid module value: %q", value) + } + args.Eval.AddModule(name, file) + case "eval": + if value, argv = shiftArgV(value, argv); value == "--" { + value = strings.Join(argv, " ") + argv = nil + } + args.Eval.Snippet = value + case "output": + value, argv = shiftArgV(value, argv) + if args.Output = OutputFromString(value); args.Output == OutputInvalid { + return argv, fmt.Errorf("Invalid output format: %q", value) + } + case "to-json": + args.Output = OutputJSON + case "help": + args.Help = true + case "array": + args.Array = true + case "yaml": + return args.parseFiles(value, argv, YAML), nil + case "json": + return args.parseFiles(value, argv, JSON), nil + default: + return argv, fmt.Errorf("Invalid option: %q", name) + } + return argv, nil +} + +func (args *Arguments) parseShort(a string, argv []string) ([]string, error) { + for ; len(a) > 0; a = a[1:] { + switch c := a[0]; c { + case 'j': + return args.parseFiles(a[1:], argv, JSON), nil + case 'y': + return args.parseFiles(a[1:], argv, YAML), nil + case 'm': + return args.parseLong("module", a[1:], argv) + case 'e': + return args.parseLong("eval", a[1:], argv) + case 'o': + return args.parseLong("output", a[1:], argv) + case 'a': + args.Array = true + case 'h': + args.Help = true + default: + return argv, fmt.Errorf("Invalid short option: -%c", c) + } + } + return argv, nil +} + +func shiftArgV(v string, argv []string) (string, []string) { + if len(v) > 0 { + if v[0] == '=' { + v = v[1:] + } + } else if len(argv) > 0 && !isOption(argv[0]) { + v = argv[0] + argv = argv[1:] + } + return v, argv +} + +func (args *Arguments) addFile(path string, format Format) { + args.Files = append(args.Files, InputFile{ + Format: format, + Path: path, + }) +} + +func (args *Arguments) parseFiles(path string, argv []string, format Format) []string { + switch { + case len(path) > 0: + if path[0] == '=' { + path = path[1:] + } + args.addFile(path, format) + case len(argv) == 0: + args.addFile("", format) + default: + for ; len(argv) > 0 && !isOption(argv[0]); argv = argv[1:] { + args.addFile(argv[0], format) + } + } + return argv +} + +func isOption(a string) bool { + return len(a) > 1 && a[0] == '-' && (a[1] != '-' || len(a) > 2) +} + +func (args *Arguments) Run(ctx context.Context) <-chan error { + ctx, cancel := withCancel(ctx) + defer cancel() + + var steps []PipelineFunc + if len(args.Files) == 0 { + steps = append(steps, ReadFrom(os.Stdin, YAML)) + } else { + steps = append(steps, ReadFiles(args.Files...)) + } + + if eval, err := args.Eval.Pipeline(); err != nil { + return wrapErr(err) + } else if eval != nil { + steps = append(steps, eval) + } + + if args.Array { + steps = append(steps, ToArray) + } + + switch args.Output { + case OutputJSON: + steps = append(steps, WriteTo(os.Stdout, JSON)) + default: + steps = append(steps, WriteTo(os.Stdout, YAML)) + } + + _, errc := BuildPipeline(ctx, steps...) + return errc + +} diff --git a/cmd/ycat/main.go b/cmd/ycat/main.go new file mode 100644 index 0000000..53bd141 --- /dev/null +++ b/cmd/ycat/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "log" + "os" + + "github.com/alxarch/ycat" +) + +var ( + logger = log.New(os.Stderr, "ycat: ", 0) +) + +func usage() { + os.Stderr.WriteString(ycat.Usage) +} + +func printUsage(err error) { + if err != nil { + logger.Println(err) + } + usage() +} + +func main() { + args := ycat.Arguments{} + if err := args.Parse(os.Args[1:]); err != nil { + printUsage(err) + os.Exit(2) + } + if args.Help { + printUsage(nil) + os.Exit(0) + } + + ok := true + for err := range args.Run(context.Background()) { + if err != nil { + ok = false + logger.Println(err) + } + } + if !ok { + os.Exit(2) + } +} diff --git a/codec.go b/codec.go new file mode 100644 index 0000000..6fa8eba --- /dev/null +++ b/codec.go @@ -0,0 +1,94 @@ +package ycat + +import ( + "encoding/json" + "io" + "strings" + + yaml "gopkg.in/yaml.v2" +) + +// Format is input file format +type Format uint + +const ( + Auto Format = iota + YAML + JSON +) + +func FormatFromString(s string) Format { + switch strings.ToLower(s) { + case "json", "j": + return JSON + case "yaml", "y": + return YAML + default: + return Auto + } +} + +func DetectFormat(path string) Format { + if strings.HasSuffix(path, ".json") { + return JSON + } + return YAML +} + +type InputFile struct { + Format Format + Path string +} + +type Output int + +const ( + OutputInvalid Output = iota - 1 + OutputYAML + OutputJSON + OutputRaw // Only with --eval +) + +func OutputFromString(s string) Output { + switch strings.ToLower(s) { + case "json", "j": + return OutputJSON + case "yaml", "y": + return OutputYAML + case "raw", "r": + return OutputRaw + default: + return OutputInvalid + } +} + +type Decoder interface { + Decode(x interface{}) error +} + +func NewDecoder(r io.Reader, format Format) Decoder { + switch format { + case JSON: + return json.NewDecoder(r) + case YAML: + return yaml.NewDecoder(r) + default: + panic("Invalid format") + } +} + +type Encoder interface { + Encode(x interface{}) error +} + +func NewEncoder(w io.Writer, format Format) Encoder { + switch format { + case JSON: + return json.NewEncoder(w) + case YAML: + return yaml.NewEncoder(w) + default: + panic("Invalid format") + } + +} diff --git a/eval.go b/eval.go new file mode 100644 index 0000000..397275a --- /dev/null +++ b/eval.go @@ -0,0 +1,88 @@ +package ycat + +import ( + "context" + "encoding/json" + "io/ioutil" + "strings" + "text/template" + + jsonnet "github.com/google/go-jsonnet" +) + +// Eval handles Jsonnet snippet evaluation +type Eval struct { + Bind string + Snippet string + Modules map[string]string +} + +// AddModule adds a module to the snippet +func (e *Eval) AddModule(name, path string) { + if e.Modules == nil { + e.Modules = make(map[string]string) + } + e.Modules[name] = path +} + +var tplSnippet = template.Must(template.New("snippet").Parse(` +{{- range $name, $_ := .Modules }} +local {{$name}} = std.extVar("{{$name}}"); +{{- end }} +local {{.Bind}} = std.extVar("{{.Bind}}"); +{{.Snippet}}`)) + +// Render renders the Jsonnet snippet to be executed +func (e *Eval) Render() (string, error) { + w := strings.Builder{} + if err := tplSnippet.Execute(&w, e); err != nil { + return "", err + } + return w.String(), nil +} + +// Pipeline builds a processing pipeline step +func (e *Eval) Pipeline() (PipelineFunc, error) { + if e.Snippet == "" { + return nil, nil + } + if e.Bind == "" { + e.Bind = "_" + } + snippet, err := e.Render() + if err != nil { + return nil, err + } + vm := jsonnet.MakeVM() + //TODO: Add FileImporter + for name, path := range e.Modules { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + vm.ExtCode(name, string(data)) + } + return func(ctx context.Context, in <-chan *Value, out chan<- *Value) error { + for v := range in { + raw, err := json.Marshal(v) + if err != nil { + return err + } + vm.ExtCode(e.Bind, string(raw)) + val, err := vm.EvaluateSnippet("", snippet) + if err != nil { + return err + } + result := new(Value) + if err := json.Unmarshal([]byte(val), result); err != nil { + return err + } + select { + case out <- result: + case <-ctx.Done(): + return nil + } + } + return nil + }, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ae3dfc6 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/alxarch/ycat + +require ( + github.com/google/go-jsonnet v0.12.1 + github.com/sergi/go-diff v1.0.0 // indirect + github.com/stretchr/testify v1.3.0 // indirect + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a17ed85 --- /dev/null +++ b/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-jsonnet v0.12.1 h1:v0iUm/b4SBz7lR/diMoz9tLAz8lqtnNRKIwMrmU2HEU= +github.com/google/go-jsonnet v0.12.1/go.mod h1:gVu3UVSfOt5fRFq+dh9duBqXa5905QY8S1QvMNcEIVs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/yjq/io.go b/internal/yjq/io.go deleted file mode 100644 index 4fe9765..0000000 --- a/internal/yjq/io.go +++ /dev/null @@ -1,78 +0,0 @@ -package yjq - -import ( - "encoding/json" - "io" - "os" - - yaml "gopkg.in/yaml.v2" -) - -func CopyFile(filename string, w io.Writer) error { - f, err := os.Open(filename) - if err != nil { - return err - } - defer f.Close() - return CopyYAMLToJSON(w, f) -} - -func CopyYAMLToJSON(w io.Writer, r io.Reader) (err error) { - dec := yaml.NewDecoder(r) - enc := json.NewEncoder(w) - for { - v := Value{} - if err = dec.Decode(&v); err != nil { - if err == io.EOF { - return nil - } - return - } - if err = enc.Encode(&v); err != nil { - return err - } - } -} - -func CopyJSONToYAML(w io.Writer, r io.Reader) (n int64, err error) { - dec := json.NewDecoder(r) - var ( - data []byte - nn int - ) - - for i := 0; true; i++ { - v := Value{} - if err = dec.Decode(&v); err != nil { - if err == io.EOF { - return 0, nil - } - return - } - if i > 0 { - nn, err = w.Write([]byte{'-', '-', '-', '\n'}) - n += int64(nn) - if err != nil { - return - } - } - if v.Type == Null { - nn, err = w.Write([]byte{'\n'}) - n += int64(nn) - if err != nil { - return - } - continue - } - data, err = yaml.Marshal(&v) - if err != nil { - return - } - nn, err = w.Write(data) - n += int64(nn) - if err != nil { - return - } - } - return -} diff --git a/pipelines.go b/pipelines.go new file mode 100644 index 0000000..3b0191d --- /dev/null +++ b/pipelines.go @@ -0,0 +1,186 @@ +package ycat + +import ( + "context" + "io" + "io/ioutil" + "os" + "sync" +) + +type PipelineFunc func(ctx context.Context, in <-chan *Value, out chan<- *Value) error + +func BuildPipeline(ctx context.Context, steps ...PipelineFunc) (<-chan *Value, <-chan error) { + var ( + last = closedIn() + errs []<-chan error + ) + + for i := range steps { + out := make(chan *Value) + errc := make(chan error, 1) + fn := steps[i] + src := last + go func() { + defer close(errc) + defer close(out) + if err := fn(ctx, src, out); err != nil { + errc <- err + } + }() + last = out + errs = append(errs, errc) + } + return last, MergeErrors(errs...) +} +func closedErrs() <-chan error { + c := make(chan error) + close(c) + return c +} + +func WriteTo(w io.WriteCloser, format Format) PipelineFunc { + enc := NewEncoder(w, format) + return func(ctx context.Context, in <-chan *Value, out chan<- *Value) error { + defer w.Close() + for v := range in { + if err := enc.Encode(v); err != nil { + return err + } + } + return nil + } +} +func closedIn() <-chan *Value { + ch := make(chan *Value) + close(ch) + return ch +} + +func Drain(ctx context.Context, src <-chan *Value, out chan<- *Value) error { + if src == nil { + return nil + } + for v := range src { + select { + case out <- v: + case <-ctx.Done(): + return nil + } + } + return nil +} + +// AndThen starts emitting values from fn after all values from in +// This allows generator Pipeline functions to be added in sequence +func AndThen(fn PipelineFunc) PipelineFunc { + return func(ctx context.Context, in <-chan *Value, out chan<- *Value) error { + if in == nil { + in = closedIn() + } else if err := Drain(ctx, in, out); err != nil { + return err + } + return fn(ctx, in, out) + } +} + +func ReadFiles(files ...InputFile) PipelineFunc { + return func(ctx context.Context, in <-chan *Value, out chan<- *Value) error { + for i := range files { + f := &files[i] + format := f.Format + if format == Auto { + format = DetectFormat(f.Path) + } + + var r io.ReadCloser + switch f.Path { + case "", "-": + r = ioutil.NopCloser(os.Stdin) + default: + f, err := os.Open(f.Path) + if err != nil { + return err + } + r = f + } + if err := ReadFrom(r, format)(ctx, in, out); err != nil { + return err + } + } + return nil + } +} + +func ReadFrom(r io.ReadCloser, format Format) PipelineFunc { + dec := NewDecoder(r, format) + return func(ctx context.Context, _ <-chan *Value, out chan<- *Value) error { + defer r.Close() + for { + v := new(Value) + if err := dec.Decode(v); err != nil { + if err == io.EOF { + return nil + } + return err + } + select { + case out <- v: + case <-ctx.Done(): + return nil + } + } + } +} + +func ToArray(ctx context.Context, in <-chan *Value, out chan<- *Value) error { + var arr []*Value + for v := range in { + arr = append(arr, v) + } + if arr != nil { + select { + case out <- &Value{ + Type: Array, + Value: arr, + }: + case <-ctx.Done(): + return nil + } + } + return nil +} + +func MergeErrors(errcs ...<-chan error) <-chan error { + out := make(chan error, len(errcs)) + wg := sync.WaitGroup{} + wg.Add(len(errcs)) + for i := range errcs { + errc := errcs[i] + go func() { + defer wg.Done() + for err := range errc { + out <- err + } + }() + } + go func() { + defer close(out) + wg.Wait() + }() + return out +} + +func withCancel(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + return context.WithCancel(ctx) +} + +func wrapErr(err error) <-chan error { + c := make(chan error, 1) + c <- err + close(c) + return c +} diff --git a/pipelines_test.go b/pipelines_test.go new file mode 100644 index 0000000..04a00c6 --- /dev/null +++ b/pipelines_test.go @@ -0,0 +1,49 @@ +package ycat_test + +import ( + "context" + "io/ioutil" + "strings" + "testing" + + "github.com/alxarch/ycat" +) + +func TestReadFrom(t *testing.T) { + src := ` +foo: bar +--- +bar: baz +` + r := strings.NewReader(src) + ctx := context.Background() + read := ycat.ReadFrom(ioutil.NopCloser(r), ycat.YAML) + out := make(chan *ycat.Value, 2) + err := read(ctx, nil, out) + if err != nil { + t.Fatal(err) + } + + { + r.Reset(src) + read := ycat.ReadFrom(ioutil.NopCloser(r), ycat.YAML) + out, errs := ycat.BuildPipeline(ctx, read) + if v := <-out; v.Type != ycat.Object { + t.Errorf("Invalid value type: %s", v.Type) + } + if v := <-out; v.Type != ycat.Object { + t.Errorf("Invalid value type: %s", v.Type) + } else if obj, ok := v.Value.(map[string]interface{}); !ok { + t.Errorf("Invalid value type: %s", v.Type) + } else if obj["bar"] != "baz" { + t.Errorf("Invalid value: %v", obj) + } + for e := range errs { + if e != nil { + t.Error(e) + } + } + + } + +} diff --git a/internal/yjq/value.go b/value.go similarity index 92% rename from internal/yjq/value.go rename to value.go index 1a60ff3..a466c20 100644 --- a/internal/yjq/value.go +++ b/value.go @@ -1,4 +1,4 @@ -package yjq +package ycat import "encoding/json" @@ -33,13 +33,8 @@ func (t ValueType) String() string { } type Value struct { - Type ValueType - Value interface{} - Object map[string]interface{} - Array []interface{} - String string - Number float64 - Bool bool + Type ValueType + Value interface{} } func TypeOf(x interface{}) ValueType { @@ -61,6 +56,7 @@ func TypeOf(x interface{}) ValueType { return 0 } } + func (v *Value) UnmarshalJSON(data []byte) error { var x interface{} if err := json.Unmarshal(data, &x); err != nil { @@ -78,9 +74,11 @@ func (v *Value) MarshalJSON() ([]byte, error) { func (v *Value) IsZero() bool { return v.Type == Null } + func (v *Value) MarshalYAML() (interface{}, error) { return v.Value, nil } + func (v *Value) UnmarshalYAML(unmarshal func(interface{}) error) (err error) { var obj map[string]interface{} if err = unmarshal(&obj); err == nil { diff --git a/internal/yjq/value_test.go b/value_test.go similarity index 53% rename from internal/yjq/value_test.go rename to value_test.go index a670e4e..ff63430 100644 --- a/internal/yjq/value_test.go +++ b/value_test.go @@ -1,31 +1,31 @@ -package yjq_test +package ycat_test import ( "reflect" "testing" - "github.com/alxarch/yjq/internal/yjq" + "github.com/alxarch/ycat" yaml "gopkg.in/yaml.v2" ) func TestValue_UnmarshalYAML(t *testing.T) { type TestCase struct { YAML string - Type yjq.ValueType + Type ycat.ValueType Value interface{} Error bool } for _, tc := range []TestCase{ - {`null`, yjq.Null, (interface{})(nil), false}, - {``, yjq.Null, (interface{})(nil), false}, - {`foo: bar`, yjq.Object, map[string]interface{}{"foo": "bar"}, false}, - {`[1,2,3]`, yjq.Array, []interface{}{1, 2, 3}, false}, - {`42.0`, yjq.Number, 42.0, false}, - {`"foo"`, yjq.String, "foo", false}, - {`true`, yjq.Boolean, true, false}, - {`false`, yjq.Boolean, false, false}, + {`null`, ycat.Null, (interface{})(nil), false}, + {``, ycat.Null, (interface{})(nil), false}, + {`foo: bar`, ycat.Object, map[string]interface{}{"foo": "bar"}, false}, + {`[1,2,3]`, ycat.Array, []interface{}{1, 2, 3}, false}, + {`42.0`, ycat.Number, 42.0, false}, + {`"foo"`, ycat.String, "foo", false}, + {`true`, ycat.Boolean, true, false}, + {`false`, ycat.Boolean, false, false}, } { - v := yjq.Value{} + v := ycat.Value{} err := yaml.Unmarshal([]byte(tc.YAML), &v) if err != nil { t.Errorf("%q Unexpected error: %s", tc.YAML, err) diff --git a/yjq.go b/yjq.go deleted file mode 100644 index f0e8f14..0000000 --- a/yjq.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "fmt" - "io" - "log" - "os" - "os/exec" - "sync" - "syscall" - - "github.com/alxarch/yjq/internal/yjq" - "github.com/spf13/pflag" -) - -var ( - logger = log.New(os.Stderr, "yjq: ", 0) - flags = pflag.NewFlagSet("yjq", pflag.ContinueOnError) - cmdName = flags.String("cmd", "jq", "Name of the jq command") - input = flags.BoolP("yaml-input", "i", false, "Convert YAML input to JSON") - output = flags.BoolP("yaml-output", "o", false, "Convert JSON output to YAML") - help = flags.BoolP("help", "h", false, "Show usage and exit") - files []string - cmdArgs []string - args []string -) - -func init() { - flags.Usage = func() { - fmt.Fprintln(os.Stderr, ` -yjq - YAML wrapper for jq commandline JSON processor - -Usage: yjq [options] [YAML_FILES...] -- [JQ_ARG...] - yjq [JQ_ARG...] - -Options:`) - flags.PrintDefaults() - } -} - -func main() { - flags.ParseErrorsWhitelist.UnknownFlags = true - args = os.Args[1:] - if hasArg(args, "--help", "-h") { - flags.Usage() - os.Exit(0) - } - if n := indexOf(args, "--"); n == -1 { - cmdArgs = args - args = args[:0] - } else { - cmdArgs = args[n+1:] - args = args[:n] - } - err := flags.Parse(args) - if err != nil { - logger.Println(err) - flags.Usage() - os.Exit(2) - } - if files = flags.Args(); len(files) > 0 { - *input = true - } - tty, err := isStdinTTY() - if err != nil { - logger.Println(err) - os.Exit(2) - } - - if *input == *output { - *input = true - *output = true - } - - cmdArgs, *input, *output = rewriteArgs(cmdArgs, *input, *output) - cmd := exec.Command(*cmdName, cmdArgs...) - cmd.Stderr = os.Stderr - wg := new(sync.WaitGroup) - var ( - jqw io.WriteCloser - jqr io.ReadCloser - ) - if *input { - if jqw, err = cmd.StdinPipe(); err != nil { - onExit(err) - } - } else { - cmd.Stdin = os.Stdin - } - if *output { - if jqr, err = cmd.StdoutPipe(); err != nil { - onExit(err) - } - } else { - cmd.Stdout = os.Stdout - } - - if err := cmd.Start(); err != nil { - onExit(err) - } - - if *input { - wg.Add(1) - go func() { - defer wg.Done() - defer jqw.Close() - if tty && len(files) > 0 { - for _, filename := range files { - f, err := os.Open(filename) - if err != nil { - logger.Printf("Failed to open file %q: %s\n", filename, err) - os.Exit(2) - } - defer f.Close() - if err := yjq.CopyYAMLToJSON(jqw, f); err != nil { - logger.Printf("Failed to parse file %q: %s\n", filename, err) - os.Exit(2) - } - } - } else { - if err := yjq.CopyYAMLToJSON(jqw, os.Stdin); err != nil { - logger.Println("Failed to read input", err) - os.Exit(1) - } - } - }() - } - - if *output { - wg.Add(1) - go func() { - defer wg.Done() - defer jqr.Close() - if _, err := yjq.CopyJSONToYAML(os.Stdout, jqr); err != nil { - logger.Printf("Failed to write to stdout: %s", err) - os.Exit(1) - } - }() - wg.Wait() - } - - if err := cmd.Wait(); err != nil { - onExit(err) - } -} - -func exitCode(err error) int { - if err, ok := err.(*exec.ExitError); ok { - if code, ok := err.ProcessState.Sys().(syscall.WaitStatus); ok { - return int(code) - } - } - return 2 -} - -func onExit(err error) { - switch code := exitCode(err); code { - case 2, 3: - logger.Println(err) - fallthrough - default: - os.Exit(code) - } -} - -func hasArg(args []string, s ...string) bool { - for _, s := range s { - if indexOf(args, s) != -1 { - return true - } - } - return false -} -func omitArgs(args []string, omit ...string) (out []string) { - for _, a := range args { - if indexOf(omit, a) == -1 { - out = append(out, a) - } - } - return -} -func indexOf(values []string, s string) int { - for i := range values { - if values[i] == s { - return i - } - } - return -1 -} - -func injectArgs(args []string, inject ...string) (out []string) { - out = append(out, inject...) - for _, a := range args { - if indexOf(inject, a) == -1 { - out = append(out, a) - } - } - return -} - -func rewriteArgs(args []string, input, output bool) ([]string, bool, bool) { - switch { - case hasArg(args, "--help", "-h"): - return []string{"--help"}, false, false - case hasArg(args, "--version"): - return []string{"--version"}, false, false - case hasArg(args, "--run-tests"): - return []string{"--run-tests"}, false, false - } - if hasArg(args, "--null-input", "-n") { - input = false - } - if input { - args = omitArgs(args, "--raw-input", "-R") - } - if output { - args = omitArgs(args, - "--color-output", "-C", - "--tab", - "--ascii-output", "-a", - "--join-output", "-j", - "--raw-output", "-r", - ) - args = injectArgs(args, "--unbuffered", "--compact-output") - } - return args, input, output -} - -func isStdinTTY() (bool, error) { - info, err := os.Stdin.Stat() - if err != nil { - return false, err - } - if info.Mode()&os.ModeCharDevice == 0 { - return false, nil - } - return true, nil -}