From 829aa8fb3dc34ce0db63c14cf5cded267513c8db Mon Sep 17 00:00:00 2001 From: nexus166 Date: Thu, 22 Jul 2021 20:01:46 +0200 Subject: [PATCH] lib: expose SetOutput for default logger xprint: various updates to template functions, tracking etc --- cmd/xprint/main.go | 23 +- cmd/xprint/template_funcmap.go | 751 ++++++++++++++++++++++----------- fmt.go | 2 +- log.go | 5 + 4 files changed, 538 insertions(+), 243 deletions(-) diff --git a/cmd/xprint/main.go b/cmd/xprint/main.go index 24d6df7..f84c766 100644 --- a/cmd/xprint/main.go +++ b/cmd/xprint/main.go @@ -5,12 +5,12 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "io/ioutil" "os" "strconv" - "strings" "text/template" log "github.com/szampardi/msg" @@ -27,6 +27,7 @@ var ( _template *template.Template // unsafe *bool = flag.Bool("u", false, "allow evaluation of dangerous template functions such as cmd,env") // showFns *bool = flag.Bool("F", false, "print available template functions and exit") // + debug *bool = flag.Bool("D", false, "debug template rendering activities") // output *os.File // argsfirst *bool = flag.Bool("a", false, "output arguments (if any) before stdin (if any), instead of the opposite") // showVersion *bool = flag.Bool("v", false, "print build version/date and exit") @@ -61,11 +62,14 @@ func setFlags() { "t", "template (string or file)", func(value string) error { + if *debug { + go usageDebugger() + } _, err := os.Stat(value) if err == nil { - _template, err = template.New(value).Funcs(tplFuncMap).ParseFiles(value) + _template, err = template.New(value).Funcs(buildFuncMap(*unsafe)).ParseFiles(value) } else { - _template, err = template.New(os.Args[0]).Funcs(tplFuncMap).Parse(value) + _template, err = template.New(os.Args[0]).Funcs(buildFuncMap(*unsafe)).Parse(value) } return err }, @@ -108,11 +112,13 @@ func init() { os.Exit(0) } if *showFns { - fns := []string{} - for s := range tplFuncMap { - fns = append(fns, s) + enc := json.NewEncoder(os.Stderr) + enc.SetIndent("", " ") + err := enc.Encode(templateFnsInfo) + if err != nil { + panic(err) } - fmt.Fprintf(os.Stderr, "%v", strings.Join(fns, "\n")) + os.Exit(0) } if err := log.IsValidLevel(int(loglvl)); err != nil { panic(err) @@ -148,6 +154,9 @@ func main() { if err := _template.Execute(buf, data); err != nil { panic(err) } + if *debug { + trackWg.Wait() + } } else { for _, s := range dataIndex { _, err := fmt.Fprintf(buf, "%s", data[s]) diff --git a/cmd/xprint/template_funcmap.go b/cmd/xprint/template_funcmap.go index 31e4a0e..b8652b2 100644 --- a/cmd/xprint/template_funcmap.go +++ b/cmd/xprint/template_funcmap.go @@ -18,261 +18,542 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strconv" "strings" + "sync" "text/template" + "time" + log "github.com/szampardi/msg" "gopkg.in/yaml.v2" ) -var tplFuncMap template.FuncMap = template.FuncMap{ - "tojson": func(in interface{}) (string, error) { - b, err := json.Marshal(in) - if err != nil { - return "", err - } - return string(b), nil +type fn struct { + _fn interface{} `json:"-"` + Description string `json:"description"` + Signature string `json:"function"` + Unsafe bool `json:"unsafe"` +} + +var templateFnsInfo = map[string]fn{ + "add": { + add, + "add value $2 to map or slice $1, map needs $3 for value's key in map", + reflect.TypeOf(add).String(), + false, }, - "fromjson": func(in string) (map[string]interface{}, error) { - var out map[string]interface{} - if err := json.Unmarshal([]byte(in), &out); err != nil { - return nil, err - } - return out, nil + "b64dec": { + b64dec, + "base64 decode", + reflect.TypeOf(b64dec).String(), + false, }, - "toyaml": func(in interface{}) (string, error) { - b, err := yaml.Marshal(in) - if err != nil { - return "", err - } - return string(b), nil + "b64enc": { + b64enc, + "base64 encode", + reflect.TypeOf(b64enc).String(), + false, }, - "fromyaml": func(in string) (map[string]interface{}, error) { - var out map[string]interface{} - if err := yaml.Unmarshal([]byte(in), &out); err != nil { - return nil, err - } - return out, nil + "cmd": { + cmd, + "execute a command on local host", + reflect.TypeOf(cmd).String(), + true, }, - "b64enc": func(in interface{}) (string, error) { - var b []byte - switch t := in.(type) { - case string: - b = make([]byte, base64.StdEncoding.EncodedLen(len(t))) - base64.StdEncoding.Encode(b, []byte(t)) - case []byte: - b = make([]byte, base64.StdEncoding.EncodedLen(len(t))) - base64.StdEncoding.Encode(b, t) - default: - return "", fmt.Errorf("can only work with string or []byte, not %T", t) - } - return string(b), nil + "decrypt": { + decrypt, + "decrypt data with AES_GCM: $1 ctxt, $2 base64 key, $3 AAD", + reflect.TypeOf(decrypt).String(), + false, }, - "b64dec": func(in interface{}) ([]byte, error) { - var b []byte - var n int - var err error - switch t := in.(type) { - case string: - b = make([]byte, base64.StdEncoding.DecodedLen(len(t))) - n, err = base64.StdEncoding.Decode(b, []byte(t)) - case []byte: - b = make([]byte, base64.StdEncoding.DecodedLen(len(t))) - n, err = base64.StdEncoding.Decode(b, t) - default: - return nil, fmt.Errorf("can only work with string or []byte, not %T", t) - } - if err != nil { - return nil, err - } - return b[:n], nil + "encrypt": { + encrypt, + "encrypt data with AES_GCM: $1 ptxt, $2 base64 key, $3 AAD", + reflect.TypeOf(encrypt).String(), + false, }, - "hexenc": func(in interface{}) (string, error) { - switch t := in.(type) { - case string: - return hex.EncodeToString([]byte(t)), nil - case []byte: - return hex.EncodeToString(t), nil - default: - return "", fmt.Errorf("can only work with string or []byte, not %T", t) - } + "env": { + env, + "get environment vars, optionally use a placeholder value $2", + reflect.TypeOf(env).String(), + true, }, - "hexdec": func(in interface{}) ([]byte, error) { - var b []byte - var n int - var err error - switch t := in.(type) { - case string: - b, err = hex.DecodeString(t) - case []byte: - b = make([]byte, hex.DecodedLen(len(t))) - n, err = hex.Decode(b, t) - default: - return nil, fmt.Errorf("can only work with string or []byte, not %T", t) - } - if err != nil { - return nil, err - } - return b[:n], nil + "fromjson": { + fromjson, + "json decode", + reflect.TypeOf(fromjson).String(), + false, }, - "gzip": func(in interface{}) ([]byte, error) { - var todo []byte - switch t := in.(type) { - case string: - todo = []byte(t) - case []byte: - todo = t - } - out := new(bytes.Buffer) - gzw, err := gzip.NewWriterLevel(out, gzip.BestCompression) - if err != nil { - return nil, err - } - _, err = io.Copy(gzw, bytes.NewBuffer(todo)) - if err != nil { - return nil, err - } - if err = gzw.Flush(); err != nil { - return nil, err - } - if err = gzw.Close(); err != nil { - return nil, err - } - return out.Bytes(), nil + "fromyaml": { + fromyaml, + "yaml decode", + reflect.TypeOf(fromyaml).String(), + false, }, - "gunzip": func(in []byte) ([]byte, error) { - gzr, err := gzip.NewReader(bytes.NewBuffer(in)) - if err != nil { - return nil, err - } - out := new(bytes.Buffer) - _, err = io.Copy(out, gzr) - if err != nil { - return nil, err - } - if err = gzr.Close(); err != nil { - return nil, err - } - return out.Bytes(), nil + "gunzip": { + _gunzip, + "extract GZIP compressed data", + reflect.TypeOf(_gunzip).String(), + false, }, - "rawfile": func(in string) ([]byte, error) { - f, err := os.Open(in) - if err != nil { - return nil, err - } - return ioutil.ReadAll(f) + "gzip": { + _gzip, + "compress with GZIP", + reflect.TypeOf(_gzip).String(), + false, }, - "textfile": func(in string) (string, error) { - f, err := os.Open(in) - if err != nil { - return "", err - } - data, err := ioutil.ReadAll(f) - if err != nil { - return "", err - } - return string(data), nil + "hexdec": { + hexdec, + "hex decode", + reflect.TypeOf(hexdec).String(), + false, }, - "writefile": func(in interface{}, fpath string) (string, error) { - f, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, os.FileMode(0600)) - if err != nil { - return "", err - } - var buf *bytes.Buffer - switch t := in.(type) { - case string: - buf = bytes.NewBuffer([]byte(t)) - case []byte: - buf = bytes.NewBuffer(t) - default: - return "", fmt.Errorf("can only work with string or []byte, not %T", t) - } - _, err = io.Copy(f, buf) - return f.Name(), err + "hexenc": { + hexenc, + "hex encode", + reflect.TypeOf(hexenc).String(), + false, }, - "string": func(in interface{}) (string, error) { - switch t := in.(type) { - case string: - return t, nil - case int: - return strconv.Itoa(t), nil - case bool: - return strconv.FormatBool(t), nil - case []byte: - return string(t), nil - default: - return "", fmt.Errorf("string: can only work with int, bool and []byte, not %T", t) - } + "join": { + strings.Join, + "strings.Join", + reflect.TypeOf(strings.Join).String(), + false, }, - "encrypt": func(in interface{}, b64key string, aad string) ([]byte, error) { - var ptxt []byte - switch t := in.(type) { - case string: - ptxt = []byte(t) - case []byte: - ptxt = t - default: - return nil, fmt.Errorf("can only work with string or []byte, not %T", t) - } - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - return nil, err - } - cb, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - aead, err := cipher.NewGCM(cb) - if err != nil { - return nil, err - } - iv := random(aead.NonceSize()) - var ctxt []byte - ctxt = aead.Seal(ctxt, iv, ptxt, []byte(aad)) - return append(ctxt, iv...), nil + "lower": { + strings.ToLower, + "strings.ToLower", + reflect.TypeOf(strings.ToLower).String(), + false, }, - "decrypt": func(ctxt []byte, b64key string, aad string) ([]byte, error) { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - return nil, err - } - cb, err := aes.NewCipher(key) - if err != nil { - return nil, err - } - aead, err := cipher.NewGCM(cb) - if err != nil { - return nil, err - } - ns := aead.NonceSize() - l := len(ctxt) - if l < ns { - return nil, io.ErrUnexpectedEOF + "pathbase": { + filepath.Base, + "filepath.Base", + reflect.TypeOf(filepath.Base).String(), + false, + }, + "pathext": { + filepath.Ext, + "filepath.Ext", + reflect.TypeOf(filepath.Ext).String(), + false, + }, + "random": { + random, + "generate a $1 sized []byte filled with bytes from crypto.Rand", + reflect.TypeOf(random).String(), + false, + }, + "rawfile": { + rawfile, + "read raw bytes from a file", + reflect.TypeOf(rawfile).String(), + true, + }, + "split": { + strings.Split, + "strings.Split", + reflect.TypeOf(strings.Split).String(), + false, + }, + "string": { + stringify, + "convert int/bool to string, retype []byte to string (handle with care)", + reflect.TypeOf(stringify).String(), + false, + }, + "textfile": { + textfile, + "read a file as a string", + reflect.TypeOf(textfile).String(), + true, + }, + "tojson": { + tojson, + "json encode", + reflect.TypeOf(tojson).String(), + false, + }, + "toyaml": { + toyaml, + "yaml encode", + reflect.TypeOf(toyaml).String(), + false, + }, + "trimprefix": { + strings.TrimPrefix, + "strings.TrimPrefix", + reflect.TypeOf(strings.TrimPrefix).String(), + false, + }, + "trimsuffix": { + strings.TrimSuffix, + "strings.TrimSuffix", + reflect.TypeOf(strings.TrimSuffix).String(), + false, + }, + "upper": { + strings.ToUpper, + "strings.ToUpper", + reflect.TypeOf(strings.ToUpper).String(), + false, + }, + "writefile": { + writefile, + "store data to a file (append if it already exists)", + reflect.TypeOf(writefile).String(), + true, + }, +} + +func buildFuncMap(addUnsafe bool) template.FuncMap { + m := make(template.FuncMap) + for name, info := range templateFnsInfo { + if !info.Unsafe || addUnsafe { + m[name] = info._fn } - var ptxt []byte - ptxt, err = aead.Open(ptxt, ctxt[l-ns:], ctxt[:l-ns], []byte(aad)) - if err != nil { - return nil, err + } + return m +} + +type fnTrack struct { + T time.Time `json:"time"` + F string `json:"function"` + Args []interface{} `json:"args,omitempty"` + Output interface{} `json:"output,omitempty"` + Err error `json:"error,omitempty"` +} + +var ( + fnTrackChan chan (*fnTrack) = make(chan *fnTrack) + trackWg *sync.WaitGroup = &sync.WaitGroup{} +) + +func trackUsage(_fn string, output interface{}, err error, args ...interface{}) { + if *debug { + trackWg.Add(1) + fnTrackChan <- &fnTrack{ + T: time.Now(), + F: _fn, + Args: args, + Output: output, + Err: err, } - return ptxt, nil - }, - "env": env, - "cmd": cmd, - "random": random, - "join": strings.Join, - "split": strings.Split, - "trimprefix": strings.TrimPrefix, - "trimsuffix": strings.TrimSuffix, - "lower": strings.ToLower, - "upper": strings.ToUpper, - "pathbase": filepath.Base, - "pathext": filepath.Ext, + } +} + +func usageDebugger() { + log.SetOutput(os.Stderr) + for x := range fnTrackChan { + j, _ := json.Marshal(x) + log.Warningf(string(j)) + trackWg.Done() + } +} + +func tojson(in interface{}) (out string, err error) { + defer trackUsage("tojson", out, err, in) + b, err := json.Marshal(in) + if err != nil { + return "", err + } + out = string(b) + return out, nil +} + +func fromjson(in string) (out map[string]interface{}, err error) { + defer trackUsage("fromjson", out, err, in) + if err := json.Unmarshal([]byte(in), &out); err != nil { + return nil, err + } + return out, nil } -func env(in string, or ...string) string { - if !*unsafe { - panic(*unsafe) +func toyaml(in interface{}) (out string, err error) { + defer trackUsage("toyaml", out, err, in) + b, err := yaml.Marshal(in) + if err != nil { + return "", err + } + out = string(b) + return out, nil +} + +func fromyaml(in string) (out map[string]interface{}, err error) { + defer trackUsage("fromyaml", out, err, in) + if err := yaml.Unmarshal([]byte(in), &out); err != nil { + return nil, err } + return out, nil +} + +func b64enc(in interface{}) (out string, err error) { + defer trackUsage("b64enc", out, err, in) + var b []byte + switch t := in.(type) { + case string: + b = make([]byte, base64.StdEncoding.EncodedLen(len(t))) + base64.StdEncoding.Encode(b, []byte(t)) + case []byte: + b = make([]byte, base64.StdEncoding.EncodedLen(len(t))) + base64.StdEncoding.Encode(b, t) + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return "", err + } + out = string(b) + return out, nil +} + +func b64dec(in interface{}) (out []byte, err error) { + defer trackUsage("b64dec", out, err, in) + var b []byte + var n int + switch t := in.(type) { + case string: + b = make([]byte, base64.StdEncoding.DecodedLen(len(t))) + n, err = base64.StdEncoding.Decode(b, []byte(t)) + case []byte: + b = make([]byte, base64.StdEncoding.DecodedLen(len(t))) + n, err = base64.StdEncoding.Decode(b, t) + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return nil, err + } + if err != nil { + return nil, err + } + out = b[:n] + return out, nil +} + +func hexenc(in interface{}) (out string, err error) { + defer trackUsage("hexenc", out, err, in) + switch t := in.(type) { + case string: + out = hex.EncodeToString([]byte(t)) + case []byte: + out = hex.EncodeToString(t) + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return "", err + } + return out, nil +} + +func hexdec(in interface{}) (out []byte, err error) { + defer trackUsage("hexdec", out, err, in) + var b []byte + var n int + switch t := in.(type) { + case string: + b, err = hex.DecodeString(t) + case []byte: + b = make([]byte, hex.DecodedLen(len(t))) + n, err = hex.Decode(b, t) + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return nil, err + } + if err != nil { + return nil, err + } + out = b[:n] + return out, nil +} + +func _gzip(in interface{}) (out []byte, err error) { + defer trackUsage("gzip", out, err, in) + var todo []byte + switch t := in.(type) { + case string: + todo = []byte(t) + case []byte: + todo = t + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return nil, err + } + buf := new(bytes.Buffer) + gzw, err := gzip.NewWriterLevel(buf, gzip.BestCompression) + if err != nil { + return nil, err + } + _, err = io.Copy(gzw, bytes.NewBuffer(todo)) + if err != nil { + return nil, err + } + if err = gzw.Flush(); err != nil { + return nil, err + } + if err = gzw.Close(); err != nil { + return nil, err + } + out = buf.Bytes() + return out, nil +} + +func _gunzip(in []byte) (out []byte, err error) { + defer trackUsage("gunzip", out, err, in) + gzr, err := gzip.NewReader(bytes.NewBuffer(in)) + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + _, err = io.Copy(buf, gzr) + if err != nil { + return nil, err + } + if err = gzr.Close(); err != nil { + return nil, err + } + out = buf.Bytes() + return out, nil +} + +func rawfile(in string) (out []byte, err error) { + defer trackUsage("rawfile", out, err, in) + f, err := os.Open(in) + if err != nil { + return nil, err + } + out, err = ioutil.ReadAll(f) + return out, err +} + +func textfile(in string) (out string, err error) { + defer trackUsage("textfile", out, err, in) + f, err := os.Open(in) + if err != nil { + return "", err + } + data, err := ioutil.ReadAll(f) + if err != nil { + return "", err + } + out = string(data) + return out, nil +} + +func writefile(in interface{}, fpath string) (out string, err error) { + defer trackUsage("writefile", "", err, in, fpath) + f, err := os.OpenFile(fpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.FileMode(0600)) + if err != nil { + return "", err + } + var buf *bytes.Buffer + switch t := in.(type) { + case string: + buf = bytes.NewBuffer([]byte(t)) + case []byte: + buf = bytes.NewBuffer(t) + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return "", err + } + _, err = io.Copy(f, buf) + return "", err +} + +func stringify(in interface{}) (out string, err error) { + defer trackUsage("string", out, err, in) + switch t := in.(type) { + case string: + out = t + case int: + out = strconv.Itoa(t) + case bool: + out = strconv.FormatBool(t) + case []byte: + out = string(t) + default: + err = fmt.Errorf("invalid argument %T, supported types: int, bool and []byte", t) + return "", err + } + return out, nil +} + +func encrypt(in interface{}, b64key string, aad string) (out []byte, err error) { + defer trackUsage("encrypt", out, err, in, b64key, aad) + var ptxt []byte + switch t := in.(type) { + case string: + ptxt = []byte(t) + case []byte: + ptxt = t + default: + err = fmt.Errorf("invalid argument %T, supported types: string or []byte", t) + return nil, err + } + key, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + return nil, err + } + cb, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + aead, err := cipher.NewGCM(cb) + if err != nil { + return nil, err + } + iv := random(aead.NonceSize()) + var ctxt []byte + ctxt = aead.Seal(ctxt, iv, ptxt, []byte(aad)) + out = append(ctxt, iv...) + return out, nil +} + +func decrypt(ctxt []byte, b64key string, aad string) (out []byte, err error) { + defer trackUsage("decrypt", out, err, ctxt, b64key, aad) + key, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + return nil, err + } + cb, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + aead, err := cipher.NewGCM(cb) + if err != nil { + return nil, err + } + ns := aead.NonceSize() + l := len(ctxt) + if l < ns { + return nil, io.ErrUnexpectedEOF + } + out, err = aead.Open(out, ctxt[l-ns:], ctxt[:l-ns], []byte(aad)) + if err != nil { + return nil, err + } + return out, nil +} + +func add(in interface{}, value interface{}, key ...interface{}) (out interface{}, err error) { + defer trackUsage("add", out, err, in, value, key) + switch t := in.(type) { + case map[interface{}]interface{}: + if len(key) < 1 { + return nil, fmt.Errorf("must provide a key for value to be added") + } + // should be safe? + t[key[0]] = value + out = t + return t, nil + case []interface{}: + t = append(t, value) + out = t + return out, nil + default: + err = fmt.Errorf("invalid argument %T, supported types: slices and maps", t) + return nil, err + } +} + +func env(in string, or ...string) (out string) { + defer trackUsage("env", out, nil, in, or) if v, ok := os.LookupEnv(in); ok { return v } @@ -282,25 +563,25 @@ func env(in string, or ...string) string { return "" } -func cmd(prog string, args ...string) (string, error) { - if !*unsafe { - panic(*unsafe) - } +func cmd(prog string, args ...string) (out string, err error) { + defer trackUsage("cmd", out, err, prog, args) x := exec.Command(prog, args...) outbuf, errbuf := new(bytes.Buffer), new(bytes.Buffer) x.Stderr = errbuf x.Stdout = outbuf - err := x.Run() + err = x.Run() if err != nil { return "", err } if errbuf.Len() > 0 { err = fmt.Errorf("%s error: %s", prog, errbuf.String()) } - return outbuf.String(), err + out = outbuf.String() + return out, err } -func random(size int) []byte { +func random(size int) (out []byte) { + defer trackUsage("random", out, nil, size) buf := make([]byte, size) _, err := io.ReadFull(rand.Reader, buf) if err != nil { diff --git a/fmt.go b/fmt.go index ebd1a5b..0b4df72 100644 --- a/fmt.go +++ b/fmt.go @@ -89,7 +89,7 @@ var ( var ( logNo uint32 - activeFormat string = Formats[StdFormat]._String + activeFormat string = Formats[PlainFormat]._String activeTimeFormat string = Formats[DefTimeFmt]._String ) diff --git a/log.go b/log.go index 3e26670..b6b7276 100644 --- a/log.go +++ b/log.go @@ -5,6 +5,7 @@ package log import ( "fmt" + "io" "os" "runtime" ) @@ -202,6 +203,10 @@ func StackAsCritical(message string) { defaultLogger.logInternal(LCrit, stack(message), 2) } +func SetOutput(w io.Writer) { + defaultLogger.SetOutput(w) +} + func stack(s string) string { if s == "" { s = "Stack info\n"