From 57a19692ea4a860e40c15a658e65092e1e285c78 Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Mon, 4 Nov 2024 23:51:55 -0800 Subject: [PATCH 1/3] add a processor to convert binary ieeefloat32 values to float --- pkg/formatters/all/all.go | 1 + .../event_ieeefloat32/event_ieeefloat32.go | 169 ++++++++++++++++++ .../event_ieeefloat32_test.go | 104 +++++++++++ pkg/formatters/processors.go | 1 + 4 files changed, 275 insertions(+) create mode 100644 pkg/formatters/event_ieeefloat32/event_ieeefloat32.go create mode 100644 pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go diff --git a/pkg/formatters/all/all.go b/pkg/formatters/all/all.go index 2e93c1a1..54a66fdb 100644 --- a/pkg/formatters/all/all.go +++ b/pkg/formatters/all/all.go @@ -20,6 +20,7 @@ import ( _ "github.com/openconfig/gnmic/pkg/formatters/event_duration_convert" _ "github.com/openconfig/gnmic/pkg/formatters/event_extract_tags" _ "github.com/openconfig/gnmic/pkg/formatters/event_group_by" + _ "github.com/openconfig/gnmic/pkg/formatters/event_ieeefloat32" _ "github.com/openconfig/gnmic/pkg/formatters/event_jq" _ "github.com/openconfig/gnmic/pkg/formatters/event_merge" _ "github.com/openconfig/gnmic/pkg/formatters/event_override_ts" diff --git a/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go b/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go new file mode 100644 index 00000000..cfaeae2a --- /dev/null +++ b/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go @@ -0,0 +1,169 @@ +// © 2024 Nokia. +// +// This code is a Contribution to the gNMIc project (“Work”) made under the Google Software Grant and Corporate Contributor License Agreement (“CLA”) and governed by the Apache License 2.0. +// No other rights or licenses in or to any of Nokia’s intellectual property are granted for any other purpose. +// This code is provided on an “as is” basis without any warranties of any kind. +// +// SPDX-License-Identifier: Apache-2.0 + +package event_ieeefloat32 + +import ( + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log" + "math" + "os" + "regexp" + "strings" + + "github.com/itchyny/gojq" + + "github.com/openconfig/gnmic/pkg/api/types" + "github.com/openconfig/gnmic/pkg/api/utils" + "github.com/openconfig/gnmic/pkg/formatters" +) + +const ( + processorType = "event-ieeefloat32" + loggingPrefix = "[" + processorType + "] " +) + +// ieeefloat32 converts values from a base64 encoded string into a float32 +type ieeefloat32 struct { + Condition string `mapstructure:"condition,omitempty"` + ValueNames []string `mapstructure:"value-names,omitempty" json:"value-names,omitempty"` + Debug bool `mapstructure:"debug,omitempty" json:"debug,omitempty"` + + valueNames []*regexp.Regexp + code *gojq.Code + logger *log.Logger +} + +func init() { + formatters.Register(processorType, func() formatters.EventProcessor { + return &ieeefloat32{ + logger: log.New(io.Discard, "", 0), + } + }) +} + +func (p *ieeefloat32) Init(cfg interface{}, opts ...formatters.Option) error { + err := formatters.DecodeConfig(cfg, p) + if err != nil { + return err + } + for _, opt := range opts { + opt(p) + } + if p.Condition != "" { + p.Condition = strings.TrimSpace(p.Condition) + q, err := gojq.Parse(p.Condition) + if err != nil { + return err + } + p.code, err = gojq.Compile(q) + if err != nil { + return err + } + } + + // init value names regex + p.valueNames, err = compileRegex(p.ValueNames) + if err != nil { + return err + } + if p.logger.Writer() != io.Discard { + b, err := json.Marshal(p) + if err != nil { + p.logger.Printf("initialized processor '%s': %+v", processorType, p) + return nil + } + p.logger.Printf("initialized processor '%s': %s", processorType, string(b)) + } + return nil +} + +func (p *ieeefloat32) Apply(es ...*formatters.EventMsg) []*formatters.EventMsg { + for _, e := range es { + if e == nil { + continue + } + // condition is set + if p.code != nil && p.Condition != "" { + ok, err := formatters.CheckCondition(p.code, e) + if err != nil { + p.logger.Printf("condition check failed: %v", err) + } + if !ok { + continue + } + } + // condition passed => check regexes + for k, v := range e.Values { + for _, re := range p.valueNames { + if re.MatchString(k) { + f, err := p.decodeBase64String(v) + if err != nil { + p.logger.Printf("failed to decode base64 string: %v", err) + } + e.Values[k] = f + break + } + } + } + } + return es +} + +func (p *ieeefloat32) WithLogger(l *log.Logger) { + if p.Debug && l != nil { + p.logger = log.New(l.Writer(), loggingPrefix, l.Flags()) + } else if p.Debug { + p.logger = log.New(os.Stderr, loggingPrefix, utils.DefaultLoggingFlags) + } +} + +func (p *ieeefloat32) WithTargets(tcs map[string]*types.TargetConfig) {} + +func (p *ieeefloat32) WithActions(act map[string]map[string]interface{}) {} + +func (p *ieeefloat32) WithProcessors(procs map[string]map[string]any) {} + +func (p *ieeefloat32) decodeBase64String(e any) (float32, error) { + switch b64 := e.(type) { + default: + return 0, fmt.Errorf("invalid type: %T", e) + case string: + + data, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return 0, fmt.Errorf("failed to decode base64: %v", err) + } + + if len(data) < 4 { + return 0, fmt.Errorf("decoded data is less than 4 bytes") + } + fmt.Printf("%x\n", data) + bits := binary.BigEndian.Uint32(data[:4]) + fmt.Printf("%v\n", bits) + floatVal := math.Float32frombits(bits) + fmt.Printf("%v\n", floatVal) + return floatVal, nil + } +} + +func compileRegex(expr []string) ([]*regexp.Regexp, error) { + res := make([]*regexp.Regexp, 0, len(expr)) + for _, reg := range expr { + re, err := regexp.Compile(reg) + if err != nil { + return nil, err + } + res = append(res, re) + } + return res, nil +} diff --git a/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go b/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go new file mode 100644 index 00000000..aad8e44a --- /dev/null +++ b/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go @@ -0,0 +1,104 @@ +// © 2024 Nokia. +// +// This code is a Contribution to the gNMIc project (“Work”) made under the Google Software Grant and Corporate Contributor License Agreement (“CLA”) and governed by the Apache License 2.0. +// No other rights or licenses in or to any of Nokia’s intellectual property are granted for any other purpose. +// This code is provided on an “as is” basis without any warranties of any kind. +// +// SPDX-License-Identifier: Apache-2.0 + +package event_ieeefloat32 + +import ( + "reflect" + "testing" + + "github.com/openconfig/gnmic/pkg/formatters" +) + +type item struct { + input []*formatters.EventMsg + output []*formatters.EventMsg +} + +var testset = map[string]struct { + processorType string + processor map[string]interface{} + tests []item +}{ + "simple": { + processorType: processorType, + processor: map[string]interface{}{ + "value-names": []string{"^components/component/power-supply/state/output-current$"}, + }, + tests: []item{ + { + input: nil, + output: nil, + }, + { + input: make([]*formatters.EventMsg, 0), + output: make([]*formatters.EventMsg, 0), + }, + { + input: []*formatters.EventMsg{ + { + Values: map[string]interface{}{"value": 1}, + Tags: map[string]string{"tag1": "1"}, + }, + }, + output: []*formatters.EventMsg{ + { + Values: map[string]interface{}{"value": 1}, + Tags: map[string]string{ + "tag1": "1", + }, + }, + }, + }, + { + input: []*formatters.EventMsg{ + { + Values: map[string]interface{}{ + "components/component/power-supply/state/output-current": "QEYAAA=="}, + }, + }, + output: []*formatters.EventMsg{ + { + Values: map[string]interface{}{ + "components/component/power-supply/state/output-current": float32(3.09375)}, + }, + }, + }, + }, + }, +} + +func TestEventIEEEFloat32(t *testing.T) { + for name, ts := range testset { + if pi, ok := formatters.EventProcessors[ts.processorType]; ok { + t.Log("found processor") + p := pi() + err := p.Init(ts.processor) + if err != nil { + t.Errorf("failed to initialize processors: %v", err) + return + } + t.Logf("processor: %+v", p) + for i, item := range ts.tests { + t.Run(name, func(t *testing.T) { + t.Logf("running test item %d", i) + outs := p.Apply(item.input...) + for j := range outs { + if !reflect.DeepEqual(outs[j], item.output[j]) { + t.Logf("failed at %s item %d, index %d, expected: %+v", name, i, j, item.output[j]) + t.Logf("failed at %s item %d, index %d, got: %+v", name, i, j, outs[j]) + t.Fail() + } + } + }) + } + } else { + t.Errorf("event processor %s not found", ts.processorType) + } + } +} diff --git a/pkg/formatters/processors.go b/pkg/formatters/processors.go index 68a654c9..be3aaad6 100644 --- a/pkg/formatters/processors.go +++ b/pkg/formatters/processors.go @@ -42,6 +42,7 @@ var EventProcessorTypes = []string{ "event-value-tag", "event-starlark", "event-combine", + "event-ieeefloat32", } type Initializer func() EventProcessor From 67ac4889b861641f34a85af2963e968ecc20528c Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Tue, 5 Nov 2024 09:48:23 -0800 Subject: [PATCH 2/3] add tests --- .../event_ieeefloat32/event_ieeefloat32.go | 4 +--- .../event_ieeefloat32_test.go | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go b/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go index cfaeae2a..fab91cc8 100644 --- a/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go +++ b/pkg/formatters/event_ieeefloat32/event_ieeefloat32.go @@ -109,6 +109,7 @@ func (p *ieeefloat32) Apply(es ...*formatters.EventMsg) []*formatters.EventMsg { f, err := p.decodeBase64String(v) if err != nil { p.logger.Printf("failed to decode base64 string: %v", err) + continue } e.Values[k] = f break @@ -147,11 +148,8 @@ func (p *ieeefloat32) decodeBase64String(e any) (float32, error) { if len(data) < 4 { return 0, fmt.Errorf("decoded data is less than 4 bytes") } - fmt.Printf("%x\n", data) bits := binary.BigEndian.Uint32(data[:4]) - fmt.Printf("%v\n", bits) floatVal := math.Float32frombits(bits) - fmt.Printf("%v\n", floatVal) return floatVal, nil } } diff --git a/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go b/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go index aad8e44a..aa3acd62 100644 --- a/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go +++ b/pkg/formatters/event_ieeefloat32/event_ieeefloat32_test.go @@ -28,7 +28,10 @@ var testset = map[string]struct { "simple": { processorType: processorType, processor: map[string]interface{}{ - "value-names": []string{"^components/component/power-supply/state/output-current$"}, + "value-names": []string{ + "^components/component/power-supply/state/output-current$", + "^components/component/power-supply/state/input-current$", + }, }, tests: []item{ { @@ -69,6 +72,24 @@ var testset = map[string]struct { }, }, }, + { + input: []*formatters.EventMsg{ + { + Values: map[string]interface{}{ + "components/component/power-supply/state/output-current": "QEYAAA==", + "components/component/power-supply/state/input-current": "QEYAAA==", + }, + }, + }, + output: []*formatters.EventMsg{ + { + Values: map[string]interface{}{ + "components/component/power-supply/state/output-current": float32(3.09375), + "components/component/power-supply/state/input-current": float32(3.09375), + }, + }, + }, + }, }, }, } From 677f8f2e9a670705d258f69133637c46702e45a4 Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Tue, 5 Nov 2024 09:48:29 -0800 Subject: [PATCH 3/3] add docs --- .../event_processors/event_ieeefloat32.md | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/user_guide/event_processors/event_ieeefloat32.md diff --git a/docs/user_guide/event_processors/event_ieeefloat32.md b/docs/user_guide/event_processors/event_ieeefloat32.md new file mode 100644 index 00000000..d8afde99 --- /dev/null +++ b/docs/user_guide/event_processors/event_ieeefloat32.md @@ -0,0 +1,53 @@ +The `event-ieeefloat32` processor allows converting binary data received from a router with the type IEEE 32-bit floating point number. + + +```yaml +processors: + # processor name + sample-processor: + # processor type + event-ieeefloat32: + # jq expression, if evaluated to true, the processor applies based on the field `value-names` + condition: + # list of regular expressions to be matched against the values names, if matched, the value is converted to a float32. + value-names: [] +``` + +### Examples + +```yaml +processors: + # processor name + sample-processor: + # processor type + event-ieeefloat32: + value-names: + - "^components/component/power-supply/state/output-current$" +``` + +=== "Event format before" + ```json + { + "name": "sub1", + "timestamp": 1607678293684962443, + "tags": { + "source": "172.20.20.5:57400" + }, + "values": { + "components/component/power-supply/state/output-current": "QEYAAA==" + } + } + ``` +=== "Event format after" + ```json + { + "name": "sub1", + "timestamp": 1607678293684962443, + "tags": { + "source": "172.20.20.5:57400", + }, + "values": { + "components/component/power-supply/state/output-current": 3.09375 + } + } + ```