From 6a101d0022a7b2e8873dd62ec46921a6260b4294 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Mon, 26 Feb 2024 12:40:22 +0100 Subject: [PATCH] Enum replacement in API First step / PoC for fixing #608 Starting with metrics filters only; other enums should follow --- Makefile | 1 + cmd/apitodoc/main.go | 21 +++++++++++-- docs/api.md | 30 +++++++++---------- hack/update-enum-docs.sh | 15 ++++++++++ pkg/api/api.go | 6 ---- pkg/api/encode_prom.go | 36 +++++++++++++---------- pkg/api/enum.go | 1 - pkg/pipeline/encode/encode_prom_metric.go | 12 ++++---- 8 files changed, 76 insertions(+), 46 deletions(-) create mode 100755 hack/update-enum-docs.sh diff --git a/Makefile b/Makefile index 5bc157f49..889e66f31 100644 --- a/Makefile +++ b/Makefile @@ -121,6 +121,7 @@ build: validate_go lint build_code docs ## Build flowlogs-pipeline executable an docs: FORCE ## Update flowlogs-pipeline documentation @./hack/update-docs.sh @go run cmd/apitodoc/main.go > docs/api.md + @./hack/update-enum-docs.sh @go run cmd/operationalmetricstodoc/main.go > docs/operational-metrics.md .PHONY: clean diff --git a/cmd/apitodoc/main.go b/cmd/apitodoc/main.go index fb2e0002b..e4710952c 100644 --- a/cmd/apitodoc/main.go +++ b/cmd/apitodoc/main.go @@ -19,6 +19,7 @@ package main import ( "bytes" + "errors" "fmt" "io" "reflect" @@ -29,9 +30,12 @@ import ( func iterate(output io.Writer, data interface{}, indent int) { newIndent := indent + 1 - dataType := reflect.ValueOf(data).Kind() - // DEBUG code: dataTypeName := reflect.ValueOf(data).Type().String() d := reflect.ValueOf(data) + dataType := d.Kind() + dataTypeName, err := getTypeName(d) + if err != nil { + dataTypeName = "(unknown)" + } if dataType == reflect.Slice || dataType == reflect.Map { // DEBUG code: fmt.Fprintf(output, "%s %s <-- %s \n",strings.Repeat(" ",4*indent),dataTypeName,dataType ) zeroElement := reflect.Zero(reflect.ValueOf(data).Type().Elem()).Interface() @@ -75,9 +79,22 @@ func iterate(output io.Writer, data interface{}, indent int) { // Since we only "converted" Ptr to Struct and the actual output is done in the next iteration, we call // iterate() with the same `indent` as the current level iterate(output, zeroElement, indent) + } else if strings.HasPrefix(dataTypeName, "api.") && strings.HasSuffix(dataTypeName, "Enum") { + // set placeholder for enum + fmt.Fprintf(output, "placeholder @%s:%d@\n", strings.TrimPrefix(dataTypeName, "api."), 4*newIndent) } } +func getTypeName(d reflect.Value) (name string, err error) { + defer func() { + if recover() != nil { + err = errors.New("unknown type name") + } + }() + name = d.Type().String() + return +} + func main() { output := new(bytes.Buffer) iterate(output, api.API{}, 0) diff --git a/docs/api.md b/docs/api.md index 26ae6e93c..7c62233c3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -20,13 +20,13 @@ Following is the supported API format for prometheus encode: filters: a list of criteria to filter entries by key: the key to match and filter by value: the value to match and filter by - type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex - equal: match exactly the provided filter value - not_equal: the value must be different from the provided filter - presence: filter key must be present (filter value is ignored) - absence: filter key must be absent (filter value is ignored) - match_regex: match filter value as a regular expression - not_match_regex: the filter value must not match the provided regular expression + type: the type of filter match (enum) + equal: match exactly the provided filter value + not_equal: the value must be different from the provided filter + presence: filter key must be present (filter value is ignored) + absence: filter key must be absent (filter value is ignored) + match_regex: match filter value as a regular expression + not_match_regex: the filter value must not match the provided regular expression valueKey: entry key from which to resolve metric value labels: labels to be associated with the metric buckets: histogram buckets @@ -347,13 +347,13 @@ Following is the supported API format for writing metrics to an OpenTelemetry co filters: a list of criteria to filter entries by key: the key to match and filter by value: the value to match and filter by - type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex - equal: match exactly the provided filter value - not_equal: the value must be different from the provided filter - presence: filter key must be present (filter value is ignored) - absence: filter key must be absent (filter value is ignored) - match_regex: match filter value as a regular expression - not_match_regex: the filter value must not match the provided regular expression + type: the type of filter match (enum) + equal: match exactly the provided filter value + not_equal: the value must be different from the provided filter + presence: filter key must be present (filter value is ignored) + absence: filter key must be absent (filter value is ignored) + match_regex: match filter value as a regular expression + not_match_regex: the filter value must not match the provided regular expression valueKey: entry key from which to resolve metric value labels: labels to be associated with the metric buckets: histogram buckets @@ -377,4 +377,4 @@ Following is the supported API format for writing traces to an OpenTelemetry col userKeyPath: path to the user private key headers: headers to add to messages (optional) spanSplitter: separate span for each prefix listed - \ No newline at end of file + diff --git a/hack/update-enum-docs.sh b/hack/update-enum-docs.sh new file mode 100755 index 000000000..66a09d549 --- /dev/null +++ b/hack/update-enum-docs.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -eou pipefail + +md_file="./docs/api.md" + +grep "placeholder @" "$md_file" | while read line; do + type=$(echo "$line" | sed -r 's/^placeholder @(.*):(.*)@/\1/') + indent=$(echo "$line" | sed -r 's/^placeholder @(.*):(.*)@/\2/') + repl=$(go doc -all -short -C pkg/api $type | sed -r ':x ; /\/\// { N ; s/\/\/ (.*)\n/\1##/ ; bx }' | grep "##" | sed -r "s/\s*(.*)##.*\"(.*)\"/$(printf "%${indent}s")\2: \1/") + awk -v inject="${repl}" "/placeholder @$type:$indent@/{print inject;next}1" $md_file > "$md_file.tmp" +done + +rm $md_file +mv "$md_file.tmp" $md_file diff --git a/pkg/api/api.go b/pkg/api/api.go index 83f921e86..274a5b0f2 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -50,12 +50,6 @@ const ( AddKubernetesRuleType = "add_kubernetes" AddKubernetesInfraRuleType = "add_kubernetes_infra" ReinterpretDirectionRuleType = "reinterpret_direction" - PromFilterEqual = "equal" - PromFilterNotEqual = "not_equal" - PromFilterPresence = "presence" - PromFilterAbsence = "absence" - PromFilterRegex = "match_regex" - PromFilterNotRegex = "not_match_regex" TagYaml = "yaml" TagDoc = "doc" diff --git a/pkg/api/encode_prom.go b/pkg/api/encode_prom.go index 001dab3f7..a483b8ac0 100644 --- a/pkg/api/encode_prom.go +++ b/pkg/api/encode_prom.go @@ -57,23 +57,27 @@ type MetricsItem struct { ValueScale float64 `yaml:"valueScale" json:"valueScale" doc:"scale factor of the value (MetricVal := FlowVal / Scale)"` } -type MetricsItems []MetricsItem +type MetricFilterEnum string -type MetricsFilter struct { - Key string `yaml:"key" json:"key" doc:"the key to match and filter by"` - Value string `yaml:"value" json:"value" doc:"the value to match and filter by"` - Type string `yaml:"type" json:"type" enum:"MetricEncodeFilterTypeEnum" doc:"the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex"` -} +const ( + // match exactly the provided filter value + MetricFilterEqual MetricFilterEnum = "equal" + // the value must be different from the provided filter + MetricFilterNotEqual MetricFilterEnum = "not_equal" + // filter key must be present (filter value is ignored) + MetricFilterPresence MetricFilterEnum = "presence" + // filter key must be absent (filter value is ignored) + MetricFilterAbsence MetricFilterEnum = "absence" + // match filter value as a regular expression + MetricFilterRegex MetricFilterEnum = "match_regex" + // the filter value must not match the provided regular expression + MetricFilterNotRegex MetricFilterEnum = "not_match_regex" +) -type MetricEncodeFilterTypeEnum struct { - Equal string `yaml:"equal" json:"equal" doc:"match exactly the provided filter value"` - NotEqual string `yaml:"not_equal" json:"not_equal" doc:"the value must be different from the provided filter"` - Presence string `yaml:"presence" json:"presence" doc:"filter key must be present (filter value is ignored)"` - Absence string `yaml:"absence" json:"absence" doc:"filter key must be absent (filter value is ignored)"` - MatchRegex string `yaml:"match_regex" json:"match_regex" doc:"match filter value as a regular expression"` - NotMatchRegex string `yaml:"not_match_regex" json:"not_match_regex" doc:"the filter value must not match the provided regular expression"` -} +type MetricsItems []MetricsItem -func MetricEncodeFilterTypeName(t string) string { - return GetEnumName(MetricEncodeFilterTypeEnum{}, t) +type MetricsFilter struct { + Key string `yaml:"key" json:"key" doc:"the key to match and filter by"` + Value string `yaml:"value" json:"value" doc:"the value to match and filter by"` + Type MetricFilterEnum `yaml:"type" json:"type" doc:"the type of filter match (enum)"` } diff --git a/pkg/api/enum.go b/pkg/api/enum.go index 80b94c5a6..2c415ee1f 100644 --- a/pkg/api/enum.go +++ b/pkg/api/enum.go @@ -24,7 +24,6 @@ import ( type enums struct { MetricEncodeOperationEnum MetricEncodeOperationEnum - MetricEncodeFilterTypeEnum MetricEncodeFilterTypeEnum TransformNetworkOperationEnum TransformNetworkOperationEnum TransformFilterOperationEnum TransformFilterOperationEnum TransformGenericOperationEnum TransformGenericOperationEnum diff --git a/pkg/pipeline/encode/encode_prom_metric.go b/pkg/pipeline/encode/encode_prom_metric.go index 618ccd2ae..6bfbb3396 100644 --- a/pkg/pipeline/encode/encode_prom_metric.go +++ b/pkg/pipeline/encode/encode_prom_metric.go @@ -76,17 +76,17 @@ func NotRegex(filter api.MetricsFilter) Predicate { func filterToPredicate(filter api.MetricsFilter) Predicate { switch filter.Type { - case api.PromFilterEqual: + case api.MetricFilterEqual: return Equal(filter) - case api.PromFilterNotEqual: + case api.MetricFilterNotEqual: return NotEqual(filter) - case api.PromFilterPresence: + case api.MetricFilterPresence: return Presence(filter) - case api.PromFilterAbsence: + case api.MetricFilterAbsence: return Absence(filter) - case api.PromFilterRegex: + case api.MetricFilterRegex: return Regex(filter) - case api.PromFilterNotRegex: + case api.MetricFilterNotRegex: return NotRegex(filter) } // Default = Exact