From 26a20d20797846f01e63ec2f03611a46440101d3 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Mon, 26 Feb 2024 15:28:28 +0100 Subject: [PATCH] API cleanup: remove deprecated metric.filter (#613) This fields was replaced with metric.filters (plural) --- cmd/flowlogs-pipeline/main_test.go | 2 +- docs/api.md | 20 -------------------- pkg/api/encode_prom.go | 8 -------- pkg/config/pipeline_builder_test.go | 2 +- pkg/pipeline/aggregate_prom_test.go | 6 +++--- pkg/pipeline/encode/encode_prom_metric.go | 2 +- pkg/pipeline/encode/encode_prom_test.go | 4 ++-- 7 files changed, 8 insertions(+), 36 deletions(-) diff --git a/cmd/flowlogs-pipeline/main_test.go b/cmd/flowlogs-pipeline/main_test.go index fca40f8e9..b9f450821 100644 --- a/cmd/flowlogs-pipeline/main_test.go +++ b/cmd/flowlogs-pipeline/main_test.go @@ -50,7 +50,7 @@ func TestPipelineConfigSetup(t *testing.T) { js := `{ "PipeLine": "[{\"name\":\"grpc\"},{\"follows\":\"grpc\",\"name\":\"enrich\"},{\"follows\":\"enrich\",\"name\":\"loki\"},{\"follows\":\"enrich\",\"name\":\"prometheus\"}]", - "Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstAddr\",\"output\":\"DstK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstPort\",\"output\":\"Service\",\"parameters\":\"Proto\",\"type\":\"add_service\"},{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"parameters\":\"/16\",\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]", + "Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstAddr\",\"output\":\"DstK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstPort\",\"output\":\"Service\",\"parameters\":\"Proto\",\"type\":\"add_service\"},{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"parameters\":\"/16\",\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]", "Health": { "Port": "8080" }, diff --git a/docs/api.md b/docs/api.md index a2d532fa7..26ae6e93c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -17,16 +17,6 @@ Following is the supported API format for prometheus encode: counter: monotonically increasing counter whose value can only increase histogram: counts samples in configurable buckets agg_histogram: counts samples in configurable buckets, pre-aggregated via an Aggregate stage - filter: an optional criterion to filter entries by. Deprecated: use filters instead. - key: the key to match and filter by - value: the value to match and filter by - type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex - equal: match exactly the provided filter value - not_equal: the value must be different from the provided filter - presence: filter key must be present (filter value is ignored) - absence: filter key must be absent (filter value is ignored) - match_regex: match filter value as a regular expression - not_match_regex: the filter value must not match the provided regular expression filters: a list of criteria to filter entries by key: the key to match and filter by value: the value to match and filter by @@ -354,16 +344,6 @@ Following is the supported API format for writing metrics to an OpenTelemetry co counter: monotonically increasing counter whose value can only increase histogram: counts samples in configurable buckets agg_histogram: counts samples in configurable buckets, pre-aggregated via an Aggregate stage - filter: an optional criterion to filter entries by. Deprecated: use filters instead. - key: the key to match and filter by - value: the value to match and filter by - type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex - equal: match exactly the provided filter value - not_equal: the value must be different from the provided filter - presence: filter key must be present (filter value is ignored) - absence: filter key must be absent (filter value is ignored) - match_regex: match filter value as a regular expression - not_match_regex: the filter value must not match the provided regular expression filters: a list of criteria to filter entries by key: the key to match and filter by value: the value to match and filter by diff --git a/pkg/api/encode_prom.go b/pkg/api/encode_prom.go index 19e64f5a2..001dab3f7 100644 --- a/pkg/api/encode_prom.go +++ b/pkg/api/encode_prom.go @@ -50,7 +50,6 @@ type PromConnectionInfo struct { type MetricsItem struct { Name string `yaml:"name" json:"name" doc:"the metric name"` Type string `yaml:"type" json:"type" enum:"MetricEncodeOperationEnum" doc:"one of the following:"` - Filter MetricsFilter `yaml:"filter,omitempty" json:"filter,omitempty" doc:"an optional criterion to filter entries by. Deprecated: use filters instead."` Filters []MetricsFilter `yaml:"filters" json:"filters" doc:"a list of criteria to filter entries by"` ValueKey string `yaml:"valueKey" json:"valueKey" doc:"entry key from which to resolve metric value"` Labels []string `yaml:"labels" json:"labels" doc:"labels to be associated with the metric"` @@ -58,13 +57,6 @@ type MetricsItem struct { ValueScale float64 `yaml:"valueScale" json:"valueScale" doc:"scale factor of the value (MetricVal := FlowVal / Scale)"` } -func (i *MetricsItem) GetFilters() []MetricsFilter { - if len(i.Filters) == 0 && i.Filter.Key != "" { - return []MetricsFilter{i.Filter} - } - return i.Filters -} - type MetricsItems []MetricsItem type MetricsFilter struct { diff --git a/pkg/config/pipeline_builder_test.go b/pkg/config/pipeline_builder_test.go index d2ef9ebec..d5670de4c 100644 --- a/pkg/config/pipeline_builder_test.go +++ b/pkg/config/pipeline_builder_test.go @@ -166,7 +166,7 @@ func TestKafkaPromPipeline(t *testing.T) { b, err = json.Marshal(params[4]) require.NoError(t, err) - require.JSONEq(t, `{"name":"prom","encode":{"type":"prom","prom":{"expiryTime":"50s", "metrics":[{"name":"connections_per_source_as","type":"counter","filter":{"key":"","type":"","value":""},"filters":[{"key":"name","type":"","value":"src_as_connection_count"}],"valueKey":"recent_count","valueScale":0,"labels":["by","aggregate"],"buckets":[]}],"prefix":"flp_"}}}`, string(b)) + require.JSONEq(t, `{"name":"prom","encode":{"type":"prom","prom":{"expiryTime":"50s", "metrics":[{"name":"connections_per_source_as","type":"counter","filters":[{"key":"name","type":"","value":"src_as_connection_count"}],"valueKey":"recent_count","valueScale":0,"labels":["by","aggregate"],"buckets":[]}],"prefix":"flp_"}}}`, string(b)) } func TestForkPipeline(t *testing.T) { diff --git a/pkg/pipeline/aggregate_prom_test.go b/pkg/pipeline/aggregate_prom_test.go index 6c820682c..776fc3c84 100644 --- a/pkg/pipeline/aggregate_prom_test.go +++ b/pkg/pipeline/aggregate_prom_test.go @@ -68,19 +68,19 @@ parameters: metrics: - name: flow_count type: counter - filter: {key: name, value: bandwidth_count} + filters: [{key: name, value: bandwidth_count}] valueKey: recent_count labels: - service - name: bytes_sum type: counter - filter: {key: name, value: bandwidth_sum} + filters: [{key: name, value: bandwidth_sum}] valueKey: recent_op_value labels: - service - name: bytes_histogram type: agg_histogram - filter: {key: name, value: bandwidth_raw_values} + filters: [{key: name, value: bandwidth_raw_values}] valueKey: recent_raw_values labels: - service diff --git a/pkg/pipeline/encode/encode_prom_metric.go b/pkg/pipeline/encode/encode_prom_metric.go index 2f95f4e21..618ccd2ae 100644 --- a/pkg/pipeline/encode/encode_prom_metric.go +++ b/pkg/pipeline/encode/encode_prom_metric.go @@ -122,7 +122,7 @@ func CreateMetricInfo(def api.MetricsItem) *MetricInfo { mi := MetricInfo{ MetricsItem: def, } - for _, f := range def.GetFilters() { + for _, f := range def.Filters { mi.FilterPredicates = append(mi.FilterPredicates, filterToPredicate(f)) } return &mi diff --git a/pkg/pipeline/encode/encode_prom_test.go b/pkg/pipeline/encode/encode_prom_test.go index 307a5f52b..0820a81b7 100644 --- a/pkg/pipeline/encode/encode_prom_test.go +++ b/pkg/pipeline/encode/encode_prom_test.go @@ -46,7 +46,7 @@ parameters: metrics: - name: Bytes type: gauge - filter: {key: dstAddr, value: 10.1.2.4} + filters: [{key: dstAddr, value: 10.1.2.4}] valueKey: bytes labels: - srcAddr @@ -54,7 +54,7 @@ parameters: - srcPort - name: Packets type: counter - filter: {key: dstAddr, value: 10.1.2.4} + filters: [{key: dstAddr, value: 10.1.2.4}] valueKey: packets labels: - srcAddr