API cleanup: remove deprecated metric.filter (#613)

This fields was replaced with metric.filters (plural)

cmd/flowlogs-pipeline/main_test.go

@@ -50,7 +50,7 @@ func TestPipelineConfigSetup(t *testing.T) {
 
 	js := `{
     "PipeLine": "[{\"name\":\"grpc\"},{\"follows\":\"grpc\",\"name\":\"enrich\"},{\"follows\":\"enrich\",\"name\":\"loki\"},{\"follows\":\"enrich\",\"name\":\"prometheus\"}]",
-    "Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstAddr\",\"output\":\"DstK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstPort\",\"output\":\"Service\",\"parameters\":\"Proto\",\"type\":\"add_service\"},{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"parameters\":\"/16\",\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]",
+    "Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstAddr\",\"output\":\"DstK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstPort\",\"output\":\"Service\",\"parameters\":\"Proto\",\"type\":\"add_service\"},{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"parameters\":\"/16\",\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]",
     "Health": {
         "Port": "8080"
     },

docs/api.md

@@ -17,16 +17,6 @@ Following is the supported API format for prometheus encode:
                      counter: monotonically increasing counter whose value can only increase
                      histogram: counts samples in configurable buckets
                      agg_histogram: counts samples in configurable buckets, pre-aggregated via an Aggregate stage
-                 filter: an optional criterion to filter entries by. Deprecated: use filters instead.
-                     key: the key to match and filter by
-                     value: the value to match and filter by
-                     type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex
-                         equal: match exactly the provided filter value
-                         not_equal: the value must be different from the provided filter
-                         presence: filter key must be present (filter value is ignored)
-                         absence: filter key must be absent (filter value is ignored)
-                         match_regex: match filter value as a regular expression
-                         not_match_regex: the filter value must not match the provided regular expression
                  filters: a list of criteria to filter entries by
                          key: the key to match and filter by
                          value: the value to match and filter by
@@ -354,16 +344,6 @@ Following is the supported API format for writing metrics to an OpenTelemetry co
                      counter: monotonically increasing counter whose value can only increase
                      histogram: counts samples in configurable buckets
                      agg_histogram: counts samples in configurable buckets, pre-aggregated via an Aggregate stage
-                 filter: an optional criterion to filter entries by. Deprecated: use filters instead.
-                     key: the key to match and filter by
-                     value: the value to match and filter by
-                     type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex
-                         equal: match exactly the provided filter value
-                         not_equal: the value must be different from the provided filter
-                         presence: filter key must be present (filter value is ignored)
-                         absence: filter key must be absent (filter value is ignored)
-                         match_regex: match filter value as a regular expression
-                         not_match_regex: the filter value must not match the provided regular expression
                  filters: a list of criteria to filter entries by
                          key: the key to match and filter by
                          value: the value to match and filter by

pkg/api/encode_prom.go

@@ -50,21 +50,13 @@ type PromConnectionInfo struct {
 type MetricsItem struct {
 	Name       string          `yaml:"name" json:"name" doc:"the metric name"`
 	Type       string          `yaml:"type" json:"type" enum:"MetricEncodeOperationEnum" doc:"one of the following:"`
-	Filter     MetricsFilter   `yaml:"filter,omitempty" json:"filter,omitempty" doc:"an optional criterion to filter entries by. Deprecated: use filters instead."`
 	Filters    []MetricsFilter `yaml:"filters" json:"filters" doc:"a list of criteria to filter entries by"`
 	ValueKey   string          `yaml:"valueKey" json:"valueKey" doc:"entry key from which to resolve metric value"`
 	Labels     []string        `yaml:"labels" json:"labels" doc:"labels to be associated with the metric"`
 	Buckets    []float64       `yaml:"buckets" json:"buckets" doc:"histogram buckets"`
 	ValueScale float64         `yaml:"valueScale" json:"valueScale" doc:"scale factor of the value (MetricVal := FlowVal / Scale)"`
 }
 
-func (i *MetricsItem) GetFilters() []MetricsFilter {
-	if len(i.Filters) == 0 && i.Filter.Key != "" {
-		return []MetricsFilter{i.Filter}
-	}
-	return i.Filters
-}
-
 type MetricsItems []MetricsItem
 
 type MetricsFilter struct {

pkg/config/pipeline_builder_test.go

@@ -166,7 +166,7 @@ func TestKafkaPromPipeline(t *testing.T) {
 
 	b, err = json.Marshal(params[4])
 	require.NoError(t, err)
-	require.JSONEq(t, `{"name":"prom","encode":{"type":"prom","prom":{"expiryTime":"50s", "metrics":[{"name":"connections_per_source_as","type":"counter","filter":{"key":"","type":"","value":""},"filters":[{"key":"name","type":"","value":"src_as_connection_count"}],"valueKey":"recent_count","valueScale":0,"labels":["by","aggregate"],"buckets":[]}],"prefix":"flp_"}}}`, string(b))
+	require.JSONEq(t, `{"name":"prom","encode":{"type":"prom","prom":{"expiryTime":"50s", "metrics":[{"name":"connections_per_source_as","type":"counter","filters":[{"key":"name","type":"","value":"src_as_connection_count"}],"valueKey":"recent_count","valueScale":0,"labels":["by","aggregate"],"buckets":[]}],"prefix":"flp_"}}}`, string(b))
 }
 
 func TestForkPipeline(t *testing.T) {

pkg/pipeline/aggregate_prom_test.go

@@ -68,19 +68,19 @@ parameters:
        metrics:
          - name: flow_count
            type: counter
-           filter: {key: name, value: bandwidth_count}
+           filters: [{key: name, value: bandwidth_count}]
            valueKey: recent_count
            labels:
              - service
          - name: bytes_sum
            type: counter
-           filter: {key: name, value: bandwidth_sum}
+           filters: [{key: name, value: bandwidth_sum}]
            valueKey: recent_op_value
            labels:
              - service
          - name: bytes_histogram
            type: agg_histogram
-           filter: {key: name, value: bandwidth_raw_values}
+           filters: [{key: name, value: bandwidth_raw_values}]
            valueKey: recent_raw_values
            labels:
              - service

pkg/pipeline/encode/encode_prom_metric.go

@@ -122,7 +122,7 @@ func CreateMetricInfo(def api.MetricsItem) *MetricInfo {
 	mi := MetricInfo{
 		MetricsItem: def,
 	}
-	for _, f := range def.GetFilters() {
+	for _, f := range def.Filters {
 		mi.FilterPredicates = append(mi.FilterPredicates, filterToPredicate(f))
 	}
 	return &mi

pkg/pipeline/encode/encode_prom_test.go

@@ -46,15 +46,15 @@ parameters:
         metrics:
           - name: Bytes
             type: gauge
-            filter: {key: dstAddr, value: 10.1.2.4}
+            filters: [{key: dstAddr, value: 10.1.2.4}]
             valueKey: bytes
             labels:
               - srcAddr
               - dstAddr
               - srcPort
           - name: Packets
             type: counter
-            filter: {key: dstAddr, value: 10.1.2.4}
+            filters: [{key: dstAddr, value: 10.1.2.4}]
             valueKey: packets
             labels:
               - srcAddr