Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

API cleanup: remove deprecated metric.filter #613

Merged
merged 1 commit into from
Feb 26, 2024
+8 −36
Merged

API cleanup: remove deprecated metric.filter #613

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/flowlogs-pipeline/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestPipelineConfigSetup(t *testing.T) {

js := `{
"PipeLine": "[{\"name\":\"grpc\"},{\"follows\":\"grpc\",\"name\":\"enrich\"},{\"follows\":\"enrich\",\"name\":\"loki\"},{\"follows\":\"enrich\",\"name\":\"prometheus\"}]",
"Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstAddr\",\"output\":\"DstK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstPort\",\"output\":\"Service\",\"parameters\":\"Proto\",\"type\":\"add_service\"},{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"parameters\":\"/16\",\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"filter\":{\"key\":\"\",\"value\":\"\"},\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]",
"Parameters": "[{\"ingest\":{\"grpc\":{\"port\":2055},\"type\":\"grpc\"},\"name\":\"grpc\"},{\"name\":\"enrich\",\"transform\":{\"network\":{\"rules\":[{\"input\":\"SrcAddr\",\"output\":\"SrcK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstAddr\",\"output\":\"DstK8S\",\"type\":\"add_kubernetes\"},{\"input\":\"DstPort\",\"output\":\"Service\",\"parameters\":\"Proto\",\"type\":\"add_service\"},{\"input\":\"SrcAddr\",\"output\":\"SrcSubnet\",\"parameters\":\"/16\",\"type\":\"add_subnet\"}]},\"type\":\"network\"}},{\"name\":\"loki\",\"write\":{\"loki\":{\"batchSize\":102400,\"batchWait\":\"1s\",\"clientConfig\":{\"follow_redirects\":false,\"proxy_url\":null,\"tls_config\":{\"insecure_skip_verify\":false}},\"labels\":[\"SrcK8S_Namespace\",\"SrcK8S_OwnerName\",\"DstK8S_Namespace\",\"DstK8S_OwnerName\",\"FlowDirection\"],\"maxBackoff\":\"5m0s\",\"maxRetries\":10,\"minBackoff\":\"1s\",\"staticLabels\":{\"app\":\"netobserv-flowcollector\"},\"tenantID\":\"netobserv\",\"timeout\":\"10s\",\"timestampLabel\":\"TimeFlowEndMs\",\"timestampScale\":\"1ms\",\"url\":\"http://loki.netobserv.svc:3100/\"},\"type\":\"loki\"}},{\"encode\":{\"prom\":{\"metrics\":[{\"buckets\":null,\"labels\":[\"Service\",\"SrcK8S_Namespace\"],\"name\":\"bandwidth_per_network_service_per_namespace\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"SrcSubnet\"],\"name\":\"bandwidth_per_source_subnet\",\"type\":\"counter\",\"valueKey\":\"Bytes\"},{\"buckets\":null,\"labels\":[\"Service\"],\"name\":\"network_service_total\",\"type\":\"counter\",\"valueKey\":\"\"}],\"prefix\":\"netobserv_\"},\"type\":\"prom\"},\"name\":\"prometheus\"}]",
"Health": {
"Port": "8080"
},
Expand Down
20 changes: 0 additions & 20 deletions docs/api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,6 @@ Following is the supported API format for prometheus encode:
counter: monotonically increasing counter whose value can only increase
histogram: counts samples in configurable buckets
agg_histogram: counts samples in configurable buckets, pre-aggregated via an Aggregate stage
filter: an optional criterion to filter entries by. Deprecated: use filters instead.
key: the key to match and filter by
value: the value to match and filter by
type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex
equal: match exactly the provided filter value
not_equal: the value must be different from the provided filter
presence: filter key must be present (filter value is ignored)
absence: filter key must be absent (filter value is ignored)
match_regex: match filter value as a regular expression
not_match_regex: the filter value must not match the provided regular expression
filters: a list of criteria to filter entries by
key: the key to match and filter by
value: the value to match and filter by
Expand Down Expand Up @@ -354,16 +344,6 @@ Following is the supported API format for writing metrics to an OpenTelemetry co
counter: monotonically increasing counter whose value can only increase
histogram: counts samples in configurable buckets
agg_histogram: counts samples in configurable buckets, pre-aggregated via an Aggregate stage
filter: an optional criterion to filter entries by. Deprecated: use filters instead.
key: the key to match and filter by
value: the value to match and filter by
type: (enum) the type of filter match: equal (default), not_equal, presence, absence, match_regex or not_match_regex
equal: match exactly the provided filter value
not_equal: the value must be different from the provided filter
presence: filter key must be present (filter value is ignored)
absence: filter key must be absent (filter value is ignored)
match_regex: match filter value as a regular expression
not_match_regex: the filter value must not match the provided regular expression
filters: a list of criteria to filter entries by
key: the key to match and filter by
value: the value to match and filter by
Expand Down
8 changes: 0 additions & 8 deletions pkg/api/encode_prom.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,21 +50,13 @@ type PromConnectionInfo struct {
type MetricsItem struct {
Name string `yaml:"name" json:"name" doc:"the metric name"`
Type string `yaml:"type" json:"type" enum:"MetricEncodeOperationEnum" doc:"one of the following:"`
Filter MetricsFilter `yaml:"filter,omitempty" json:"filter,omitempty" doc:"an optional criterion to filter entries by. Deprecated: use filters instead."`
Filters []MetricsFilter `yaml:"filters" json:"filters" doc:"a list of criteria to filter entries by"`
ValueKey string `yaml:"valueKey" json:"valueKey" doc:"entry key from which to resolve metric value"`
Labels []string `yaml:"labels" json:"labels" doc:"labels to be associated with the metric"`
Buckets []float64 `yaml:"buckets" json:"buckets" doc:"histogram buckets"`
ValueScale float64 `yaml:"valueScale" json:"valueScale" doc:"scale factor of the value (MetricVal := FlowVal / Scale)"`
}

func (i *MetricsItem) GetFilters() []MetricsFilter {
if len(i.Filters) == 0 && i.Filter.Key != "" {
return []MetricsFilter{i.Filter}
}
return i.Filters
}

type MetricsItems []MetricsItem

type MetricsFilter struct {
Expand Down
2 changes: 1 addition & 1 deletion pkg/config/pipeline_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func TestKafkaPromPipeline(t *testing.T) {

b, err = json.Marshal(params[4])
require.NoError(t, err)
require.JSONEq(t, `{"name":"prom","encode":{"type":"prom","prom":{"expiryTime":"50s", "metrics":[{"name":"connections_per_source_as","type":"counter","filter":{"key":"","type":"","value":""},"filters":[{"key":"name","type":"","value":"src_as_connection_count"}],"valueKey":"recent_count","valueScale":0,"labels":["by","aggregate"],"buckets":[]}],"prefix":"flp_"}}}`, string(b))
require.JSONEq(t, `{"name":"prom","encode":{"type":"prom","prom":{"expiryTime":"50s", "metrics":[{"name":"connections_per_source_as","type":"counter","filters":[{"key":"name","type":"","value":"src_as_connection_count"}],"valueKey":"recent_count","valueScale":0,"labels":["by","aggregate"],"buckets":[]}],"prefix":"flp_"}}}`, string(b))
}

func TestForkPipeline(t *testing.T) {
Expand Down
6 changes: 3 additions & 3 deletions pkg/pipeline/aggregate_prom_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,19 +68,19 @@ parameters:
metrics:
- name: flow_count
type: counter
filter: {key: name, value: bandwidth_count}
filters: [{key: name, value: bandwidth_count}]
valueKey: recent_count
labels:
- service
- name: bytes_sum
type: counter
filter: {key: name, value: bandwidth_sum}
filters: [{key: name, value: bandwidth_sum}]
valueKey: recent_op_value
labels:
- service
- name: bytes_histogram
type: agg_histogram
filter: {key: name, value: bandwidth_raw_values}
filters: [{key: name, value: bandwidth_raw_values}]
valueKey: recent_raw_values
labels:
- service
Expand Down
2 changes: 1 addition & 1 deletion pkg/pipeline/encode/encode_prom_metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func CreateMetricInfo(def api.MetricsItem) *MetricInfo {
mi := MetricInfo{
MetricsItem: def,
}
for _, f := range def.GetFilters() {
for _, f := range def.Filters {
mi.FilterPredicates = append(mi.FilterPredicates, filterToPredicate(f))
}
return &mi
Expand Down
4 changes: 2 additions & 2 deletions pkg/pipeline/encode/encode_prom_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ parameters:
metrics:
- name: Bytes
type: gauge
filter: {key: dstAddr, value: 10.1.2.4}
filters: [{key: dstAddr, value: 10.1.2.4}]
valueKey: bytes
labels:
- srcAddr
- dstAddr
- srcPort
- name: Packets
type: counter
filter: {key: dstAddr, value: 10.1.2.4}
filters: [{key: dstAddr, value: 10.1.2.4}]
valueKey: packets
labels:
- srcAddr
Expand Down
Loading