add support for label and annotation inclusion and exclusion lists

Signed-off-by: Alex Price <[email protected]>

Author: awprice

docs/api.md

@@ -266,7 +266,11 @@ Following is the supported API format for network transformations:
                      output: entry output field
                      assignee: value needs to assign to output field
                      labels_prefix: labels prefix to use to copy input labels, if empty labels will not be copied
+                     label_inclusions: labels to include, if empty all labels will be included. Only used if labels_prefix is specified
+                     label_exclusions: labels to exclude, if empty no labels will be excluded. Only used if labels_prefix is specified
                      annotations_prefix: annotations prefix to use to copy input annotations, if empty annotations will not be copied
+                     annotation_inclusions: annotations to include, if empty all annotations will be included. Only used if annotations_prefix is specified
+                     annotation_exclusions: annotations to exclude, if empty no annotations will be excluded. Only used if annotations_prefix is specified
                      add_zone: if true the rule will add the zone
                  add_subnet: Add subnet rule configuration
                      input: entry input field

pkg/api/transform_network.go

@@ -98,16 +98,24 @@ type K8sReference struct {
 }
 
 type K8sRule struct {
-	IPField           string        `yaml:"ipField,omitempty" json:"ipField,omitempty" doc:"entry IP input field"`
-	InterfacesField   string        `yaml:"interfacesField,omitempty" json:"interfacesField,omitempty" doc:"entry Interfaces input field"`
-	UDNsField         string        `yaml:"udnsField,omitempty" json:"udnsField,omitempty" doc:"entry UDNs input field"`
-	MACField          string        `yaml:"macField,omitempty" json:"macField,omitempty" doc:"entry MAC input field"`
-	Output            string        `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
-	Assignee          string        `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
-	LabelsPrefix      string        `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input labels, if empty labels will not be copied"`
-	AnnotationsPrefix string        `yaml:"annotations_prefix,omitempty" json:"annotations_prefix,omitempty" doc:"annotations prefix to use to copy input annotations, if empty annotations will not be copied"`
-	AddZone           bool          `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"if true the rule will add the zone"`
-	OutputKeys        K8SOutputKeys `yaml:"-" json:"-"`
+	IPField                 string              `yaml:"ipField,omitempty" json:"ipField,omitempty" doc:"entry IP input field"`
+	InterfacesField         string              `yaml:"interfacesField,omitempty" json:"interfacesField,omitempty" doc:"entry Interfaces input field"`
+	UDNsField               string              `yaml:"udnsField,omitempty" json:"udnsField,omitempty" doc:"entry UDNs input field"`
+	MACField                string              `yaml:"macField,omitempty" json:"macField,omitempty" doc:"entry MAC input field"`
+	Output                  string              `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
+	Assignee                string              `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
+	LabelsPrefix            string              `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input labels, if empty labels will not be copied"`
+	LabelInclusions         []string            `yaml:"label_inclusions,omitempty" json:"label_inclusions,omitempty" doc:"labels to include, if empty all labels will be included. Only used if labels_prefix is specified"`
+	LabelExclusions         []string            `yaml:"label_exclusions,omitempty" json:"label_exclusions,omitempty" doc:"labels to exclude, if empty no labels will be excluded. Only used if labels_prefix is specified"`
+	AnnotationsPrefix       string              `yaml:"annotations_prefix,omitempty" json:"annotations_prefix,omitempty" doc:"annotations prefix to use to copy input annotations, if empty annotations will not be copied"`
+	AnnotationInclusions    []string            `yaml:"annotation_inclusions,omitempty" json:"annotation_inclusions,omitempty" doc:"annotations to include, if empty all annotations will be included. Only used if annotations_prefix is specified"`
+	AnnotationExclusions    []string            `yaml:"annotation_exclusions,omitempty" json:"annotation_exclusions,omitempty" doc:"annotations to exclude, if empty no annotations will be excluded. Only used if annotations_prefix is specified"`
+	AddZone                 bool                `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"if true the rule will add the zone"`
+	OutputKeys              K8SOutputKeys       `yaml:"-" json:"-"`
+	LabelInclusionsMap      map[string]struct{} `yaml:"-" json:"-"`
+	LabelExclusionsMap      map[string]struct{} `yaml:"-" json:"-"`
+	AnnotationInclusionsMap map[string]struct{} `yaml:"-" json:"-"`
+	AnnotationExclusionsMap map[string]struct{} `yaml:"-" json:"-"`
 }
 
 type K8SOutputKeys struct {
@@ -151,6 +159,11 @@ func (r *K8sRule) preprocess() {
 			Zone:        r.Output + "_Zone",
 		}
 	}
+
+	r.LabelInclusionsMap = buildStringMap(r.LabelInclusions)
+	r.LabelExclusionsMap = buildStringMap(r.LabelExclusions)
+	r.AnnotationInclusionsMap = buildStringMap(r.AnnotationInclusions)
+	r.AnnotationExclusionsMap = buildStringMap(r.AnnotationExclusions)
 }
 
 type SecondaryNetwork struct {
@@ -200,3 +213,11 @@ type NetworkTransformSubnetLabel struct {
 	CIDRs []string `yaml:"cidrs,omitempty" json:"cidrs,omitempty" doc:"list of CIDRs to match a label"`
 	Name  string   `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the label"`
 }
+
+func buildStringMap(items []string) map[string]struct{} {
+	m := make(map[string]struct{}, len(items))
+	for _, item := range items {
+		m[item] = struct{}{}
+	}
+	return m
+}

pkg/pipeline/transform/kubernetes/enrich.go

@@ -54,12 +54,16 @@ func Enrich(outputEntry config.GenericMap, rule *api.K8sRule) {
 	outputEntry[rule.OutputKeys.NetworkName] = kubeInfo.NetworkName
 	if rule.LabelsPrefix != "" {
 		for labelKey, labelValue := range kubeInfo.Labels {
-			outputEntry[rule.LabelsPrefix+"_"+labelKey] = labelValue
+			if shouldInclude(labelKey, rule.LabelInclusionsMap, rule.LabelExclusionsMap) {
+				outputEntry[rule.LabelsPrefix+"_"+labelKey] = labelValue
+			}
 		}
 	}
 	if rule.AnnotationsPrefix != "" {
 		for annotationKey, annotationValue := range kubeInfo.Annotations {
-			outputEntry[rule.AnnotationsPrefix+"_"+annotationKey] = annotationValue
+			if shouldInclude(annotationKey, rule.AnnotationInclusionsMap, rule.AnnotationExclusionsMap) {
+				outputEntry[rule.AnnotationsPrefix+"_"+annotationKey] = annotationValue
+			}
 		}
 	}
 	if kubeInfo.HostIP != "" {
@@ -147,3 +151,15 @@ func objectIsApp(namespace, name string, rule *api.K8sInfraRule) bool {
 	}
 	return true
 }
+
+// shouldInclude determines if an item should be included based on inclusion/exclusion maps.
+func shouldInclude(key string, inclusions, exclusions map[string]struct{}) bool {
+	if _, excluded := exclusions[key]; excluded {
+		return false
+	}
+	if len(inclusions) > 0 {
+		_, included := inclusions[key]
+		return included
+	}
+	return true
+}

pkg/pipeline/transform/kubernetes/enrich_test.go

@@ -663,3 +663,223 @@ func TestEnrich_LabelsAndAnnotationsPrefixes(t *testing.T) {
 		})
 	}
 }
+
+func TestEnrich_LabelsAnnotationsFiltering(t *testing.T) {
+	testData := map[string]*model.ResourceMetaData{
+		"10.0.0.10": {
+			ObjectMeta: v1.ObjectMeta{
+				Name:      "test-pod",
+				Namespace: "test-ns",
+				Labels: map[string]string{
+					"app":         "myapp",
+					"version":     "v1",
+					"environment": "prod",
+					"team":        "backend",
+				},
+				Annotations: map[string]string{
+					"annotation1": "value1",
+					"annotation2": "value2",
+					"annotation3": "value3",
+					"annotation4": "value4",
+				},
+			},
+			Kind: "Pod",
+		},
+	}
+
+	setupStubs(testData, nil, nodes)
+
+	tests := []struct {
+		name                 string
+		labelsPrefix         string
+		labelInclusions      []string
+		labelExclusions      []string
+		annotationsPrefix    string
+		annotationInclusions []string
+		annotationExclusions []string
+		expectedLabels       []string
+		expectedAnnotations  []string
+		notExpect            []string
+	}{
+		{
+			name:                "No filtering - all labels and annotations included",
+			labelsPrefix:        "k8s_labels",
+			annotationsPrefix:   "k8s_annotations",
+			expectedLabels:      []string{"k8s_labels_app", "k8s_labels_version", "k8s_labels_environment", "k8s_labels_team"},
+			expectedAnnotations: []string{"k8s_annotations_annotation1", "k8s_annotations_annotation2", "k8s_annotations_annotation3", "k8s_annotations_annotation4"},
+		},
+		{
+			name:                "Only label inclusions specified",
+			labelsPrefix:        "k8s_labels",
+			labelInclusions:     []string{"app", "version"},
+			annotationsPrefix:   "k8s_annotations",
+			expectedLabels:      []string{"k8s_labels_app", "k8s_labels_version"},
+			expectedAnnotations: []string{"k8s_annotations_annotation1", "k8s_annotations_annotation2", "k8s_annotations_annotation3", "k8s_annotations_annotation4"},
+			notExpect:           []string{"k8s_labels_environment", "k8s_labels_team"},
+		},
+		{
+			name:                "Only label exclusions specified",
+			labelsPrefix:        "k8s_labels",
+			labelExclusions:     []string{"environment", "team"},
+			annotationsPrefix:   "k8s_annotations",
+			expectedLabels:      []string{"k8s_labels_app", "k8s_labels_version"},
+			expectedAnnotations: []string{"k8s_annotations_annotation1", "k8s_annotations_annotation2", "k8s_annotations_annotation3", "k8s_annotations_annotation4"},
+			notExpect:           []string{"k8s_labels_environment", "k8s_labels_team"},
+		},
+		{
+			name:                "Both label inclusions and exclusions - exclusions take precedence",
+			labelsPrefix:        "k8s_labels",
+			labelInclusions:     []string{"app", "version", "environment"},
+			labelExclusions:     []string{"environment"},
+			annotationsPrefix:   "k8s_annotations",
+			expectedLabels:      []string{"k8s_labels_app", "k8s_labels_version"},
+			expectedAnnotations: []string{"k8s_annotations_annotation1", "k8s_annotations_annotation2", "k8s_annotations_annotation3", "k8s_annotations_annotation4"},
+			notExpect:           []string{"k8s_labels_environment", "k8s_labels_team"},
+		},
+		{
+			name:                 "Only annotation inclusions specified",
+			labelsPrefix:         "k8s_labels",
+			annotationsPrefix:    "k8s_annotations",
+			annotationInclusions: []string{"annotation1", "annotation3"},
+			expectedLabels:       []string{"k8s_labels_app", "k8s_labels_version", "k8s_labels_environment", "k8s_labels_team"},
+			expectedAnnotations:  []string{"k8s_annotations_annotation1", "k8s_annotations_annotation3"},
+			notExpect:            []string{"k8s_annotations_annotation2", "k8s_annotations_annotation4"},
+		},
+		{
+			name:                 "Only annotation exclusions specified",
+			labelsPrefix:         "k8s_labels",
+			annotationsPrefix:    "k8s_annotations",
+			annotationExclusions: []string{"annotation2", "annotation4"},
+			expectedLabels:       []string{"k8s_labels_app", "k8s_labels_version", "k8s_labels_environment", "k8s_labels_team"},
+			expectedAnnotations:  []string{"k8s_annotations_annotation1", "k8s_annotations_annotation3"},
+			notExpect:            []string{"k8s_annotations_annotation2", "k8s_annotations_annotation4"},
+		},
+		{
+			name:                 "Both annotation inclusions and exclusions - exclusions take precedence",
+			labelsPrefix:         "k8s_labels",
+			annotationsPrefix:    "k8s_annotations",
+			annotationInclusions: []string{"annotation1", "annotation2", "annotation3"},
+			annotationExclusions: []string{"annotation2"},
+			expectedLabels:       []string{"k8s_labels_app", "k8s_labels_version", "k8s_labels_environment", "k8s_labels_team"},
+			expectedAnnotations:  []string{"k8s_annotations_annotation1", "k8s_annotations_annotation3"},
+			notExpect:            []string{"k8s_annotations_annotation2", "k8s_annotations_annotation4"},
+		},
+		{
+			name:                 "Combined filtering for both labels and annotations",
+			labelsPrefix:         "k8s_labels",
+			labelInclusions:      []string{"app", "version", "team"},
+			labelExclusions:      []string{"team"},
+			annotationsPrefix:    "k8s_annotations",
+			annotationInclusions: []string{"annotation1", "annotation2"},
+			annotationExclusions: []string{"annotation1"},
+			expectedLabels:       []string{"k8s_labels_app", "k8s_labels_version"},
+			expectedAnnotations:  []string{"k8s_annotations_annotation2"},
+			notExpect:            []string{"k8s_labels_environment", "k8s_labels_team", "k8s_annotations_annotation1", "k8s_annotations_annotation3", "k8s_annotations_annotation4"},
+		},
+		{
+			name:                 "Empty prefix - no labels or annotations added",
+			labelsPrefix:         "",
+			labelInclusions:      []string{"app"},
+			annotationsPrefix:    "",
+			annotationInclusions: []string{"annotation1"},
+			notExpect:            []string{"k8s_labels_app", "k8s_labels_version", "k8s_labels_environment", "k8s_labels_team", "k8s_annotations_annotation1", "k8s_annotations_annotation2", "k8s_annotations_annotation3", "k8s_annotations_annotation4"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rule := api.TransformNetwork{
+				Rules: api.NetworkTransformRules{{
+					Type: api.NetworkAddKubernetes,
+					Kubernetes: &api.K8sRule{
+						IPField:              "SrcAddr",
+						Output:               "SrcK8s",
+						LabelsPrefix:         tt.labelsPrefix,
+						LabelInclusions:      tt.labelInclusions,
+						LabelExclusions:      tt.labelExclusions,
+						AnnotationsPrefix:    tt.annotationsPrefix,
+						AnnotationInclusions: tt.annotationInclusions,
+						AnnotationExclusions: tt.annotationExclusions,
+					},
+				}},
+			}
+			rule.Preprocess()
+
+			entry := config.GenericMap{
+				"SrcAddr": "10.0.0.10",
+			}
+
+			Enrich(entry, rule.Rules[0].Kubernetes)
+
+			for _, label := range tt.expectedLabels {
+				assert.Contains(t, entry, label, "Expected label %s to be present", label)
+			}
+			for _, annotation := range tt.expectedAnnotations {
+				assert.Contains(t, entry, annotation, "Expected annotation %s to be present", annotation)
+			}
+			for _, k := range tt.notExpect {
+				assert.NotContains(t, entry, k)
+			}
+		})
+	}
+}
+
+func TestShouldInclude(t *testing.T) {
+	tests := []struct {
+		name       string
+		key        string
+		inclusions map[string]struct{}
+		exclusions map[string]struct{}
+		expected   bool
+	}{
+		{
+			name:       "No inclusions or exclusions - should include",
+			key:        "test",
+			inclusions: map[string]struct{}{},
+			exclusions: map[string]struct{}{},
+			expected:   true,
+		},
+		{
+			name:       "Key in inclusions - should include",
+			key:        "test",
+			inclusions: map[string]struct{}{"test": {}},
+			exclusions: map[string]struct{}{},
+			expected:   true,
+		},
+		{
+			name:       "Key not in inclusions - should not include",
+			key:        "test",
+			inclusions: map[string]struct{}{"other": {}},
+			exclusions: map[string]struct{}{},
+			expected:   false,
+		},
+		{
+			name:       "Key in exclusions - should not include",
+			key:        "test",
+			inclusions: map[string]struct{}{},
+			exclusions: map[string]struct{}{"test": {}},
+			expected:   false,
+		},
+		{
+			name:       "Key in both inclusions and exclusions - exclusions win",
+			key:        "test",
+			inclusions: map[string]struct{}{"test": {}},
+			exclusions: map[string]struct{}{"test": {}},
+			expected:   false,
+		},
+		{
+			name:       "Key not in exclusions, no inclusions - should include",
+			key:        "test",
+			inclusions: map[string]struct{}{},
+			exclusions: map[string]struct{}{"other": {}},
+			expected:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := shouldInclude(tt.key, tt.inclusions, tt.exclusions)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}