From ccbe0149b3c246f289eccd151e06bd223bc3e670 Mon Sep 17 00:00:00 2001 From: Olivier Cazade Date: Tue, 12 Dec 2023 13:17:01 +0000 Subject: [PATCH 1/2] Add Kubernetes Infra transform rule --- docs/api.md | 5 +++ pkg/api/api.go | 1 + pkg/api/transform_network.go | 19 ++++++--- pkg/pipeline/transform/transform_network.go | 47 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/api.md b/docs/api.md index 894052604..e8e503d73 100644 --- a/docs/api.md +++ b/docs/api.md @@ -178,10 +178,15 @@ Following is the supported API format for network transformations: add_location: add output location fields from input add_service: add output network service field from input port and parameters protocol field add_kubernetes: add output kubernetes fields from input + add_kubernetes_infra: add output kubernetes isInfra field from input reinterpret_direction: reinterpret flow direction at the node level (instead of net interface), to ease the deduplication process add_ip_category: categorize IPs based on known subnets configuration parameters: parameters specific to type assignee: value needs to assign to output field + kubernetes_infra: Kubernetes infra rule specific configuration + inputs: entry inputs fields + output: entry output field + infra_prefixes: Namespace prefixes that will be tagged as infra kubeConfigPath: path to kubeconfig file (optional) servicesFile: path to services file (optional, default: /etc/services) protocolsFile: path to protocols file (optional, default: /etc/protocols) diff --git a/pkg/api/api.go b/pkg/api/api.go index 62c0758c9..2eb99a58b 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -48,6 +48,7 @@ const ( AddLocationRuleType = "add_location" AddServiceRuleType = "add_service" AddKubernetesRuleType = "add_kubernetes" + AddKubernetesInfraRuleType = "add_kubernetes_infra" ReinterpretDirectionRuleType = "reinterpret_direction" PromFilterExact = "exact" PromFilterPresence = "presence" diff --git a/pkg/api/transform_network.go b/pkg/api/transform_network.go index 53f685bd3..e27866de1 100644 --- a/pkg/api/transform_network.go +++ b/pkg/api/transform_network.go @@ -43,6 +43,7 @@ const ( OpAddLocation = "add_location" OpAddService = "add_service" OpAddKubernetes = "add_kubernetes" + OpAddKubernetesInfra = "add_kubernetes_infra" OpReinterpretDirection = "reinterpret_direction" OpAddIPCategory = "add_ip_category" ) @@ -52,6 +53,7 @@ type TransformNetworkOperationEnum struct { AddLocation string `yaml:"add_location" json:"add_location" doc:"add output location fields from input"` AddService string `yaml:"add_service" json:"add_service" doc:"add output network service field from input port and parameters protocol field"` AddKubernetes string `yaml:"add_kubernetes" json:"add_kubernetes" doc:"add output kubernetes fields from input"` + AddKubernetesInfra string `yaml:"add_kubernetes_infra" json:"add_kubernetes_infra" doc:"add output kubernetes isInfra field from input"` ReinterpretDirection string `yaml:"reinterpret_direction" json:"reinterpret_direction" doc:"reinterpret flow direction at the node level (instead of net interface), to ease the deduplication process"` AddIPCategory string `yaml:"add_ip_category" json:"add_ip_category" doc:"categorize IPs based on known subnets configuration"` } @@ -61,11 +63,18 @@ func TransformNetworkOperationName(operation string) string { } type NetworkTransformRule struct { - Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` - Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` - Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` - Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"` - Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"` + Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` + Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"` + Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"` + KubernetesInfra *K8sInfraRule `yaml:"kubernetes_infra,omitempty" json:"kubernetes_infra,omitempty" doc:"Kubernetes infra rule specific configuration"` +} + +type K8sInfraRule struct { + Inputs []string `yaml:"inputs,omitempty" json:"inputs,omitempty" doc:"entry inputs fields"` + Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` + InfraPrefix string `yaml:"infra_prefixes,omitempty" json:"infra_prefixes,omitempty" doc:"Namespace prefixes that will be tagged as infra"` } type NetworkTransformDirectionInfo struct { diff --git a/pkg/pipeline/transform/transform_network.go b/pkg/pipeline/transform/transform_network.go index ced776e4b..3969b5828 100644 --- a/pkg/pipeline/transform/transform_network.go +++ b/pkg/pipeline/transform/transform_network.go @@ -99,6 +99,10 @@ func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bo outputEntry[rule.Output] = serviceName case api.OpAddKubernetes: fillInK8s(outputEntry, rule) + + case api.OpAddKubernetesInfra: + fillInK8sInfra(outputEntry, rule) + case api.OpReinterpretDirection: reinterpretDirection(outputEntry, &n.DirectionInfo) case api.OpAddIPCategory: @@ -182,6 +186,47 @@ func fillInK8s(outputEntry config.GenericMap, rule api.NetworkTransformRule) { } } +func fillInK8sInfra(outputEntry config.GenericMap, rule api.NetworkTransformRule) { + if rule.KubernetesInfra == nil { + logrus.Error("transformation rule: Missing Kubernetes Infra configuration ") + return + } + outputEntry[rule.KubernetesInfra.Output] = "infra" + for _, input := range rule.KubernetesInfra.Inputs { + if objectIsApp(fmt.Sprintf("%s", outputEntry[input]), rule.KubernetesInfra.InfraPrefix) { + outputEntry[rule.KubernetesInfra.Output] = "app" + return + } + } +} + +const openshiftNamespacePrefix = "openshift-" +const openshiftPrefixLen = len(openshiftNamespacePrefix) + +func objectIsApp(addr string, additionalInfraPrefix string) bool { + obj, err := kubernetes.Data.GetInfo(addr) + if err != nil { + logrus.WithError(err).Tracef("can't find kubernetes info for IP %s", addr) + return false + } + nsLen := len(obj.Namespace) + additionalPrefixLen := len(additionalInfraPrefix) + if nsLen == 0 { + return false + } + if nsLen >= openshiftPrefixLen && obj.Namespace[:openshiftPrefixLen] == openshiftNamespacePrefix { + return false + } + if nsLen >= additionalPrefixLen && obj.Namespace[:additionalPrefixLen] == additionalInfraPrefix { + return false + } + //Special case with openshift and kubernetes service in default namespace + if obj.Namespace == "default" && (obj.Name == "kubernetes" || obj.Name == "openshift") { + return false + } + return true +} + func (n *Network) categorizeIP(ip net.IP) string { if ip != nil { for _, subnetCat := range n.categories { @@ -211,6 +256,8 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) { needToInitLocationDB = true case api.OpAddKubernetes: needToInitKubeData = true + case api.OpAddKubernetesInfra: + needToInitKubeData = true case api.OpAddService: needToInitNetworkServices = true case api.OpReinterpretDirection: From cff2ee7f8432fdf53dbf7ae9c3a424ac0f0d69c6 Mon Sep 17 00:00:00 2001 From: Olivier Cazade Date: Fri, 19 Jan 2024 15:17:11 +0000 Subject: [PATCH 2/2] Update pkg/pipeline/transform/transform_network.go Co-authored-by: Julien Pinsonneau <91894519+jpinsonneau@users.noreply.github.com> --- pkg/pipeline/transform/transform_network.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/pipeline/transform/transform_network.go b/pkg/pipeline/transform/transform_network.go index 3969b5828..c37a9f56f 100644 --- a/pkg/pipeline/transform/transform_network.go +++ b/pkg/pipeline/transform/transform_network.go @@ -99,10 +99,8 @@ func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bo outputEntry[rule.Output] = serviceName case api.OpAddKubernetes: fillInK8s(outputEntry, rule) - case api.OpAddKubernetesInfra: fillInK8sInfra(outputEntry, rule) - case api.OpReinterpretDirection: reinterpretDirection(outputEntry, &n.DirectionInfo) case api.OpAddIPCategory: