Skip to content

Commit

Permalink
Subnet labels: some renaming, allow skipping
Browse files Browse the repository at this point in the history 
- rename IPCategories -> SubnetLabels
- allow to skip applying labels when a given field is present (in order
  to save space / processing by avoiding redundant information)

Adapt to refactored API

remove skipping option (not useful until proved otherwise)
  • Loading branch information
@jotak
jotak committed Mar 25, 2024
1 parent d2b2352 commit b22cb55
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 41 deletions.
10 changes: 5 additions & 5 deletions docs/api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ Following is the supported API format for network transformations:
add_kubernetes: add output kubernetes fields from input
add_kubernetes_infra: add output kubernetes isInfra field from input
reinterpret_direction: reinterpret flow direction at the node level (instead of net interface), to ease the deduplication process
add_ip_category: categorize IPs based on known subnets configuration
add_subnet_label: categorize IPs based on known subnets configuration
kubernetes_infra: Kubernetes infra rule configuration
inputs: entry inputs fields
output: entry output field
Expand All @@ -239,7 +239,7 @@ Following is the supported API format for network transformations:
add_location: Add location rule configuration
input: entry input field
output: entry output field
add_ip_category: Add ip category rule configuration
add_subnet_label: Add subnet label rule configuration
input: entry input field
output: entry output field
add_service: Add service rule configuration
Expand All @@ -249,9 +249,9 @@ Following is the supported API format for network transformations:
kubeConfigPath: path to kubeconfig file (optional)
servicesFile: path to services file (optional, default: /etc/services)
protocolsFile: path to protocols file (optional, default: /etc/protocols)
ipCategories: configure IP categories
cidrs: list of CIDRs to match a category
name: name of the category
subnetLabels: configure subnet and IPs custom labels
cidrs: list of CIDRs to match a label
name: name of the label
directionInfo: information to reinterpret flow direction (optional, to use with reinterpret_direction rule)
reporterIPField: field providing the reporter (agent) host IP
srcHostField: source host field
Expand Down
17 changes: 11 additions & 6 deletions pkg/api/transform_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type TransformNetwork struct {
KubeConfigPath string `yaml:"kubeConfigPath,omitempty" json:"kubeConfigPath,omitempty" doc:"path to kubeconfig file (optional)"`
ServicesFile string `yaml:"servicesFile,omitempty" json:"servicesFile,omitempty" doc:"path to services file (optional, default: /etc/services)"`
ProtocolsFile string `yaml:"protocolsFile,omitempty" json:"protocolsFile,omitempty" doc:"path to protocols file (optional, default: /etc/protocols)"`
IPCategories []NetworkTransformIPCategory `yaml:"ipCategories,omitempty" json:"ipCategories,omitempty" doc:"configure IP categories"`
SubnetLabels []NetworkTransformSubnetLabel `yaml:"subnetLabels,omitempty" json:"subnetLabels,omitempty" doc:"configure subnet and IPs custom labels"`
DirectionInfo NetworkTransformDirectionInfo `yaml:"directionInfo,omitempty" json:"directionInfo,omitempty" doc:"information to reinterpret flow direction (optional, to use with reinterpret_direction rule)"`
}

Expand All @@ -48,7 +48,7 @@ const (
NetworkAddKubernetes TransformNetworkOperationEnum = "add_kubernetes" // add output kubernetes fields from input
NetworkAddKubernetesInfra TransformNetworkOperationEnum = "add_kubernetes_infra" // add output kubernetes isInfra field from input
NetworkReinterpretDirection TransformNetworkOperationEnum = "reinterpret_direction" // reinterpret flow direction at the node level (instead of net interface), to ease the deduplication process
NetworkAddIPCategory TransformNetworkOperationEnum = "add_ip_category" // categorize IPs based on known subnets configuration
NetworkAddSubnetLabel TransformNetworkOperationEnum = "add_subnet_label" // categorize IPs based on known subnets configuration
)

type NetworkTransformRule struct {
Expand All @@ -57,7 +57,7 @@ type NetworkTransformRule struct {
Kubernetes *K8sRule `yaml:"kubernetes,omitempty" json:"kubernetes,omitempty" doc:"Kubernetes rule configuration"`
AddSubnet *NetworkAddSubnetRule `yaml:"add_subnet,omitempty" json:"add_subnet,omitempty" doc:"Add subnet rule configuration"`
AddLocation *NetworkGenericRule `yaml:"add_location,omitempty" json:"add_location,omitempty" doc:"Add location rule configuration"`
AddIPCategory *NetworkGenericRule `yaml:"add_ip_category,omitempty" json:"add_ip_category,omitempty" doc:"Add ip category rule configuration"`
AddSubnetLabel *NetworkAddSubnetLabelRule `yaml:"add_subnet_label,omitempty" json:"add_subnet_label,omitempty" doc:"Add subnet label rule configuration"`
AddService *NetworkAddServiceRule `yaml:"add_service,omitempty" json:"add_service,omitempty" doc:"Add service rule configuration"`
}

Expand Down Expand Up @@ -92,6 +92,11 @@ type NetworkAddSubnetRule struct {
SubnetMask string `yaml:"subnet_mask,omitempty" json:"subnet_mask,omitempty" doc:"subnet mask field"`
}

type NetworkAddSubnetLabelRule struct {
Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"`
Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
}

type NetworkAddServiceRule struct {
Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"`
Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
Expand All @@ -108,7 +113,7 @@ type NetworkTransformDirectionInfo struct {

type NetworkTransformRules []NetworkTransformRule

type NetworkTransformIPCategory struct {
CIDRs []string `yaml:"cidrs,omitempty" json:"cidrs,omitempty" doc:"list of CIDRs to match a category"`
Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the category"`
type NetworkTransformSubnetLabel struct {
CIDRs []string `yaml:"cidrs,omitempty" json:"cidrs,omitempty" doc:"list of CIDRs to match a label"`
Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the label"`
}
53 changes: 29 additions & 24 deletions pkg/pipeline/transform/transform_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@ var log = logrus.WithField("component", "transform.Network")

type Network struct {
api.TransformNetwork
svcNames *netdb.ServiceNames
categories []subnetCategory
ipCatCache *utils.TimedCache
svcNames *netdb.ServiceNames
snLabels []subnetLabel
ipLabelCache *utils.TimedCache
}

type subnetCategory struct {
type subnetLabel struct {
cidrs []*net.IPNet
name string
}
Expand Down Expand Up @@ -120,18 +120,22 @@ func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bo
kubernetes.EnrichLayer(outputEntry, rule.KubernetesInfra)
case api.NetworkReinterpretDirection:
reinterpretDirection(outputEntry, &n.DirectionInfo)
case api.NetworkAddIPCategory:
if rule.AddIPCategory == nil {
logrus.Error("AddIPCategory rule: Missing configuration ")
case api.NetworkAddSubnetLabel:
if rule.AddSubnetLabel == nil {
logrus.Error("AddSubnetLabel rule: Missing configuration ")
continue
}
if strIP, ok := outputEntry[rule.AddIPCategory.Input].(string); ok {
cat, ok := n.ipCatCache.GetCacheEntry(strIP)
if !ok {
cat = n.categorizeIP(net.ParseIP(strIP))
n.ipCatCache.UpdateCacheEntry(strIP, cat)
if anyIP, ok := outputEntry[rule.AddSubnetLabel.Input]; ok {
if strIP, ok := anyIP.(string); ok {
lbl, ok := n.ipLabelCache.GetCacheEntry(strIP)
if !ok {
lbl = n.applySubnetLabel(strIP)
n.ipLabelCache.UpdateCacheEntry(strIP, lbl)
}
if lbl != "" {
outputEntry[rule.AddSubnetLabel.Output] = lbl
}
}
outputEntry[rule.AddIPCategory.Output] = cat
}

default:
Expand All @@ -142,9 +146,10 @@ func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bo
return outputEntry, true
}

func (n *Network) categorizeIP(ip net.IP) string {
func (n *Network) applySubnetLabel(strIP string) string {
ip := net.ParseIP(strIP)
if ip != nil {
for _, subnetCat := range n.categories {
for _, subnetCat := range n.snLabels {
for _, cidr := range subnetCat.cidrs {
if cidr.Contains(ip) {
return subnetCat.name
Expand Down Expand Up @@ -181,9 +186,9 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
if err := validateReinterpretDirectionConfig(&jsonNetworkTransform.DirectionInfo); err != nil {
return nil, err
}
case api.NetworkAddIPCategory:
if len(jsonNetworkTransform.IPCategories) == 0 {
return nil, fmt.Errorf("a rule '%s' was found, but there are no IP categories configured", api.NetworkAddIPCategory)
case api.NetworkAddSubnetLabel:
if len(jsonNetworkTransform.SubnetLabels) == 0 {
return nil, fmt.Errorf("a rule '%s' was found, but there are no subnet labels configured", api.NetworkAddSubnetLabel)
}
case api.NetworkAddSubnet:
}
Expand Down Expand Up @@ -223,8 +228,8 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
}
}

var subnetCats []subnetCategory
for _, category := range jsonNetworkTransform.IPCategories {
var subnetCats []subnetLabel
for _, category := range jsonNetworkTransform.SubnetLabels {
var cidrs []*net.IPNet
for _, cidr := range category.CIDRs {
_, parsed, err := net.ParseCIDR(cidr)
Expand All @@ -234,7 +239,7 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
cidrs = append(cidrs, parsed)
}
if len(cidrs) > 0 {
subnetCats = append(subnetCats, subnetCategory{name: category.Name, cidrs: cidrs})
subnetCats = append(subnetCats, subnetLabel{name: category.Name, cidrs: cidrs})
}
}

Expand All @@ -243,8 +248,8 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
Rules: jsonNetworkTransform.Rules,
DirectionInfo: jsonNetworkTransform.DirectionInfo,
},
svcNames: servicesDB,
categories: subnetCats,
ipCatCache: utils.NewQuietExpiringTimedCache(2 * time.Minute),
svcNames: servicesDB,
snLabels: subnetCats,
ipLabelCache: utils.NewQuietExpiringTimedCache(2 * time.Minute),
}, nil
}
11 changes: 5 additions & 6 deletions pkg/pipeline/transform/transform_network_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,12 +227,12 @@ func Test_Categorize(t *testing.T) {
Transform: &config.Transform{
Network: &api.TransformNetwork{
Rules: []api.NetworkTransformRule{
{Type: api.NetworkAddIPCategory, AddIPCategory: &api.NetworkGenericRule{Input: "addr1", Output: "cat1"}},
{Type: api.NetworkAddIPCategory, AddIPCategory: &api.NetworkGenericRule{Input: "addr2", Output: "cat2"}},
{Type: api.NetworkAddIPCategory, AddIPCategory: &api.NetworkGenericRule{Input: "addr3", Output: "cat3"}},
{Type: api.NetworkAddIPCategory, AddIPCategory: &api.NetworkGenericRule{Input: "addr4", Output: "cat4"}},
{Type: api.NetworkAddSubnetLabel, AddSubnetLabel: &api.NetworkAddSubnetLabelRule{Input: "addr1", Output: "cat1"}},
{Type: api.NetworkAddSubnetLabel, AddSubnetLabel: &api.NetworkAddSubnetLabelRule{Input: "addr2", Output: "cat2"}},
{Type: api.NetworkAddSubnetLabel, AddSubnetLabel: &api.NetworkAddSubnetLabelRule{Input: "addr3", Output: "cat3"}},
{Type: api.NetworkAddSubnetLabel, AddSubnetLabel: &api.NetworkAddSubnetLabelRule{Input: "addr4", Output: "cat4"}},
},
IPCategories: []api.NetworkTransformIPCategory{{
SubnetLabels: []api.NetworkTransformSubnetLabel{{
Name: "Pods overlay",
CIDRs: []string{"10.0.0.0/8"},
}, {
Expand All @@ -257,7 +257,6 @@ func Test_Categorize(t *testing.T) {
"addr2": "100.1.2.3",
"cat2": "MySite.com",
"addr3": "100.2.3.4",
"cat3": "",
"addr4": "101.1.0.0",
"cat4": "MySite.com",
}, output)
Expand Down

0 comments on commit b22cb55

Please sign in to comment.