Enum replacement in API (#614)

* Enum replacement in API

First step / PoC for fixing #608
Starting with metrics filters only; other enums should follow

* simplify sed for docgen

* Complete using new enum types

* Generic transform: use same pattern as network transform

Makefile

@@ -121,6 +121,7 @@ build: validate_go lint build_code docs ## Build flowlogs-pipeline executable an
 docs: FORCE ## Update flowlogs-pipeline documentation
 	@./hack/update-docs.sh
 	@go run cmd/apitodoc/main.go > docs/api.md
+	@./hack/update-enum-docs.sh
 	@go run cmd/operationalmetricstodoc/main.go > docs/operational-metrics.md
 
 .PHONY: clean

README.md

@@ -374,9 +374,11 @@ pipeline:
     - type: filter
       filter:
         rules:
-        - input: SrcPort
-          type: remove_entry_if_exists 
+        - type: remove_entry_if_exists
+          removeEntryIfExists:
+            input: SrcPort
 ```
+
 Using `remove_entry_if_doesnt_exist` in the rule reverses the logic and will not remove the above example entry
 Using `remove_field` in the rule `type` instead, results in outputting the entry after
 removal of only the `SrcPort` key and value 

cmd/apitodoc/main.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"reflect"
@@ -29,9 +30,12 @@ import (
 
 func iterate(output io.Writer, data interface{}, indent int) {
 	newIndent := indent + 1
-	dataType := reflect.ValueOf(data).Kind()
-	// DEBUG code: dataTypeName := reflect.ValueOf(data).Type().String()
 	d := reflect.ValueOf(data)
+	dataType := d.Kind()
+	dataTypeName, err := getTypeName(d)
+	if err != nil {
+		dataTypeName = "(unknown)"
+	}
 	if dataType == reflect.Slice || dataType == reflect.Map {
 		// DEBUG code: fmt.Fprintf(output, "%s %s <-- %s \n",strings.Repeat(" ",4*indent),dataTypeName,dataType )
 		zeroElement := reflect.Zero(reflect.ValueOf(data).Type().Elem()).Interface()
@@ -43,17 +47,8 @@ func iterate(output io.Writer, data interface{}, indent int) {
 			val := reflect.Indirect(reflect.ValueOf(data))
 			fieldName := val.Type().Field(i).Tag.Get(api.TagYaml)
 			fieldName = strings.ReplaceAll(fieldName, ",omitempty", "")
-
 			fieldDocTag := val.Type().Field(i).Tag.Get(api.TagDoc)
-			fieldEnumTag := val.Type().Field(i).Tag.Get(api.TagEnum)
 
-			if fieldEnumTag != "" {
-				enumType := api.GetEnumReflectionTypeByFieldName(fieldEnumTag)
-				zeroElement := reflect.Zero(enumType).Interface()
-				fmt.Fprintf(output, "%s %s: (enum) %s\n", strings.Repeat(" ", 4*newIndent), fieldName, fieldDocTag)
-				iterate(output, zeroElement, newIndent)
-				continue
-			}
 			if fieldDocTag != "" {
 				if fieldDocTag[0:1] == "#" {
 					fmt.Fprintf(output, "\n%s\n", fieldDocTag)
@@ -75,9 +70,22 @@ func iterate(output io.Writer, data interface{}, indent int) {
 		// Since we only "converted" Ptr to Struct and the actual output is done in the next iteration, we call
 		// iterate() with the same `indent` as the current level
 		iterate(output, zeroElement, indent)
+	} else if strings.HasPrefix(dataTypeName, "api.") && strings.HasSuffix(dataTypeName, "Enum") {
+		// set placeholder for enum
+		fmt.Fprintf(output, "placeholder @%s:%d@\n", strings.TrimPrefix(dataTypeName, "api."), 4*newIndent)
 	}
 }
 
+func getTypeName(d reflect.Value) (name string, err error) {
+	defer func() {
+		if recover() != nil {
+			err = errors.New("unknown type name")
+		}
+	}()
+	name = d.Type().String()
+	return
+}
+
 func main() {
 	output := new(bytes.Buffer)
 	iterate(output, api.API{}, 0)

hack/update-enum-docs.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+md_file="./docs/api.md"
+
+grep "placeholder @" "$md_file" | while read line; do
+    type=$(echo "$line" | sed -r 's/^placeholder @(.*):(.*)@/\1/')
+    indent=$(echo "$line" | sed -r 's/^placeholder @(.*):(.*)@/\2/')
+    repl=$(go doc -all -short -C pkg/api $type | grep "${type} =" | sed -r "s/^.*\"(.*)\".*\/\/ (.*)/$(printf "%${indent}s")\1: \2/")
+    awk -v inject="${repl}" "/placeholder @$type:$indent@/{print inject;next}1" $md_file > "$md_file.tmp"
+    rm $md_file
+    mv "$md_file.tmp" $md_file
+done

pkg/api/api.go

@@ -18,47 +18,33 @@
 package api
 
 const (
-	FileType                     = "file"
-	FileLoopType                 = "file_loop"
-	FileChunksType               = "file_chunks"
-	SyntheticType                = "synthetic"
-	CollectorType                = "collector"
-	StdinType                    = "stdin"
-	GRPCType                     = "grpc"
-	FakeType                     = "fake"
-	KafkaType                    = "kafka"
-	S3Type                       = "s3"
-	OtlpLogsType                 = "otlplogs"
-	OtlpMetricsType              = "otlpmetrics"
-	OtlpTracesType               = "otlptraces"
-	StdoutType                   = "stdout"
-	LokiType                     = "loki"
-	IpfixType                    = "ipfix"
-	AggregateType                = "aggregates"
-	TimebasedType                = "timebased"
-	PromType                     = "prom"
-	GenericType                  = "generic"
-	NetworkType                  = "network"
-	FilterType                   = "filter"
-	ConnTrackType                = "conntrack"
-	NoneType                     = "none"
-	AddRegExIfRuleType           = "add_regex_if"
-	AddSubnetRuleType            = "add_subnet"
-	AddLocationRuleType          = "add_location"
-	AddServiceRuleType           = "add_service"
-	AddKubernetesRuleType        = "add_kubernetes"
-	AddKubernetesInfraRuleType   = "add_kubernetes_infra"
-	ReinterpretDirectionRuleType = "reinterpret_direction"
-	PromFilterEqual              = "equal"
-	PromFilterNotEqual           = "not_equal"
-	PromFilterPresence           = "presence"
-	PromFilterAbsence            = "absence"
-	PromFilterRegex              = "match_regex"
-	PromFilterNotRegex           = "not_match_regex"
+	FileType        = "file"
+	FileLoopType    = "file_loop"
+	FileChunksType  = "file_chunks"
+	SyntheticType   = "synthetic"
+	CollectorType   = "collector"
+	StdinType       = "stdin"
+	GRPCType        = "grpc"
+	FakeType        = "fake"
+	KafkaType       = "kafka"
+	S3Type          = "s3"
+	OtlpLogsType    = "otlplogs"
+	OtlpMetricsType = "otlpmetrics"
+	OtlpTracesType  = "otlptraces"
+	StdoutType      = "stdout"
+	LokiType        = "loki"
+	IpfixType       = "ipfix"
+	AggregateType   = "aggregates"
+	TimebasedType   = "timebased"
+	PromType        = "prom"
+	GenericType     = "generic"
+	NetworkType     = "network"
+	FilterType      = "filter"
+	ConnTrackType   = "conntrack"
+	NoneType        = "none"
 
 	TagYaml = "yaml"
 	TagDoc  = "doc"
-	TagEnum = "enum"
 )
 
 // Note: items beginning with doc: "## title" are top level items that get divided into sections inside api.md.

pkg/api/conntrack.go

@@ -28,24 +28,23 @@ const (
 )
 
 type ConnTrack struct {
-	KeyDefinition         KeyDefinition              `yaml:"keyDefinition,omitempty" json:"keyDefinition,omitempty" doc:"fields that are used to identify the connection"`
-	OutputRecordTypes     []string                   `yaml:"outputRecordTypes,omitempty" json:"outputRecordTypes,omitempty" enum:"ConnTrackOutputRecordTypeEnum" doc:"output record types to emit"`
-	OutputFields          []OutputField              `yaml:"outputFields,omitempty" json:"outputFields,omitempty" doc:"list of output fields"`
-	Scheduling            []ConnTrackSchedulingGroup `yaml:"scheduling,omitempty" json:"scheduling,omitempty" doc:"list of timeouts and intervals to apply per selector"`
-	MaxConnectionsTracked int                        `yaml:"maxConnectionsTracked,omitempty" json:"maxConnectionsTracked,omitempty" doc:"maximum number of connections we keep in our cache (0 means no limit)"`
-	TCPFlags              ConnTrackTCPFlags          `yaml:"tcpFlags,omitempty" json:"tcpFlags,omitempty" doc:"settings for handling TCP flags"`
+	KeyDefinition         KeyDefinition                   `yaml:"keyDefinition,omitempty" json:"keyDefinition,omitempty" doc:"fields that are used to identify the connection"`
+	OutputRecordTypes     []ConnTrackOutputRecordTypeEnum `yaml:"outputRecordTypes,omitempty" json:"outputRecordTypes,omitempty" doc:"(enum) output record types to emit"`
+	OutputFields          []OutputField                   `yaml:"outputFields,omitempty" json:"outputFields,omitempty" doc:"list of output fields"`
+	Scheduling            []ConnTrackSchedulingGroup      `yaml:"scheduling,omitempty" json:"scheduling,omitempty" doc:"list of timeouts and intervals to apply per selector"`
+	MaxConnectionsTracked int                             `yaml:"maxConnectionsTracked,omitempty" json:"maxConnectionsTracked,omitempty" doc:"maximum number of connections we keep in our cache (0 means no limit)"`
+	TCPFlags              ConnTrackTCPFlags               `yaml:"tcpFlags,omitempty" json:"tcpFlags,omitempty" doc:"settings for handling TCP flags"`
 }
 
-type ConnTrackOutputRecordTypeEnum struct {
-	NewConnection string `yaml:"newConnection" json:"newConnection" doc:"New connection"`
-	EndConnection string `yaml:"endConnection" json:"endConnection" doc:"End connection"`
-	Heartbeat     string `yaml:"heartbeat" json:"heartbeat" doc:"Heartbeat"`
-	FlowLog       string `yaml:"flowLog" json:"flowLog" doc:"Flow log"`
-}
+type ConnTrackOutputRecordTypeEnum string
 
-func ConnTrackOutputRecordTypeName(operation string) string {
-	return GetEnumName(ConnTrackOutputRecordTypeEnum{}, operation)
-}
+const (
+	// For doc generation, enum definitions must match format `Constant Type = "value" // doc`
+	ConnTrackNewConnection ConnTrackOutputRecordTypeEnum = "newConnection" // New connection
+	ConnTrackEndConnection ConnTrackOutputRecordTypeEnum = "endConnection" // End connection
+	ConnTrackHeartbeat     ConnTrackOutputRecordTypeEnum = "heartbeat"     // Heartbeat
+	ConnTrackFlowLog       ConnTrackOutputRecordTypeEnum = "flowLog"       // Flow log
+)
 
 type KeyDefinition struct {
 	FieldGroups []FieldGroup  `yaml:"fieldGroups,omitempty" json:"fieldGroups,omitempty" doc:"list of field group definitions"`
@@ -70,21 +69,24 @@ type ConnTrackHash struct {
 }
 
 type OutputField struct {
-	Name          string `yaml:"name,omitempty" json:"name,omitempty" doc:"output field name"`
-	Operation     string `yaml:"operation,omitempty" json:"operation,omitempty" enum:"ConnTrackOperationEnum" doc:"aggregate operation on the field value"`
-	SplitAB       bool   `yaml:"splitAB,omitempty" json:"splitAB,omitempty" doc:"When true, 2 output fields will be created. One for A->B and one for B->A flows."`
-	Input         string `yaml:"input,omitempty" json:"input,omitempty" doc:"The input field to base the operation on. When omitted, 'name' is used"`
-	ReportMissing bool   `yaml:"reportMissing,omitempty" json:"reportMissing,omitempty" doc:"When true, missing input will produce MissingFieldError metric and error logs"`
+	Name          string                 `yaml:"name,omitempty" json:"name,omitempty" doc:"output field name"`
+	Operation     ConnTrackOperationEnum `yaml:"operation,omitempty" json:"operation,omitempty" doc:"(enum) aggregate operation on the field value"`
+	SplitAB       bool                   `yaml:"splitAB,omitempty" json:"splitAB,omitempty" doc:"When true, 2 output fields will be created. One for A->B and one for B->A flows."`
+	Input         string                 `yaml:"input,omitempty" json:"input,omitempty" doc:"The input field to base the operation on. When omitted, 'name' is used"`
+	ReportMissing bool                   `yaml:"reportMissing,omitempty" json:"reportMissing,omitempty" doc:"When true, missing input will produce MissingFieldError metric and error logs"`
 }
 
-type ConnTrackOperationEnum struct {
-	Sum   string `yaml:"sum" json:"sum" doc:"sum"`
-	Count string `yaml:"count" json:"count" doc:"count"`
-	Min   string `yaml:"min" json:"min" doc:"min"`
-	Max   string `yaml:"max" json:"max" doc:"max"`
-	First string `yaml:"first" json:"first" doc:"first"`
-	Last  string `yaml:"last" json:"last" doc:"last"`
-}
+type ConnTrackOperationEnum string
+
+const (
+	// For doc generation, enum definitions must match format `Constant Type = "value" // doc`
+	ConnTrackSum   ConnTrackOperationEnum = "sum"   // sum
+	ConnTrackCount ConnTrackOperationEnum = "count" // count
+	ConnTrackMin   ConnTrackOperationEnum = "min"   // min
+	ConnTrackMax   ConnTrackOperationEnum = "max"   // max
+	ConnTrackFirst ConnTrackOperationEnum = "first" // first
+	ConnTrackLast  ConnTrackOperationEnum = "last"  // last
+)
 
 type ConnTrackSchedulingGroup struct {
 	Selector             map[string]interface{} `yaml:"selector,omitempty" json:"selector,omitempty" doc:"key-value map to match against connection fields to apply this scheduling"`
@@ -93,10 +95,6 @@ type ConnTrackSchedulingGroup struct {
 	HeartbeatInterval    Duration               `yaml:"heartbeatInterval,omitempty" json:"heartbeatInterval,omitempty" doc:"duration of time to wait between heartbeat reports of a connection"`
 }
 
-func ConnTrackOperationName(operation string) string {
-	return GetEnumName(ConnTrackOperationEnum{}, operation)
-}
-
 type ConnTrackTCPFlags struct {
 	FieldName           string `yaml:"fieldName,omitempty" json:"fieldName,omitempty" doc:"name of the field containing TCP flags"`
 	DetectEndConnection bool   `yaml:"detectEndConnection,omitempty" json:"detectEndConnection,omitempty" doc:"detect end connections by FIN flag"`
@@ -255,28 +253,28 @@ func addToSet(set map[string]struct{}, item string) bool {
 	return true
 }
 
-func isOperationValid(value string, splitAB bool) bool {
+func isOperationValid(value ConnTrackOperationEnum, splitAB bool) bool {
 	valid := true
 	switch value {
-	case ConnTrackOperationName("Sum"):
-	case ConnTrackOperationName("Count"):
-	case ConnTrackOperationName("Min"):
-	case ConnTrackOperationName("Max"):
-	case ConnTrackOperationName("First"), ConnTrackOperationName("Last"):
+	case ConnTrackSum:
+	case ConnTrackCount:
+	case ConnTrackMin:
+	case ConnTrackMax:
+	case ConnTrackFirst, ConnTrackLast:
 		valid = !splitAB
 	default:
 		valid = false
 	}
 	return valid
 }
 
-func isOutputRecordTypeValid(value string) bool {
+func isOutputRecordTypeValid(value ConnTrackOutputRecordTypeEnum) bool {
 	valid := true
 	switch value {
-	case ConnTrackOutputRecordTypeName("NewConnection"):
-	case ConnTrackOutputRecordTypeName("EndConnection"):
-	case ConnTrackOutputRecordTypeName("Heartbeat"):
-	case ConnTrackOutputRecordTypeName("FlowLog"):
+	case ConnTrackNewConnection:
+	case ConnTrackEndConnection:
+	case ConnTrackHeartbeat:
+	case ConnTrackFlowLog:
 	default:
 		valid = false
 	}

pkg/api/conntrack_test.go

@@ -141,7 +141,7 @@ func TestConnTrackValidate(t *testing.T) {
 		{
 			"Unknown output record",
 			ConnTrack{
-				OutputRecordTypes: []string{"unknown"},
+				OutputRecordTypes: []ConnTrackOutputRecordTypeEnum{"unknown"},
 			},
 			conntrackInvalidError{unknownOutputRecord: true},
 		},

pkg/api/decoder.go

@@ -1,14 +1,13 @@
 package api
 
 type Decoder struct {
-	Type string `yaml:"type" json:"type" enum:"DecoderEnum" doc:"one of the following:"`
+	Type DecoderEnum `yaml:"type" json:"type" doc:"(enum) one of the following:"`
 }
 
-type DecoderEnum struct {
-	JSON     string `yaml:"json" json:"json" doc:"JSON decoder"`
-	Protobuf string `yaml:"protobuf" json:"protobuf" doc:"Protobuf decoder"`
-}
+type DecoderEnum string
 
-func DecoderName(decoder string) string {
-	return GetEnumName(DecoderEnum{}, decoder)
-}
+const (
+	// For doc generation, enum definitions must match format `Constant Type = "value" // doc`
+	DecoderJSON     DecoderEnum = "json"     // JSON decoder
+	DecoderProtobuf DecoderEnum = "protobuf" // Protobuf decoder
+)

pkg/api/encode_kafka.go

@@ -18,25 +18,24 @@
 package api
 
 type EncodeKafka struct {
-	Address      string      `yaml:"address" json:"address" doc:"address of kafka server"`
-	Topic        string      `yaml:"topic" json:"topic" doc:"kafka topic to write to"`
-	Balancer     string      `yaml:"balancer,omitempty" json:"balancer,omitempty" enum:"KafkaEncodeBalancerEnum" doc:"one of the following:"`
-	WriteTimeout int64       `yaml:"writeTimeout,omitempty" json:"writeTimeout,omitempty" doc:"timeout (in seconds) for write operation performed by the Writer"`
-	ReadTimeout  int64       `yaml:"readTimeout,omitempty" json:"readTimeout,omitempty" doc:"timeout (in seconds) for read operation performed by the Writer"`
-	BatchBytes   int64       `yaml:"batchBytes,omitempty" json:"batchBytes,omitempty" doc:"limit the maximum size of a request in bytes before being sent to a partition"`
-	BatchSize    int         `yaml:"batchSize,omitempty" json:"batchSize,omitempty" doc:"limit on how many messages will be buffered before being sent to a partition"`
-	TLS          *ClientTLS  `yaml:"tls" json:"tls" doc:"TLS client configuration (optional)"`
-	SASL         *SASLConfig `yaml:"sasl" json:"sasl" doc:"SASL configuration (optional)"`
+	Address      string                  `yaml:"address" json:"address" doc:"address of kafka server"`
+	Topic        string                  `yaml:"topic" json:"topic" doc:"kafka topic to write to"`
+	Balancer     KafkaEncodeBalancerEnum `yaml:"balancer,omitempty" json:"balancer,omitempty" doc:"(enum) one of the following:"`
+	WriteTimeout int64                   `yaml:"writeTimeout,omitempty" json:"writeTimeout,omitempty" doc:"timeout (in seconds) for write operation performed by the Writer"`
+	ReadTimeout  int64                   `yaml:"readTimeout,omitempty" json:"readTimeout,omitempty" doc:"timeout (in seconds) for read operation performed by the Writer"`
+	BatchBytes   int64                   `yaml:"batchBytes,omitempty" json:"batchBytes,omitempty" doc:"limit the maximum size of a request in bytes before being sent to a partition"`
+	BatchSize    int                     `yaml:"batchSize,omitempty" json:"batchSize,omitempty" doc:"limit on how many messages will be buffered before being sent to a partition"`
+	TLS          *ClientTLS              `yaml:"tls" json:"tls" doc:"TLS client configuration (optional)"`
+	SASL         *SASLConfig             `yaml:"sasl" json:"sasl" doc:"SASL configuration (optional)"`
 }
 
-type KafkaEncodeBalancerEnum struct {
-	RoundRobin string `yaml:"roundRobin" json:"roundRobin" doc:"RoundRobin balancer"`
-	LeastBytes string `yaml:"leastBytes" json:"leastBytes" doc:"LeastBytes balancer"`
-	Hash       string `yaml:"hash" json:"hash" doc:"Hash balancer"`
-	Crc32      string `yaml:"crc32" json:"crc32" doc:"Crc32 balancer"`
-	Murmur2    string `yaml:"murmur2" json:"murmur2" doc:"Murmur2 balancer"`
-}
+type KafkaEncodeBalancerEnum string
 
-func KafkaEncodeBalancerName(operation string) string {
-	return GetEnumName(KafkaEncodeBalancerEnum{}, operation)
-}
+const (
+	// For doc generation, enum definitions must match format `Constant Type = "value" // doc`
+	KafkaRoundRobin KafkaEncodeBalancerEnum = "roundRobin" // RoundRobin balancer
+	KafkaLeastBytes KafkaEncodeBalancerEnum = "leastBytes" // LeastBytes balancer
+	KafkaHash       KafkaEncodeBalancerEnum = "hash"       // Hash balancer
+	KafkaCrc32      KafkaEncodeBalancerEnum = "crc32"      // Crc32 balancer
+	KafkaMurmur2    KafkaEncodeBalancerEnum = "murmur2"    // Murmur2 balancer
+)