Split agent into agent and worker

Signed-off-by: Patryk Strusiewicz-Surmacki <[email protected]>

.golangci.yml

@@ -138,6 +138,9 @@ linters-settings:
         disabled: true
   funlen:
     lines: 65
+  gosec:
+    excludes:
+      - G115
 
 issues:
   exclude-rules:

Makefile

@@ -5,6 +5,8 @@ AGENT_IMG ?= ghcr.io/telekom/das-schiff-network-operator-agent:latest
 SIDECAR_IMG ?= ghcr.io/telekom/frr-exporter:latest
 # Operator image URL to use all building/pushing image targets
 OPERATOR_IMG ?= ghcr.io/telekom/das-schiff-network-opeator:latest
+# Worker image URL to use all building/pushing image targets
+WORKER_IMG ?= ghcr.io/telekom/worker:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.25
 
@@ -52,8 +54,13 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: generate
-generate: controller-gen bpf-generate ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate: controller-gen bpf-generate ## Generate code containing DeepCopy, DeepCopyInto and DeepCopyObject method implementations and GRPC code.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+	PATH=$(PATH):$(shell pwd)/bin $(PROTOC) --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative pkg/worker/pb/worker.proto
+
+.PHONY: generate-protobuff
+generate-protobuff: protoc ## Generate code containing DeepCopy, DeepCopyInto and DeepCopyObject method implementations.
+	PATH=$(PATH):$(shell pwd)/bin $(PROTOC) --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative pkg/worker/pb/worker.proto
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -83,10 +90,14 @@ operator-build: generate fmt vet ## Build agent binary.
 agent-build: generate fmt vet ## Build agent binary.
 	go build -o bin/agent cmd/agent/main.go
 
-.PHONY: sidecar-build
+.PHONY: sidecar-build ## Build sidecar (frr-exporter) binary.
 sidecar-build: build
 	go build -o bin/frr-exporter cmd/frr-exporter/main.go
 
+.PHONY: worker-build ## Build worker binary.
+agent-build: generate fmt vet
+	go build -o bin/worker cmd/worker/main.go
+
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/agent/main.go
@@ -96,6 +107,7 @@ docker-build: test ## Build docker image with the manager.
 	docker build -t ${OPERATOR_IMG} .
 	docker build -t ${AGENT_IMG} -f agent.Dockerfile .
 	docker build -t ${SIDECAR_IMG} -f frr-exporter.Dockerfile .
+	docker build -t ${WORKER_IMG} -f worker.Dockerfile .
 
 .PHONY: docker-build-agent
 docker-build-agent: test ## Build docker image with the manager.
@@ -109,6 +121,10 @@ docker-build-sidecar: test ## Build docker image with the manager.
 docker-build-operator: test ## Build docker image with the manager.
 	docker build -t ${OPERATOR_IMG} .
 
+.PHONY: docker-build-worker
+docker-build-worker: test ## Build docker image with the manager.
+	docker build -t ${WORKER_IMG} -f worker.Dockerfile .
+
 .PHONY: docker-push
 docker-push: docker-push-agent docker-push-sidecar docker-push-operator
 
@@ -124,6 +140,10 @@ docker-push-sidecar: ## Push docker image with the manager.
 docker-push-operator: ## Push docker image with the manager.
 	docker push ${OPERATOR_IMG}
 
+.PHONY: docker-push-worker
+docker-push-worker: ## Push docker image with the manager.
+	docker push ${WORKER_IMG}
+
 ##@ Release
 
 RELEASE_DIR ?= out
@@ -164,6 +184,7 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 	cd config/agent && $(KUSTOMIZE) edit set image agent=${AGENT_IMG}
 	cd config/agent && $(KUSTOMIZE) edit set image frr-exporter=${SIDECAR_IMG}
 	cd config/operator && $(KUSTOMIZE) edit set image operator=${OPERATOR_IMG}
+	cd config/worker && $(KUSTOMIZE) edit set image worker=${WORKER_IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 .PHONY: undeploy
@@ -203,3 +224,10 @@ GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
 endef
+
+PROTOC_DIR = $(shell pwd)/bin/protoc
+PROTOC = $(shell pwd)/bin/protoc/bin/protoc
+.PHONY: protoc
+protoc: ## Download controller-gen locally if necessary.
+	mkdir -p $(PROTOC_DIR) && cd $(PROTOC_DIR) && wget -nc https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip 2> /dev/null && unzip -nqq protoc-27.0-linux-x86_64.zip
+	$(call go-get-tool,$(PROTOC_DIR),google.golang.org/protobuf/cmd/[email protected])

cmd/agent/main.go

@@ -31,17 +31,19 @@ import (
 
 	networkv1alpha1 "github.com/telekom/das-schiff-network-operator/api/v1alpha1"
 	"github.com/telekom/das-schiff-network-operator/controllers"
+	adnetns "github.com/telekom/das-schiff-network-operator/pkg/adapters/netns"
+	advrfigbp "github.com/telekom/das-schiff-network-operator/pkg/adapters/vrf_igbp"
 	"github.com/telekom/das-schiff-network-operator/pkg/anycast"
 	"github.com/telekom/das-schiff-network-operator/pkg/bpf"
 	"github.com/telekom/das-schiff-network-operator/pkg/config"
-	"github.com/telekom/das-schiff-network-operator/pkg/frr"
 	"github.com/telekom/das-schiff-network-operator/pkg/healthcheck"
 	"github.com/telekom/das-schiff-network-operator/pkg/macvlan"
 	"github.com/telekom/das-schiff-network-operator/pkg/managerconfig"
 	"github.com/telekom/das-schiff-network-operator/pkg/monitoring"
 	"github.com/telekom/das-schiff-network-operator/pkg/nl"
 	"github.com/telekom/das-schiff-network-operator/pkg/notrack"
 	"github.com/telekom/das-schiff-network-operator/pkg/reconciler"
+	"github.com/telekom/das-schiff-network-operator/pkg/worker"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) //nolint:gci
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -89,29 +91,45 @@ func initCollectors() error {
 	return nil
 }
 
-func main() {
-	var onlyBPFMode bool
-	var configFile string
-	var interfacePrefix string
-	var nodeNetworkConfigPath string
-	flag.StringVar(&configFile, "config", "",
+type flags struct {
+	onlyBPFMode           bool
+	configFile            string
+	interfacePrefix       string
+	nodeNetworkConfigPath string
+	workerType            string
+	workerPort            int
+	workerAddr            string
+}
+
+func getFlags() *flags {
+	f := flags{}
+	flag.StringVar(&f.configFile, "config", "",
 		"The controller will load its initial configuration from this file. "+
 			"Omit this flag to use the default configuration values. "+
 			"Command-line flags override configuration from this file.")
-	flag.BoolVar(&onlyBPFMode, "only-attach-bpf", false,
+	flag.BoolVar(&f.onlyBPFMode, "only-attach-bpf", false,
 		"Only attach BPF to specified interfaces in config. This will not start any reconciliation. Perfect for masters.")
-	flag.StringVar(&interfacePrefix, "macvlan-interface-prefix", "",
+	flag.StringVar(&f.interfacePrefix, "macvlan-interface-prefix", "",
 		"Interface prefix for bridge devices for MACVlan sync")
-	flag.StringVar(&nodeNetworkConfigPath, "nodenetworkconfig-path", reconciler.DefaultNodeNetworkConfigPath,
+	flag.StringVar(&f.nodeNetworkConfigPath, "nodenetworkconfig-path", reconciler.DefaultNodeNetworkConfigPath,
 		"Path to store working node configuration.")
+	flag.StringVar(&f.workerType, "agent", "vrf-igbp", "Use selected worker type (default: vrf-igbp).")
+	flag.StringVar(&f.workerAddr, "agentAddr", "", "Worker's address (default: '').")
+	flag.IntVar(&f.workerPort, "agentPort", worker.DefaultPort, fmt.Sprintf("Worker's port (default: %d).", worker.DefaultPort))
+	flag.Parse()
+	return &f
+}
+
+func main() {
+	f := getFlags()
 	opts := zap.Options{
 		Development: true,
 	}
+
 	opts.BindFlags(flag.CommandLine)
-	flag.Parse()
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
-	options, err := setManagerOptions(configFile)
+	options, err := setManagerOptions(f.configFile)
 	if err != nil {
 		setupLog.Error(err, "unable to configure manager's options")
 		os.Exit(1)
@@ -137,14 +155,26 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err := initComponents(mgr, anycastTracker, cfg, clientConfig, onlyBPFMode, nodeNetworkConfigPath); err != nil {
+	var workerClient worker.Client
+	if f.workerType == "vrf-igbp" {
+		workerClient, err = advrfigbp.NewClient(fmt.Sprintf("%s:%d", f.workerAddr, f.workerPort))
+	} else {
+		workerClient, err = adnetns.NewClient()
+	}
+
+	if err != nil {
+		setupLog.Error(err, "failed to create adapter")
+		os.Exit(1)
+	}
+
+	if err := initComponents(mgr, anycastTracker, cfg, clientConfig, f.onlyBPFMode, f.nodeNetworkConfigPath, workerClient); err != nil {
 		setupLog.Error(err, "unable to initialize components")
 		os.Exit(1)
 	}
 
-	if interfacePrefix != "" {
+	if f.interfacePrefix != "" {
 		setupLog.Info("start macvlan sync")
-		macvlan.RunMACSync(interfacePrefix)
+		macvlan.RunMACSync(f.interfacePrefix)
 	}
 
 	setupLog.Info("starting manager")
@@ -176,10 +206,10 @@ func setManagerOptions(configFile string) (*manager.Options, error) {
 	return &options, nil
 }
 
-func initComponents(mgr manager.Manager, anycastTracker *anycast.Tracker, cfg *config.Config, clientConfig *rest.Config, onlyBPFMode bool, nodeConfigPath string) error {
+func initComponents(mgr manager.Manager, anycastTracker *anycast.Tracker, cfg *config.Config, clientConfig *rest.Config, onlyBPFMode bool, nodeConfigPath string, workerClient worker.Client) error {
 	// Start VRFRouteConfigurationReconciler when we are not running in only BPF mode.
 	if !onlyBPFMode {
-		if err := setupReconcilers(mgr, anycastTracker, nodeConfigPath); err != nil {
+		if err := setupReconcilers(mgr, nodeConfigPath, workerClient); err != nil {
 			return fmt.Errorf("unable to setup reconcilers: %w", err)
 		}
 	}
@@ -238,9 +268,21 @@ func initComponents(mgr manager.Manager, anycastTracker *anycast.Tracker, cfg *c
 	return nil
 }
 
-func setupReconcilers(mgr manager.Manager, anycastTracker *anycast.Tracker, nodeConfigPath string) error {
-	r, err := reconciler.NewNodeNetworkConfigReconciler(mgr.GetClient(), anycastTracker, mgr.GetLogger(),
-		nodeConfigPath, frr.NewFRRManager(), nl.NewManager(&nl.Toolkit{}))
+func setupReconcilers(mgr manager.Manager, nodeConfigPath string, workerClient worker.Client) error {
+	nc, err := healthcheck.LoadConfig(healthcheck.NetHealthcheckFile)
+	if err != nil {
+		return fmt.Errorf("error loading networking healthcheck config: %w", err)
+	}
+
+	tcpDialer := healthcheck.NewTCPDialer(nc.Timeout)
+	healthchecker, err := healthcheck.NewHealthChecker(mgr.GetClient(),
+		healthcheck.NewDefaultHealthcheckToolkit(nil, tcpDialer), nc)
+	if err != nil {
+		return fmt.Errorf("error creating networking healthchecker: %w", err)
+	}
+
+	r, err := reconciler.NewNodeNetworkConfigReconciler(mgr.GetClient(), mgr.GetLogger(),
+		nodeConfigPath, workerClient, healthchecker)
 	if err != nil {
 		return fmt.Errorf("unable to create debounced reconciler: %w", err)
 	}

cmd/worker/main.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/go-logr/zapr"
+	vrfigbpadapter "github.com/telekom/das-schiff-network-operator/pkg/adapters/vrf_igbp"
+	"github.com/telekom/das-schiff-network-operator/pkg/anycast"
+	"github.com/telekom/das-schiff-network-operator/pkg/frr"
+	"github.com/telekom/das-schiff-network-operator/pkg/nl"
+	"github.com/telekom/das-schiff-network-operator/pkg/worker"
+	workerpb "github.com/telekom/das-schiff-network-operator/pkg/worker/pb"
+	"go.uber.org/zap"
+	"google.golang.org/grpc"
+)
+
+func main() {
+	var agentType string
+	var port int
+	flag.StringVar(&agentType, "agent", "vrf-igbp", "Use selected agent type (default: vrf-igbp).")
+	flag.IntVar(&port, "port", worker.DefaultPort, fmt.Sprintf("gRPC listening port. (default: %d)", worker.DefaultPort))
+
+	zc := zap.NewProductionConfig()
+	zc.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	zc.DisableStacktrace = true
+	z, _ := zc.Build()
+	log := zapr.NewLogger(z)
+	log = log.WithName("agent")
+
+	log.Info("agent's port", "port", port)
+
+	anycastTracker := anycast.NewTracker(&nl.Toolkit{})
+
+	var err error
+	var adapter worker.Adapter
+	switch agentType {
+	case "vrf-igbp":
+		adapter, err = vrfigbpadapter.New(anycastTracker, log, frr.NewFRRManager(), nl.NewManager(&nl.Toolkit{}))
+	default:
+		log.Error(fmt.Errorf("agent is currently not supported"), "type", agentType)
+		os.Exit(1)
+	}
+
+	if err != nil {
+		log.Error(err, "error creating adapter")
+		os.Exit(1)
+	}
+
+	log.Info("created adapter", "type", agentType)
+
+	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		log.Error(err, "error on listening start")
+		os.Exit(1)
+	}
+
+	grpcServer := grpc.NewServer([]grpc.ServerOption{}...)
+	srv := worker.NewServer(adapter, &log)
+	workerpb.RegisterAgentServer(grpcServer, srv)
+
+	log.Info("created server, start listening...")
+
+	if err := grpcServer.Serve(lis); err != nil {
+		log.Error(err, "grpc server error")
+		os.Exit(1)
+	}
+}

config/agent/agent.yaml

@@ -74,10 +74,6 @@ spec:
         - mountPath: /opt/network-operator/config.yaml
           name: network-config
           subPath: config.yaml
-        - mountPath: /etc/frr
-          name: frr-config
-        - mountPath: /var/run/dbus/system_bus_socket
-          name: dbus-socket
       - command:
         - /frr-exporter
         env:
@@ -119,10 +115,6 @@ spec:
         hostPath:
           path: /var/run/frr
           type: Directory
-      - name: dbus-socket
-        hostPath:
-          path: /var/run/dbus/system_bus_socket
-          type: Socket
       - configMap:
           name: network-operator-config
         name: network-config

config/default/kustomization.yaml

@@ -44,6 +44,7 @@ resources:
 - ../rbac
 - ../agent
 - ../operator
+- ../worker
 - ../webhook
 - ../prometheus
 labels:

config/worker/kustomization.yaml

@@ -0,0 +1,12 @@
+resources:
+- worker.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: worker
+  newName: ghcr.io/telekom/worker
+  newTag: latest