Tie up the API with the FRR configuration mechanism

Dockerfile

@@ -15,16 +15,20 @@ RUN go mod download
 COPY cmd/main.go cmd/main.go
 COPY api/ api/
 COPY internal/ internal/
+COPY frr-tools/metrics ./frr-tools/metrics/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command
 # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o frr-k8s cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /build/frr-metrics frr-tools/metrics/exporter.go
 
 FROM alpine:latest
 WORKDIR /
 COPY --from=builder /workspace/frr-k8s .
+COPY --from=builder /build/frr-metrics /frr-metrics
+COPY frr-tools/reloader/frr-reloader.sh /frr-reloader.sh
 
 ENTRYPOINT ["/frr-k8s"]

Makefile

@@ -137,6 +137,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 .PHONY: deploy
 deploy: kubectl manifests kustomize kind load-on-kind ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/frr-k8s && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUBECTL) -n frr-k8s-system delete ds frr-k8s-daemon || true
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 	$(KUBECTL) -n frr-k8s-system wait --for=condition=Ready --all pods --timeout 300s
 

cmd/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -33,6 +34,8 @@ import (
 
 	frrk8sv1beta1 "github.com/metallb/frrk8s/api/v1beta1"
 	"github.com/metallb/frrk8s/internal/controller"
+	"github.com/metallb/frrk8s/internal/frr"
+	"github.com/metallb/frrk8s/internal/logging"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -52,19 +55,27 @@ func main() {
 	var (
 		metricsAddr string
 		probeAddr   string
+		logLevel    string
+		nodeName    string // TODO not using this now, but we'll need it when we implement the node selector
 	)
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
+	flag.StringVar(&logLevel, "log-level", "info", fmt.Sprintf("log level. must be one of: [%s]", logging.Levels.String()))
+	flag.StringVar(&nodeName, "node-name", "", "The node this daemon is running on.")
 
 	opts := zap.Options{
 		Development: true,
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
-	logger := zap.New(zap.UseFlagOptions(&opts))
-	ctrl.SetLogger(logger)
+	logger, err := logging.Init(logLevel)
+	if err != nil {
+		fmt.Printf("failed to initialize logging: %s\n", err)
+		os.Exit(1)
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
 		MetricsBindAddress:     metricsAddr,
@@ -78,8 +89,10 @@ func main() {
 
 	ctx := ctrl.SetupSignalHandler()
 	if err = (&controller.FRRConfigurationReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:     mgr.GetClient(),
+		Scheme:     mgr.GetScheme(),
+		FRRHandler: frr.NewFRR(ctx, logger, logging.Level(logLevel)),
+		Logger:     logger,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "FRRConfiguration")
 		os.Exit(1)

config/default/frr-k8s_auth_proxy_patch.yaml

@@ -52,3 +52,4 @@ spec:
         args:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
+        - "--node-name=$(NODE_NAME)"

config/frr-k8s/frr-k8s.yaml

@@ -41,6 +41,7 @@ spec:
       containers:
       - command:
         - /frr-k8s
+        args: ["--node-name", "$(NODE_NAME)"]
         image: controller:latest
         imagePullPolicy: IfNotPresent
         name: frr-k8s
@@ -62,19 +63,137 @@ spec:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
+        # TODO(user): Configure the resources accordingly based on the project requirements.
+        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
             cpu: 500m
             memory: 128Mi
           requests:
             cpu: 10m
             memory: 64Mi
+        volumeMounts:
+        - name: reloader
+          mountPath: /etc/frr_reloader
+        env:
+        - name: FRR_CONFIG_FILE
+          value: /etc/frr_reloader/frr.conf
+        - name: FRR_RELOADER_PID_FILE
+          value: /etc/frr_reloader/reloader.pid
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      - name: frr
+        securityContext:
+          capabilities:
+            add: ["NET_ADMIN", "NET_RAW", "SYS_ADMIN", "NET_BIND_SERVICE"]
+        image: quay.io/frrouting/frr:8.4.2
+        env:
+          - name: TINI_SUBREAPER
+            value: "true"
+        volumeMounts:
+          - name: frr-sockets
+            mountPath: /var/run/frr
+          - name: frr-conf
+            mountPath: /etc/frr
+        # The command is FRR's default entrypoint & waiting for the log file to appear and tailing it.
+        # If the log file isn't created in 60 seconds the tail fails and the container is restarted.
+        # This workaround is needed to have the frr logs as part of kubectl logs -c frr < k8s-frr-podname >.
+        command:
+          - /bin/sh
+          - -c
+          - |
+            /sbin/tini -- /usr/lib/frr/docker-start &
+            attempts=0
+            until [[ -f /etc/frr/frr.log || $attempts -eq 60 ]]; do
+              sleep 1
+              attempts=$(( $attempts + 1 ))
+            done
+            tail -f /etc/frr/frr.log
+        livenessProbe:
+          httpGet:
+            path: /livez
+            port: 7473
+          periodSeconds: 5
+          failureThreshold: 3
+        startupProbe:
+          httpGet:
+            path: /livez
+            port: 7473
+          failureThreshold: 30
+          periodSeconds: 5
+      - name: frr-metrics
+        image: quay.io/frrouting/frr:8.4.2
+        command: ["/etc/frr_metrics/frr-metrics"]
+        args:
+          - --metrics-port=7473
+        ports:
+          - containerPort: 7473
+            name: monitoring
+        volumeMounts:
+          - name: frr-sockets
+            mountPath: /var/run/frr
+          - name: frr-conf
+            mountPath: /etc/frr
+          - name: metrics
+            mountPath: /etc/frr_metrics
+      - name: reloader
+        image: quay.io/frrouting/frr:8.4.2
+        command: ["/etc/frr_reloader/frr-reloader.sh"]
+        volumeMounts:
+          - name: frr-sockets
+            mountPath: /var/run/frr
+          - name: frr-conf
+            mountPath: /etc/frr
+          - name: reloader
+            mountPath: /etc/frr_reloader
       tolerations:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
         operator: Exists
       - effect: NoSchedule
         key: node-role.kubernetes.io/control-plane
         operator: Exists
+      volumes:
+        - name: frr-sockets
+          emptyDir: {}
+        - name: frr-startup
+          configMap:
+            name: frr-startup
+        - name: frr-conf
+          emptyDir: {}
+        - name: reloader
+          emptyDir: {}
+        - name: metrics
+          emptyDir: {}
+      initContainers:
+        # Copies the initial config files with the right permissions to the shared volume.
+        - name: cp-frr-files
+          securityContext:
+            runAsUser: 100
+            runAsGroup: 101
+          image: quay.io/frrouting/frr:8.4.2
+          command: ["/bin/sh", "-c", "cp -rLf /tmp/frr/* /etc/frr/"]
+          volumeMounts:
+            - name: frr-startup
+              mountPath: /tmp/frr
+            - name: frr-conf
+              mountPath: /etc/frr
+        # Copies the reloader to the shared volume between the k8s-frr controller and reloader.
+        - name: cp-reloader
+          image: controller:latest
+          command: ["/bin/sh", "-c", "cp -f /frr-reloader.sh /etc/frr_reloader/"]
+          volumeMounts:
+            - name: reloader
+              mountPath: /etc/frr_reloader
+        - name: cp-metrics
+          image: controller:latest
+          command: ["/bin/sh", "-c", "cp -f /frr-metrics /etc/frr_metrics/"]
+          volumeMounts:
+            - name: metrics
+              mountPath: /etc/frr_metrics
       serviceAccountName: daemon
       terminationGracePeriodSeconds: 10
+      shareProcessNamespace: true
+      hostNetwork: true