diff --git a/Dockerfile b/Dockerfile index 74f94916..3ec5ebf8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN go mod download COPY cmd/main.go cmd/main.go COPY api/ api/ COPY internal/ internal/ +COPY frr-tools/metrics ./frr-tools/metrics/ # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command @@ -22,9 +23,12 @@ COPY internal/ internal/ # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o frr-k8s cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /build/frr-metrics frr-tools/metrics/exporter.go FROM alpine:latest WORKDIR / COPY --from=builder /workspace/frr-k8s . +COPY --from=builder /build/frr-metrics /frr-metrics +COPY frr-tools/reloader/frr-reloader.sh /frr-reloader.sh ENTRYPOINT ["/frr-k8s"] diff --git a/Makefile b/Makefile index 9b90adbe..e4d44697 100644 --- a/Makefile +++ b/Makefile @@ -137,6 +137,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: kubectl manifests kustomize kind load-on-kind ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/frr-k8s && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUBECTL) -n frr-k8s-system delete ds frr-k8s-daemon || true $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - $(KUBECTL) -n frr-k8s-system wait --for=condition=Ready --all pods --timeout 300s diff --git a/cmd/main.go b/cmd/main.go index 67aaf59e..e097f8b3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -18,6 +18,7 @@ package main import ( "flag" + "fmt" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -33,6 +34,8 @@ import ( frrk8sv1beta1 "github.com/metallb/frrk8s/api/v1beta1" "github.com/metallb/frrk8s/internal/controller" + "github.com/metallb/frrk8s/internal/frr" + "github.com/metallb/frrk8s/internal/logging" //+kubebuilder:scaffold:imports ) @@ -52,10 +55,14 @@ func main() { var ( metricsAddr string probeAddr string + logLevel string + nodeName string // TODO not using this now, but we'll need it when we implement the node selector ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.StringVar(&logLevel, "log-level", "info", fmt.Sprintf("log level. must be one of: [%s]", logging.Levels.String())) + flag.StringVar(&nodeName, "node-name", "", "The node this daemon is running on.") opts := zap.Options{ Development: true, @@ -63,8 +70,12 @@ func main() { opts.BindFlags(flag.CommandLine) flag.Parse() - logger := zap.New(zap.UseFlagOptions(&opts)) - ctrl.SetLogger(logger) + logger, err := logging.Init(logLevel) + if err != nil { + fmt.Printf("failed to initialize logging: %s\n", err) + os.Exit(1) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, @@ -78,8 +89,10 @@ func main() { ctx := ctrl.SetupSignalHandler() if err = (&controller.FRRConfigurationReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + FRRHandler: frr.NewFRR(ctx, logger, logging.Level(logLevel)), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "FRRConfiguration") os.Exit(1) diff --git a/config/default/frr-k8s_auth_proxy_patch.yaml b/config/default/frr-k8s_auth_proxy_patch.yaml index 99cd445d..449ff5ae 100644 --- a/config/default/frr-k8s_auth_proxy_patch.yaml +++ b/config/default/frr-k8s_auth_proxy_patch.yaml @@ -52,3 +52,4 @@ spec: args: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" + - "--node-name=$(NODE_NAME)" diff --git a/config/frr-k8s/frr-k8s.yaml b/config/frr-k8s/frr-k8s.yaml index 0fce6847..a9682cf0 100644 --- a/config/frr-k8s/frr-k8s.yaml +++ b/config/frr-k8s/frr-k8s.yaml @@ -41,6 +41,7 @@ spec: containers: - command: - /frr-k8s + args: ["--node-name", "$(NODE_NAME)"] image: controller:latest imagePullPolicy: IfNotPresent name: frr-k8s @@ -62,6 +63,8 @@ spec: port: 8081 initialDelaySeconds: 5 periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: cpu: 500m @@ -69,6 +72,82 @@ spec: requests: cpu: 10m memory: 64Mi + volumeMounts: + - name: reloader + mountPath: /etc/frr_reloader + env: + - name: FRR_CONFIG_FILE + value: /etc/frr_reloader/frr.conf + - name: FRR_RELOADER_PID_FILE + value: /etc/frr_reloader/reloader.pid + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: frr + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW", "SYS_ADMIN", "NET_BIND_SERVICE"] + image: quay.io/frrouting/frr:8.4.2 + env: + - name: TINI_SUBREAPER + value: "true" + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + # The command is FRR's default entrypoint & waiting for the log file to appear and tailing it. + # If the log file isn't created in 60 seconds the tail fails and the container is restarted. + # This workaround is needed to have the frr logs as part of kubectl logs -c frr < k8s-frr-podname >. + command: + - /bin/sh + - -c + - | + /sbin/tini -- /usr/lib/frr/docker-start & + attempts=0 + until [[ -f /etc/frr/frr.log || $attempts -eq 60 ]]; do + sleep 1 + attempts=$(( $attempts + 1 )) + done + tail -f /etc/frr/frr.log + livenessProbe: + httpGet: + path: /livez + port: 7473 + periodSeconds: 5 + failureThreshold: 3 + startupProbe: + httpGet: + path: /livez + port: 7473 + failureThreshold: 30 + periodSeconds: 5 + - name: frr-metrics + image: quay.io/frrouting/frr:8.4.2 + command: ["/etc/frr_metrics/frr-metrics"] + args: + - --metrics-port=7473 + ports: + - containerPort: 7473 + name: monitoring + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: metrics + mountPath: /etc/frr_metrics + - name: reloader + image: quay.io/frrouting/frr:8.4.2 + command: ["/etc/frr_reloader/frr-reloader.sh"] + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: reloader + mountPath: /etc/frr_reloader tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master @@ -76,5 +155,45 @@ spec: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists + volumes: + - name: frr-sockets + emptyDir: {} + - name: frr-startup + configMap: + name: frr-startup + - name: frr-conf + emptyDir: {} + - name: reloader + emptyDir: {} + - name: metrics + emptyDir: {} + initContainers: + # Copies the initial config files with the right permissions to the shared volume. + - name: cp-frr-files + securityContext: + runAsUser: 100 + runAsGroup: 101 + image: quay.io/frrouting/frr:8.4.2 + command: ["/bin/sh", "-c", "cp -rLf /tmp/frr/* /etc/frr/"] + volumeMounts: + - name: frr-startup + mountPath: /tmp/frr + - name: frr-conf + mountPath: /etc/frr + # Copies the reloader to the shared volume between the k8s-frr controller and reloader. + - name: cp-reloader + image: controller:latest + command: ["/bin/sh", "-c", "cp -f /frr-reloader.sh /etc/frr_reloader/"] + volumeMounts: + - name: reloader + mountPath: /etc/frr_reloader + - name: cp-metrics + image: controller:latest + command: ["/bin/sh", "-c", "cp -f /frr-metrics /etc/frr_metrics/"] + volumeMounts: + - name: metrics + mountPath: /etc/frr_metrics serviceAccountName: daemon terminationGracePeriodSeconds: 10 + shareProcessNamespace: true + hostNetwork: true diff --git a/frr-tools/metrics/collector/bfd.go b/frr-tools/metrics/collector/bfd.go new file mode 100644 index 00000000..31a4ca7f --- /dev/null +++ b/frr-tools/metrics/collector/bfd.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier:Apache-2.0 + +package collector + +import ( + "encoding/json" + "fmt" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/metallb/frrk8s/frr-tools/metrics/vtysh" + "github.com/metallb/frrk8s/internal/frr" + "github.com/prometheus/client_golang/prometheus" +) + +type bfdPeerCounters struct { + Peer string `json:"peer"` + ControlPacketInput int `json:"control-packet-input"` + ControlPacketOutput int `json:"control-packet-output"` + EchoPacketInput int `json:"echo-packet-input"` + EchoPacketOutput int `json:"echo-packet-output"` + SessionUpEvents int `json:"session-up"` + SessionDownEvents int `json:"session-down"` + ZebraNotifications int `json:"zebra-notifications"` +} + +const subsystem = "bfd" + +var ( + bfdSessionUpDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, SessionUp.Name), + "BFD session state (1 is up, 0 is down)", + labels, + nil, + ) + + controlPacketInputDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "control_packet_input"), + "Number of received BFD control packets", + labels, + nil, + ) + + controlPacketOutputDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "control_packet_output"), + "Number of sent BFD control packets", + labels, + nil, + ) + + echoPacketInputDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "echo_packet_input"), + "Number of received BFD echo packets", + labels, + nil, + ) + + echoPacketOutputDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "echo_packet_output"), + "Number of sent BFD echo packets", + labels, + nil, + ) + + sessionUpEventsDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "session_up_events"), + "Number of BFD session up events", + labels, + nil, + ) + + sessionDownEventsDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "session_down_events"), + "Number of BFD session down events", + labels, + nil, + ) + + zebraNotificationsDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, subsystem, "zebra_notifications"), + "Number of BFD zebra notifications", + labels, + nil, + ) +) + +type bfd struct { + Log log.Logger + frrCli vtysh.Cli +} + +func NewBFD(l log.Logger) prometheus.Collector { + log := log.With(l, "collector", subsystem) + return &bfd{Log: log, frrCli: vtysh.Run} +} + +func mockNewBFD(l log.Logger) *bfd { + log := log.With(l, "collector", subsystem) + return &bfd{Log: log, frrCli: vtysh.Run} +} + +func (c *bfd) Describe(ch chan<- *prometheus.Desc) { + ch <- bfdSessionUpDesc + ch <- controlPacketInputDesc + ch <- controlPacketOutputDesc + ch <- echoPacketInputDesc + ch <- echoPacketOutputDesc + ch <- sessionUpEventsDesc + ch <- sessionDownEventsDesc + ch <- zebraNotificationsDesc +} + +func (c *bfd) Collect(ch chan<- prometheus.Metric) { + peers, err := getBFDPeers(c.frrCli) + if err != nil { + level.Error(c.Log).Log("error", err, "msg", "failed to fetch BFD peers from FRR") + return + } + + updatePeersMetrics(ch, peers) + + peersCounters, err := getBFDPeersCounters(c.frrCli) + if err != nil { + level.Error(c.Log).Log("error", err, "msg", "failed to fetch BFD peers counters from FRR") + return + } + + updatePeersCountersMetrics(ch, peersCounters) +} + +func updatePeersMetrics(ch chan<- prometheus.Metric, peersPerVRF map[string][]frr.BFDPeer) { + for vrf, peers := range peersPerVRF { + for _, p := range peers { + sessionUp := 1 + if p.Status != "up" { + sessionUp = 0 + } + + ch <- prometheus.MustNewConstMetric(bfdSessionUpDesc, prometheus.GaugeValue, float64(sessionUp), p.Peer, vrf) + } + } +} + +func updatePeersCountersMetrics(ch chan<- prometheus.Metric, peersCountersPerVRF map[string][]bfdPeerCounters) { + for vrf, peersCounters := range peersCountersPerVRF { + for _, p := range peersCounters { + ch <- prometheus.MustNewConstMetric(controlPacketInputDesc, prometheus.CounterValue, float64(p.ControlPacketInput), p.Peer, vrf) + ch <- prometheus.MustNewConstMetric(controlPacketOutputDesc, prometheus.CounterValue, float64(p.ControlPacketOutput), p.Peer, vrf) + ch <- prometheus.MustNewConstMetric(echoPacketInputDesc, prometheus.CounterValue, float64(p.EchoPacketInput), p.Peer, vrf) + ch <- prometheus.MustNewConstMetric(echoPacketOutputDesc, prometheus.CounterValue, float64(p.EchoPacketOutput), p.Peer, vrf) + ch <- prometheus.MustNewConstMetric(sessionUpEventsDesc, prometheus.CounterValue, float64(p.SessionUpEvents), p.Peer, vrf) + ch <- prometheus.MustNewConstMetric(sessionDownEventsDesc, prometheus.CounterValue, float64(p.SessionDownEvents), p.Peer, vrf) + ch <- prometheus.MustNewConstMetric(zebraNotificationsDesc, prometheus.CounterValue, float64(p.ZebraNotifications), p.Peer, vrf) + } + } +} + +func getBFDPeers(frrCli vtysh.Cli) (map[string][]frr.BFDPeer, error) { + vrfs, err := vtysh.VRFs(frrCli) + if err != nil { + return nil, err + } + res := make(map[string][]frr.BFDPeer) + for _, vrf := range vrfs { + peersJSON, err := frrCli(fmt.Sprintf("show bfd vrf %s peers json", vrf)) + if err != nil { + return nil, err + } + peers, err := frr.ParseBFDPeers(peersJSON) + if err != nil { + return nil, err + } + res[vrf] = peers + } + return res, nil +} + +func getBFDPeersCounters(frrCli vtysh.Cli) (map[string][]bfdPeerCounters, error) { + vrfs, err := vtysh.VRFs(frrCli) + if err != nil { + return nil, err + } + + res := make(map[string][]bfdPeerCounters) + for _, vrf := range vrfs { + countersJSON, err := frrCli(fmt.Sprintf("show bfd vrf %s peers counters json", vrf)) + if err != nil { + return nil, err + } + + parseRes := []bfdPeerCounters{} + err = json.Unmarshal([]byte(countersJSON), &parseRes) + if err != nil { + return nil, err + } + res[vrf] = parseRes + } + return res, nil +} diff --git a/frr-tools/metrics/collector/bfd_test.go b/frr-tools/metrics/collector/bfd_test.go new file mode 100644 index 00000000..545f60ed --- /dev/null +++ b/frr-tools/metrics/collector/bfd_test.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier:Apache-2.0 + +package collector + +import ( + "bytes" + "testing" + "text/template" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +var ( + bfdMetricsTmpl = ` + # HELP frrk8s_bfd_session_up BFD session state (1 is up, 0 is down) + # TYPE frrk8s_bfd_session_up gauge + frrk8s_bfd_session_up{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .SessionUp }} + # HELP frrk8s_bfd_control_packet_input Number of received BFD control packets + # TYPE frrk8s_bfd_control_packet_input counter + frrk8s_bfd_control_packet_input{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .ControlPacketInput }} + # HELP frrk8s_bfd_control_packet_output Number of sent BFD control packets + # TYPE frrk8s_bfd_control_packet_output counter + frrk8s_bfd_control_packet_output{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .ControlPacketOutput }} + # HELP frrk8s_bfd_echo_packet_input Number of received BFD echo packets + # TYPE frrk8s_bfd_echo_packet_input counter + frrk8s_bfd_echo_packet_input{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .EchoPacketInput }} + # HELP frrk8s_bfd_echo_packet_output Number of sent BFD echo packets + # TYPE frrk8s_bfd_echo_packet_output counter + frrk8s_bfd_echo_packet_output{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .EchoPacketOutput }} + # HELP frrk8s_bfd_session_down_events Number of BFD session down events + # TYPE frrk8s_bfd_session_down_events counter + frrk8s_bfd_session_down_events{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .SessionDownEvents }} + # HELP frrk8s_bfd_session_up_events Number of BFD session up events + # TYPE frrk8s_bfd_session_up_events counter + frrk8s_bfd_session_up_events{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .SessionUpEvents }} + # HELP frrk8s_bfd_zebra_notifications Number of BFD zebra notifications + # TYPE frrk8s_bfd_zebra_notifications counter + frrk8s_bfd_zebra_notifications{peer="{{ .Peer }}", vrf="{{ .NeighborVRF }}"} {{ .ZebraNotifications }} + ` + + bfdTests = []struct { + desc string + vtyshPeersOutput string + vtyshPeersCountersOutput string + peer string + vrf string + sessionUp int + controlPacketInput int + controlPacketOutput int + echoPacketInput int + echoPacketOutput int + sessionUpEvents int + sessionDownEvents int + zebraNotifications int + }{ + { + desc: "Output contains IPv4", + vtyshPeersOutput: peersIPv4, + vtyshPeersCountersOutput: peersCountersIPv4, + peer: "172.18.0.4", + vrf: "default", + sessionUp: 1, + controlPacketInput: 5, + controlPacketOutput: 5, + echoPacketInput: 0, + echoPacketOutput: 0, + sessionUpEvents: 1, + sessionDownEvents: 0, + zebraNotifications: 4, + }, + { + desc: "Output contains IPv6", + vtyshPeersOutput: peersIPv6, + vtyshPeersCountersOutput: peersCountersIPv6, + peer: "fc00:f853:ccd:e793::4", + vrf: "default", + sessionUp: 0, + controlPacketInput: 10, + controlPacketOutput: 10, + echoPacketInput: 0, + echoPacketOutput: 0, + sessionUpEvents: 1, + sessionDownEvents: 0, + zebraNotifications: 4, + }, + } + peersIPv4 = ` + [ + { + "multihop":false, + "peer":"172.18.0.4", + "vrf":"default", + "interface":"eth0", + "id":2508913041, + "remote-id":3444899611, + "passive-mode":false, + "status":"up", + "uptime":13, + "diagnostic":"ok", + "remote-diagnostic":"ok", + "receive-interval":300, + "transmit-interval":300, + "echo-interval":0, + "detect-multiplier":3, + "remote-receive-interval":300, + "remote-transmit-interval":300, + "remote-echo-interval":50, + "remote-detect-multiplier":3 + } + ] + ` + peersIPv6 = ` + [ + { + "multihop":false, + "peer":"fc00:f853:ccd:e793::4", + "local":"fc00:f853:ccd:e793::6", + "vrf":"default", + "interface":"eth0", + "id":1975516641, + "remote-id":505304921, + "passive-mode":false, + "status":"down", + "uptime":33, + "diagnostic":"ok", + "remote-diagnostic":"ok", + "receive-interval":300, + "transmit-interval":300, + "echo-interval":0, + "detect-multiplier":3, + "remote-receive-interval":300, + "remote-transmit-interval":300, + "remote-echo-interval":50, + "remote-detect-multiplier":3 + } + ] + ` + peersCountersIPv4 = ` + [ + { + "multihop":false, + "peer":"172.18.0.4", + "vrf":"default", + "interface":"eth0", + "control-packet-input":5, + "control-packet-output":5, + "echo-packet-input":0, + "echo-packet-output":0, + "session-up":1, + "session-down":0, + "zebra-notifications":4 + } + ] + ` + peersCountersIPv6 = ` + [ + { + "multihop":false, + "peer":"fc00:f853:ccd:e793::4", + "local":"fc00:f853:ccd:e793::6", + "vrf":"default", + "interface":"eth0", + "control-packet-input":10, + "control-packet-output":10, + "echo-packet-input":0, + "echo-packet-output":0, + "session-up":1, + "session-down":0, + "zebra-notifications":4 + } + ] + ` +) + +func TestBFDCollect(t *testing.T) { + for _, test := range bfdTests { + t.Run(test.desc, func(t *testing.T) { + tmpl, err := template.New(test.desc).Parse(bfdMetricsTmpl) + if err != nil { + t.Errorf("expected no error but got %s", err) + } + + var w bytes.Buffer + err = tmpl.Execute(&w, map[string]interface{}{ + "Peer": test.peer, + "SessionUp": test.sessionUp, + "ControlPacketInput": test.controlPacketInput, + "ControlPacketOutput": test.controlPacketOutput, + "EchoPacketInput": test.echoPacketInput, + "EchoPacketOutput": test.echoPacketOutput, + "SessionUpEvents": test.sessionUpEvents, + "SessionDownEvents": test.sessionDownEvents, + "ZebraNotifications": test.zebraNotifications, + "NeighborVRF": "default", + }) + + if err != nil { + t.Errorf("expected no error but got %s", err) + } + + l := log.NewNopLogger() + collector := mockNewBFD(l) + cmdOutput := map[string]string{ + "show bgp vrf all json": vrfVtysh, + "show bfd vrf default peers json": test.vtyshPeersOutput, + "show bfd vrf red peers json": "[]", + "show bfd vrf default peers counters json": test.vtyshPeersCountersOutput, + "show bfd vrf red peers counters json": "[]", + } + collector.frrCli = func(args string) (string, error) { + res, ok := cmdOutput[args] + if !ok { + return "{}", nil + } + return res, nil + } + buf := bytes.NewReader(w.Bytes()) + err = testutil.CollectAndCompare(collector, buf) + if err != nil { + t.Errorf("expected no error but got %s", err) + } + }) + } +} diff --git a/frr-tools/metrics/collector/bgp.go b/frr-tools/metrics/collector/bgp.go new file mode 100644 index 00000000..79dc0015 --- /dev/null +++ b/frr-tools/metrics/collector/bgp.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier:Apache-2.0 + +package collector + +import ( + "fmt" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/metallb/frrk8s/internal/frr" + "github.com/prometheus/client_golang/prometheus" + + "github.com/metallb/frrk8s/frr-tools/metrics/vtysh" +) + +var labels = []string{"peer", "vrf"} + +var ( + sessionUpDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, SessionUp.Name), + SessionUp.Help, + labels, + nil, + ) + + prefixesDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, Prefixes.Name), + Prefixes.Help, + labels, + nil, + ) + + opensSentDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "opens_sent"), + "Number of BGP open messages sent", + labels, + nil, + ) + + opensReceivedDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "opens_received"), + "Number of BGP open messages received", + labels, + nil, + ) + + notificationsSentDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "notifications_sent"), + "Number of BGP notification messages sent", + labels, + nil, + ) + + updatesSentDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, UpdatesSent.Name), + UpdatesSent.Help, + labels, + nil, + ) + + updatesReceivedDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "updates_total_received"), + "Number of BGP UPDATE messages received", + labels, + nil, + ) + + keepalivesSentDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "keepalives_sent"), + "Number of BGP keepalive messages sent", + labels, + nil, + ) + + keepalivesReceivedDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "keepalives_received"), + "Number of BGP keepalive messages received", + labels, + nil, + ) + + routeRefreshSentedDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "route_refresh_sent"), + "Number of BGP route refresh messages sent", + labels, + nil, + ) + + totalSentDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "total_sent"), + "Number of total BGP messages sent", + labels, + nil, + ) + + totalReceivedDesc = prometheus.NewDesc( + prometheus.BuildFQName(Namespace, Subsystem, "total_received"), + "Number of total BGP messages received", + labels, + nil, + ) +) + +type bgp struct { + Log log.Logger + frrCli vtysh.Cli +} + +func NewBGP(l log.Logger) prometheus.Collector { + log := log.With(l, "collector", Subsystem) + return &bgp{Log: log, frrCli: vtysh.Run} +} + +func mocknewBGP(l log.Logger) *bgp { + log := log.With(l, "collector", Subsystem) + return &bgp{Log: log, frrCli: vtysh.Run} +} + +func (c *bgp) Describe(ch chan<- *prometheus.Desc) { + ch <- sessionUpDesc + ch <- prefixesDesc + ch <- opensSentDesc + ch <- opensReceivedDesc + ch <- notificationsSentDesc + ch <- updatesSentDesc + ch <- updatesReceivedDesc + ch <- keepalivesSentDesc + ch <- keepalivesReceivedDesc + ch <- routeRefreshSentedDesc + ch <- totalSentDesc + ch <- totalReceivedDesc +} + +func (c *bgp) Collect(ch chan<- prometheus.Metric) { + neighbors, err := getBGPNeighbors(c.frrCli) + if err != nil { + level.Error(c.Log).Log("error", err, "msg", "failed to fetch BGP neighbors from FRR") + return + } + + updateNeighborsMetrics(ch, neighbors) +} + +func updateNeighborsMetrics(ch chan<- prometheus.Metric, neighbors map[string][]*frr.Neighbor) { + for vrf, nn := range neighbors { + for _, n := range nn { + sessionUp := 1 + if !n.Connected { + sessionUp = 0 + } + peerLabel := fmt.Sprintf("%s:%d", n.IP.String(), n.Port) + + ch <- prometheus.MustNewConstMetric(sessionUpDesc, prometheus.GaugeValue, float64(sessionUp), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(prefixesDesc, prometheus.GaugeValue, float64(n.PrefixSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(opensSentDesc, prometheus.CounterValue, float64(n.MsgStats.OpensSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(opensReceivedDesc, prometheus.CounterValue, float64(n.MsgStats.OpensReceived), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(notificationsSentDesc, prometheus.CounterValue, float64(n.MsgStats.NotificationsSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(updatesSentDesc, prometheus.CounterValue, float64(n.MsgStats.UpdatesSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(updatesReceivedDesc, prometheus.CounterValue, float64(n.MsgStats.UpdatesReceived), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(keepalivesSentDesc, prometheus.CounterValue, float64(n.MsgStats.KeepalivesSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(keepalivesReceivedDesc, prometheus.CounterValue, float64(n.MsgStats.KeepalivesReceived), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(routeRefreshSentedDesc, prometheus.CounterValue, float64(n.MsgStats.RouteRefreshSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(totalSentDesc, prometheus.CounterValue, float64(n.MsgStats.TotalSent), peerLabel, vrf) + ch <- prometheus.MustNewConstMetric(totalReceivedDesc, prometheus.CounterValue, float64(n.MsgStats.TotalReceived), peerLabel, vrf) + } + } +} + +func getBGPNeighbors(frrCli vtysh.Cli) (map[string][]*frr.Neighbor, error) { + vrfs, err := vtysh.VRFs(frrCli) + if err != nil { + return nil, err + } + neighbors := make(map[string][]*frr.Neighbor, 0) + for _, vrf := range vrfs { + res, err := frrCli(fmt.Sprintf("show bgp vrf %s neighbors json", vrf)) + if err != nil { + return nil, err + } + + neighborsPerVRF, err := frr.ParseNeighbours(res) + if err != nil { + return nil, err + } + neighbors[vrf] = neighborsPerVRF + } + return neighbors, nil +} diff --git a/frr-tools/metrics/collector/bgp_test.go b/frr-tools/metrics/collector/bgp_test.go new file mode 100644 index 00000000..d0566d29 --- /dev/null +++ b/frr-tools/metrics/collector/bgp_test.go @@ -0,0 +1,488 @@ +// SPDX-License-Identifier:Apache-2.0 + +package collector + +import ( + "bytes" + "testing" + "text/template" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +var ( + metricsTmpl = ` + # HELP frrk8s_bgp_announced_prefixes_total Number of prefixes currently being advertised on the BGP session + # TYPE frrk8s_bgp_announced_prefixes_total gauge + frrk8s_bgp_announced_prefixes_total{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .AnnouncedPrefixes }} + # HELP frrk8s_bgp_keepalives_received Number of BGP keepalive messages received + # TYPE frrk8s_bgp_keepalives_received counter + frrk8s_bgp_keepalives_received{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .KeepalivesReceived }} + # HELP frrk8s_bgp_keepalives_sent Number of BGP keepalive messages sent + # TYPE frrk8s_bgp_keepalives_sent counter + frrk8s_bgp_keepalives_sent{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .KeepalivesSent }} + # HELP frrk8s_bgp_notifications_sent Number of BGP notification messages sent + # TYPE frrk8s_bgp_notifications_sent counter + frrk8s_bgp_notifications_sent{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .NotificationsSent }} + # HELP frrk8s_bgp_opens_received Number of BGP open messages received + # TYPE frrk8s_bgp_opens_received counter + frrk8s_bgp_opens_received{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .OpensReceived }} + # HELP frrk8s_bgp_opens_sent Number of BGP open messages sent + # TYPE frrk8s_bgp_opens_sent counter + frrk8s_bgp_opens_sent{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .OpensSent }} + # HELP frrk8s_bgp_route_refresh_sent Number of BGP route refresh messages sent + # TYPE frrk8s_bgp_route_refresh_sent counter + frrk8s_bgp_route_refresh_sent{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .RouteRefreshSent }} + # HELP frrk8s_bgp_session_up BGP session state (1 is up, 0 is down) + # TYPE frrk8s_bgp_session_up gauge + frrk8s_bgp_session_up{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .SessionUp }} + # HELP frrk8s_bgp_total_received Number of total BGP messages received + # TYPE frrk8s_bgp_total_received counter + frrk8s_bgp_total_received{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .TotalReceived }} + # HELP frrk8s_bgp_total_sent Number of total BGP messages sent + # TYPE frrk8s_bgp_total_sent counter + frrk8s_bgp_total_sent{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .TotalSent }} + # HELP frrk8s_bgp_updates_total Number of BGP UPDATE messages sent + # TYPE frrk8s_bgp_updates_total counter + frrk8s_bgp_updates_total{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .UpdatesTotal }} + # HELP frrk8s_bgp_updates_total_received Number of BGP UPDATE messages received + # TYPE frrk8s_bgp_updates_total_received counter + frrk8s_bgp_updates_total_received{peer="{{ .NeighborIP }}", vrf="{{ .NeighborVRF }}"} {{ .UpdatesTotalReceived }} + ` + + tests = []struct { + desc string + vtyshOutput string + neighborIP string + neighborVRF string + announcedPrefixes int + sessionUp int + updatesTotal int + updatesTotalReceived int + keepalivesSent int + keepalivesReceived int + opensSent int + opensReceived int + routeRefreshSent int + notificationsSent int + totalSent int + totalReceived int + }{ + { + desc: "Output contains only IPv4 advertisements", + vtyshOutput: neighborsIPv4Only, + neighborIP: "172.18.0.4:179", + neighborVRF: "default", + announcedPrefixes: 3, + sessionUp: 1, + updatesTotal: 3, + updatesTotalReceived: 3, + keepalivesSent: 4, + keepalivesReceived: 4, + opensSent: 1, + opensReceived: 1, + routeRefreshSent: 5, + notificationsSent: 2, + totalSent: 15, + totalReceived: 15, + }, + { + desc: "Output contains mixed IPv4 and IPv6 advertisements", + vtyshOutput: neighborsDual, + neighborIP: "172.18.0.4:180", + neighborVRF: "default", + announcedPrefixes: 6, + sessionUp: 1, + updatesTotal: 3, + updatesTotalReceived: 3, + keepalivesSent: 4, + keepalivesReceived: 4, + opensSent: 1, + opensReceived: 1, + routeRefreshSent: 5, + notificationsSent: 2, + totalSent: 15, + totalReceived: 15, + }, + } + neighborsIPv4Only = ` + { + "172.18.0.4":{ + "remoteAs":64512, + "localAs":64513, + "nbrExternalLink":true, + "hostname":"bgpd", + "bgpVersion":4, + "remoteRouterId":"172.18.0.4", + "localRouterId":"172.18.0.3", + "bgpState":"Established", + "bgpTimerUpMsec":1082000, + "bgpTimerUpString":"00:18:02", + "bgpTimerUpEstablishedEpoch":1632032518, + "bgpTimerLastRead":2000, + "bgpTimerLastWrite":2000, + "bgpInUpdateElapsedTimeMsecs":1081000, + "bgpTimerHoldTimeMsecs":180000, + "bgpTimerKeepAliveIntervalMsecs":60000, + "neighborCapabilities":{ + "4byteAs":"advertisedAndReceived", + "addPath":{ + "ipv4Unicast":{ + "rxAdvertisedAndReceived":true + } + }, + "routeRefresh":"advertisedAndReceivedOldNew", + "multiprotocolExtensions":{ + "ipv4Unicast":{ + "advertisedAndReceived":true + } + }, + "hostName":{ + "advHostName":"kind-control-plane", + "advDomainName":"n\/a", + "rcvHostName":"bgpd", + "rcvDomainName":"n\/a" + }, + "gracefulRestart":"advertisedAndReceived", + "gracefulRestartRemoteTimerMsecs":120000, + "addressFamiliesByPeer":"none" + }, + "gracefulRestartInfo":{ + "endOfRibSend":{ + "ipv4Unicast":true + }, + "endOfRibRecv":{ + "ipv4Unicast":true + }, + "localGrMode":"Helper*", + "remoteGrMode":"Helper", + "rBit":true, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":120 + }, + "ipv4Unicast":{ + "fBit":false, + "endOfRibStatus":{ + "endOfRibSend":true, + "endOfRibSentAfterUpdate":true, + "endOfRibRecv":true + }, + "timers":{ + "stalePathTimer":360 + } + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":1, + "notificationsSent":2, + "notificationsRecv":2, + "updatesSent":3, + "updatesRecv":3, + "keepalivesSent":4, + "keepalivesRecv":4, + "routeRefreshSent":5, + "routeRefreshRecv":5, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":15, + "totalRecv":15 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv4Unicast":{ + "updateGroupId":1, + "subGroupId":1, + "packetQueueLength":0, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0, + "sentPrefixCounter":3 + } + }, + "connectionsEstablished":1, + "connectionsDropped":0, + "lastResetTimerMsecs":1083000, + "lastResetDueTo":"Waiting for peer OPEN", + "lastResetCode":32, + "hostLocal":"172.18.0.3", + "portLocal":42692, + "hostForeign":"172.18.0.4", + "portForeign":179, + "nexthop":"172.18.0.3", + "nexthopGlobal":"fc00:f853:ccd:e793::3", + "nexthopLocal":"fe80::42:acff:fe12:3", + "bgpConnection":"sharedNetwork", + "connectRetryTimer":120, + "estimatedRttInMsecs":2, + "readThread":"on", + "writeThread":"on" + } + } + ` + neighborsDual = ` + { + "172.18.0.4":{ + "remoteAs":64512, + "localAs":64513, + "nbrExternalLink":true, + "hostname":"bgpd", + "bgpVersion":4, + "remoteRouterId":"172.18.0.4", + "localRouterId":"172.18.0.3", + "bgpState":"Established", + "bgpTimerUpMsec":1082000, + "bgpTimerUpString":"00:18:02", + "bgpTimerUpEstablishedEpoch":1632032518, + "bgpTimerLastRead":2000, + "bgpTimerLastWrite":2000, + "bgpInUpdateElapsedTimeMsecs":1081000, + "bgpTimerHoldTimeMsecs":180000, + "bgpTimerKeepAliveIntervalMsecs":60000, + "neighborCapabilities":{ + "4byteAs":"advertisedAndReceived", + "addPath":{ + "ipv4Unicast":{ + "rxAdvertisedAndReceived":true + }, + "ipv6Unicast":{ + "rxAdvertisedAndReceived":true + } + }, + "routeRefresh":"advertisedAndReceivedOldNew", + "multiprotocolExtensions":{ + "ipv4Unicast":{ + "advertisedAndReceived":true + }, + "ipv6Unicast":{ + "advertisedAndReceived":true + } + }, + "hostName":{ + "advHostName":"kind-control-plane", + "advDomainName":"n\/a", + "rcvHostName":"bgpd", + "rcvDomainName":"n\/a" + }, + "gracefulRestart":"advertisedAndReceived", + "gracefulRestartRemoteTimerMsecs":120000, + "addressFamiliesByPeer":"none" + }, + "gracefulRestartInfo":{ + "endOfRibSend":{ + "ipv4Unicast":true, + "ipv6Unicast":true + }, + "endOfRibRecv":{ + "ipv4Unicast":true, + "ipv6Unicast":true + }, + "localGrMode":"Helper*", + "remoteGrMode":"Helper", + "rBit":true, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":120 + }, + "ipv4Unicast":{ + "fBit":false, + "endOfRibStatus":{ + "endOfRibSend":true, + "endOfRibSentAfterUpdate":true, + "endOfRibRecv":true + }, + "timers":{ + "stalePathTimer":360 + } + }, + "ipv6Unicast":{ + "fBit":false, + "endOfRibStatus":{ + "endOfRibSend":true, + "endOfRibSentAfterUpdate":true, + "endOfRibRecv":true + }, + "timers":{ + "stalePathTimer":360 + } + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":1, + "notificationsSent":2, + "notificationsRecv":2, + "updatesSent":3, + "updatesRecv":3, + "keepalivesSent":4, + "keepalivesRecv":4, + "routeRefreshSent":5, + "routeRefreshRecv":5, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":15, + "totalRecv":15 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv4Unicast":{ + "updateGroupId":1, + "subGroupId":1, + "packetQueueLength":0, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0, + "sentPrefixCounter":3 + }, + "ipv6Unicast":{ + "peerGroupMember":"uplink", + "updateGroupId":2, + "subGroupId":2, + "packetQueueLength":0, + "commAttriSentToNbr":"extendedAndStandard", + "outboundPathPolicyConfig":true, + "outgoingUpdatePrefixFilterList":"only-host-prefixes", + "acceptedPrefixCounter":13, + "sentPrefixCounter":3 + } + }, + "connectionsEstablished":1, + "connectionsDropped":0, + "lastResetTimerMsecs":1083000, + "lastResetDueTo":"Waiting for peer OPEN", + "lastResetCode":32, + "hostLocal":"172.18.0.3", + "portLocal":42692, + "hostForeign":"172.18.0.4", + "portForeign":180, + "nexthop":"172.18.0.3", + "nexthopGlobal":"fc00:f853:ccd:e793::3", + "nexthopLocal":"fe80::42:acff:fe12:3", + "bgpConnection":"sharedNetwork", + "connectRetryTimer":120, + "estimatedRttInMsecs":2, + "readThread":"on", + "writeThread":"on" + } + } + ` + vrfVtysh = `{ + "default":{ + "vrfId": 0, + "vrfName": "default", + "tableVersion": 1, + "routerId": "172.18.0.3", + "defaultLocPrf": 100, + "localAS": 64512, + "routes": { "192.168.10.0/32": [ + { + "valid":true, + "bestpath":true, + "pathFrom":"external", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "metric":0, + "weight":32768, + "peerId":"(unspec)", + "path":"", + "origin":"IGP", + "nexthops":[ + { + "ip":"0.0.0.0", + "hostname":"kind-control-plane", + "afi":"ipv4", + "used":true + } + ] + } + ] } } + , + "red":{ + "vrfId": 5, + "vrfName": "red", + "tableVersion": 1, + "routerId": "172.31.0.4", + "defaultLocPrf": 100, + "localAS": 64512, + "routes": { "192.168.10.0/32": [ + { + "valid":true, + "bestpath":true, + "pathFrom":"external", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "metric":0, + "weight":32768, + "peerId":"(unspec)", + "path":"", + "origin":"IGP", + "nexthops":[ + { + "ip":"0.0.0.0", + "hostname":"kind-control-plane", + "afi":"ipv4", + "used":true + } + ] + } + ] } } + }` +) + +func TestCollect(t *testing.T) { + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + tmpl, err := template.New(tc.desc).Parse(metricsTmpl) + if err != nil { + t.Errorf("expected no error but got %s", err) + } + + var w bytes.Buffer + err = tmpl.Execute(&w, map[string]interface{}{ + "NeighborIP": tc.neighborIP, + "NeighborVRF": tc.neighborVRF, + "AnnouncedPrefixes": tc.announcedPrefixes, + "SessionUp": tc.sessionUp, + "UpdatesTotal": tc.updatesTotal, + "UpdatesTotalReceived": tc.updatesTotalReceived, + "KeepalivesReceived": tc.keepalivesReceived, + "KeepalivesSent": tc.keepalivesSent, + "NotificationsSent": tc.notificationsSent, + "OpensReceived": tc.opensReceived, + "OpensSent": tc.opensSent, + "RouteRefreshSent": tc.routeRefreshSent, + "TotalReceived": tc.totalReceived, + "TotalSent": tc.totalSent, + }) + + if err != nil { + t.Errorf("expected no error but got %s", err) + } + + l := log.NewNopLogger() + collector := mocknewBGP(l) + cmdOutput := map[string]string{ + "show bgp vrf all json": vrfVtysh, + "show bgp vrf default neighbors json": tc.vtyshOutput, + } + collector.frrCli = func(args string) (string, error) { + res, ok := cmdOutput[args] + if !ok { + return "{}", nil + } + return res, nil + } + buf := bytes.NewReader(w.Bytes()) + err = testutil.CollectAndCompare(collector, buf) + if err != nil { + t.Errorf("expected no error but got %s", err) + } + }) + } +} diff --git a/frr-tools/metrics/collector/metrics.go b/frr-tools/metrics/collector/metrics.go new file mode 100644 index 00000000..8cacc0cb --- /dev/null +++ b/frr-tools/metrics/collector/metrics.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier:Apache-2.0 + +package collector + +type metric struct { + Name string + Help string +} + +var ( + Namespace = "frrk8s" + Subsystem = "bgp" + + SessionUp = metric{ + Name: "session_up", + Help: "BGP session state (1 is up, 0 is down)", + } + + UpdatesSent = metric{ + Name: "updates_total", + Help: "Number of BGP UPDATE messages sent", + } + + Prefixes = metric{ + Name: "announced_prefixes_total", + Help: "Number of prefixes currently being advertised on the BGP session", + } +) diff --git a/frr-tools/metrics/exporter.go b/frr-tools/metrics/exporter.go new file mode 100644 index 00000000..cebe1091 --- /dev/null +++ b/frr-tools/metrics/exporter.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier:Apache-2.0 + +package main + +import ( + "flag" + "fmt" + stdlog "log" + "net/http" + "os" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/metallb/frrk8s/frr-tools/metrics/collector" + "github.com/metallb/frrk8s/frr-tools/metrics/liveness" + "github.com/metallb/frrk8s/frr-tools/metrics/vtysh" + "github.com/metallb/frrk8s/internal/logging" + "github.com/metallb/frrk8s/internal/version" +) + +var ( + metricsPort = flag.Uint("metrics-port", 7473, "Port to listen on for web interface.") + metricsPath = flag.String("metrics-path", "/metrics", "Path under which to expose metrics.") +) + +func metricsHandler(logger log.Logger) http.Handler { + BGPCollector := collector.NewBGP(logger) + BFDCollector := collector.NewBFD(logger) + + registry := prometheus.NewRegistry() + registry.MustRegister(BGPCollector) + registry.MustRegister(BFDCollector) + + gatherers := prometheus.Gatherers{ + prometheus.DefaultGatherer, + registry, + } + + handlerOpts := promhttp.HandlerOpts{ + ErrorLog: stdlog.New(log.NewStdlibAdapter(level.Error(logger)), "", 0), + ErrorHandling: promhttp.ContinueOnError, + Registry: registry, + } + + return promhttp.HandlerFor(gatherers, handlerOpts) +} + +func main() { + flag.Parse() + + logger, err := logging.Init("error") + if err != nil { + fmt.Printf("failed to initialize logging: %s\n", err) + os.Exit(1) + } + + level.Info(logger).Log("version", version.Version(), "commit", version.CommitHash(), "branch", version.Branch(), "goversion", version.GoString(), "msg", "FRR metrics exporter starting "+version.String()) + + mux := http.NewServeMux() + mux.Handle(*metricsPath, metricsHandler(logger)) + mux.Handle("/livez", liveness.Handler(vtysh.Run, logger)) + level.Info(logger).Log("msg", "Starting exporter", "metricsPath", metricsPath, "port", metricsPort) + + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", *metricsPort), + ReadTimeout: 3 * time.Second, + Handler: mux, + } + + if err := srv.ListenAndServe(); err != nil { + level.Error(logger).Log("error", err) + os.Exit(1) + } +} diff --git a/frr-tools/metrics/liveness/liveness.go b/frr-tools/metrics/liveness/liveness.go new file mode 100644 index 00000000..e9569bf7 --- /dev/null +++ b/frr-tools/metrics/liveness/liveness.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier:Apache-2.0 + +package liveness + +import ( + "net/http" + "strings" + + "github.com/go-kit/log" + "github.com/metallb/frrk8s/frr-tools/metrics/vtysh" +) + +func Handler(frrCli vtysh.Cli, logger log.Logger) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + res, err := frrCli("show daemons") + if err != nil { + http.Error(w, "failed to call show daemons", http.StatusInternalServerError) + logger.Log("failed to call show daemons", err) + return + } + expected := map[string]struct{}{ + "bfdd": {}, + "bgpd": {}, + "staticd": {}, + "watchfrr": {}, + "zebra": {}, + } + + runningDaemons := strings.Split(strings.TrimSuffix(res, "\n"), " ") + for _, d := range runningDaemons { + delete(expected, d) + } + if len(expected) > 0 { + logger.Log("daemons not running. got: ", res, "missing: ", expected) + http.NotFound(w, r) + } + }) +} diff --git a/frr-tools/metrics/liveness/liveness_test.go b/frr-tools/metrics/liveness/liveness_test.go new file mode 100644 index 00000000..f727e93c --- /dev/null +++ b/frr-tools/metrics/liveness/liveness_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier:Apache-2.0 + +package liveness + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/metallb/frrk8s/internal/logging" +) + +func TestLiveness(t *testing.T) { + tests := []struct { + desc string + vtyshRes string + vtyshError error + expectedStatusCode int + }{ + { + desc: "regular", + vtyshRes: " zebra bgpd watchfrr staticd bfdd\n", + expectedStatusCode: http.StatusOK, + }, + { + desc: "returns error", + vtyshError: fmt.Errorf("failed to run"), + expectedStatusCode: http.StatusInternalServerError, + }, + { + desc: "less daemons", + vtyshRes: " zebra bgpd staticd bfdd\n", + expectedStatusCode: http.StatusNotFound, + }, + } + + logger, err := logging.Init("error") + if err != nil { + t.Fatalf("failed to create logger %v", err) + } + req := httptest.NewRequest(http.MethodGet, "/livez", nil) + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + w := httptest.NewRecorder() + vtysh := func(args string) (string, error) { + return test.vtyshRes, test.vtyshError + } + handler := Handler(vtysh, logger) + handler.ServeHTTP(w, req) + res := w.Result() + defer res.Body.Close() + if res.StatusCode != test.expectedStatusCode { + t.Errorf("status code %d different from expected %d", res.StatusCode, test.expectedStatusCode) + } + }) + } +} diff --git a/frr-tools/metrics/vtysh/vtysh.go b/frr-tools/metrics/vtysh/vtysh.go new file mode 100644 index 00000000..69422cec --- /dev/null +++ b/frr-tools/metrics/vtysh/vtysh.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier:Apache-2.0 + +package vtysh + +import ( + "os/exec" + + "github.com/metallb/frrk8s/internal/frr" +) + +type Cli func(args string) (string, error) + +func Run(args string) (string, error) { + out, err := exec.Command("/usr/bin/vtysh", "-c", args).CombinedOutput() + return string(out), err +} + +var _ Cli = Run + +func VRFs(frrCli Cli) ([]string, error) { + vrfs, err := frrCli("show bgp vrf all json") + if err != nil { + return nil, err + } + parsedVRFs, err := frr.ParseVRFs(vrfs) + if err != nil { + return nil, err + } + return parsedVRFs, nil +} diff --git a/frr-tools/reloader/frr-reloader.sh b/frr-tools/reloader/frr-reloader.sh new file mode 100755 index 00000000..111fcd6b --- /dev/null +++ b/frr-tools/reloader/frr-reloader.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -o pipefail + +cleanup() { + echo "Caught an exit signal.." + clean_files + kill_sleep + exit +} + +reload_frr() { + flock 200 + echo "Caught SIGHUP and acquired lock! Reloading FRR.." + SECONDS=0 + + kill_sleep + + echo "Checking the configuration file syntax" + if ! python3 /usr/lib/frr/frr-reload.py --test --stdout "$FILE_TO_RELOAD" 2>&1 | sed 's/password.*/password /g'; then + echo "Syntax error spotted: aborting.. $SECONDS seconds" + echo -n "$(date +%s) failure" > "$STATUSFILE" + return + fi + + echo "Applying the configuration file" + if ! python3 /usr/lib/frr/frr-reload.py --reload --overwrite --stdout "$FILE_TO_RELOAD" 2>&1 | sed 's/password.*/password /g'; then + echo "Failed to fully apply configuration file $SECONDS seconds" + echo -n "$(date +%s) failure" > "$STATUSFILE" + return + fi + + echo "FRR reloaded successfully! $SECONDS seconds" + echo -n "$(date +%s) success" > "$STATUSFILE" +} 200<"$LOCKFILE" + +kill_sleep() { + kill "$sleep_pid" +} + +clean_files() { + rm -f "$PIDFILE" + rm -f "$LOCKFILE" +} + +trap cleanup SIGTERM SIGINT +# The need for & is explained here: https://github.com/metallb/metallb/pull/935#issuecomment-943097999 +# TLDR: & allows signals to trigger reload_frr immediately, flock keeps the order and creates a queue. +trap 'reload_frr &' HUP + +SHARED_VOLUME="${SHARED_VOLUME:-/etc/frr_reloader}" +PIDFILE="$SHARED_VOLUME/reloader.pid" +FILE_TO_RELOAD="$SHARED_VOLUME/frr.conf" +LOCKFILE="$SHARED_VOLUME/lock" +STATUSFILE="$SHARED_VOLUME/.status" + +clean_files +echo "PID is: $$, writing to $PIDFILE" +printf "$$" > "$PIDFILE" +touch "$LOCKFILE" + +while true +do + sleep infinity & + sleep_pid=$! + wait $sleep_pid 2>/dev/null +done diff --git a/go.mod b/go.mod index 4573ef97..8455018a 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,37 @@ module github.com/metallb/frrk8s go 1.19 require ( + github.com/go-kit/log v0.2.1 + github.com/google/go-cmp v0.5.9 github.com/onsi/ginkgo/v2 v2.6.0 github.com/onsi/gomega v1.24.1 + github.com/ory/dockertest/v3 v3.10.0 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.14.0 k8s.io/api v0.26.4 k8s.io/apimachinery v0.26.4 k8s.io/client-go v1.5.2 + k8s.io/klog v1.0.0 sigs.k8s.io/controller-runtime v0.14.4 ) require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/continuity v0.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/cli v20.10.17+incompatible // indirect + github.com/docker/docker v20.10.7+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -27,32 +43,41 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect + github.com/opencontainers/runc v1.1.5 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.39.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.24.0 // indirect + golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.7.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 67825cbc..3fb79355 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -8,18 +14,38 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= +github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -34,9 +60,15 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= @@ -49,6 +81,9 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -78,12 +113,15 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -105,21 +143,38 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= +github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= +github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -135,13 +190,20 @@ github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NX github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -150,8 +212,15 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -177,6 +246,8 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -188,6 +259,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= @@ -202,15 +274,24 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -231,10 +312,13 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -289,6 +373,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= @@ -301,6 +387,8 @@ k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a h1:s6zvHjyDQX1NtVT88pvw2tddqhqY0Bz0Gbnn+yctsFU= diff --git a/hack/external_frr.sh b/hack/external_frr.sh new file mode 100755 index 00000000..c12437ab --- /dev/null +++ b/hack/external_frr.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -x + +NODES=$(kubectl get nodes -l node-role.kubernetes.io/worker=worker -o jsonpath={.items[*].status.addresses[?\(@.type==\"InternalIP\"\)].address}) +echo $NODES +pushd ./hack/frr/ +go run . -nodes "$NODES" +popd + +FRR_CONFIG=$(mktemp -d -t frr-XXXXXXXXXX) +cp hack/frr/*.conf $FRR_CONFIG +cp hack/frr/daemons $FRR_CONFIG +chmod a+rw $FRR_CONFIG/* + +docker rm -f frr +docker run -d --privileged --network kind --rm --ulimit core=-1 --name frr --volume "$FRR_CONFIG":/etc/frr quay.io/frrouting/frr:8.4.2 + +FRR_IP=$(docker inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" frr) + + +cat < 1 { // TODO implement merging + return ctrl.Result{}, nil + } + + if len(configs.Items) == 0 { + empty := frrk8sv1beta1.FRRConfiguration{} + config, err := apiToFRR(empty) + if err != nil { + level.Error(r.Logger).Log("controller", "FRRConfigurationReconciler", "failed to translate the empty config", req.NamespacedName.String(), "error", err) + return ctrl.Result{}, nil + } + + if err := r.FRRHandler.ApplyConfig(config); err != nil { + level.Error(r.Logger).Log("controller", "FRRConfigurationReconciler", "failed to apply the empty config", req.NamespacedName.String(), "error", err) + } + return ctrl.Result{}, nil + } + config, err := apiToFRR(configs.Items[0]) + if err != nil { + level.Error(r.Logger).Log("controller", "FRRConfigurationReconciler", "failed to apply the config", req.NamespacedName.String(), "error", err) + return ctrl.Result{}, nil + } + + if err := r.FRRHandler.ApplyConfig(config); err != nil { + level.Error(r.Logger).Log("controller", "FRRConfigurationReconciler", "failed to apply the config", req.NamespacedName.String(), "error", err) + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil } diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go deleted file mode 100644 index 228f263a..00000000 --- a/internal/controller/suite_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - frrk8sv1beta1 "github.com/metallb/frrk8s/api/v1beta1" - //+kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - RegisterFailHandler(Fail) - - RunSpecs(t, "Controller Suite") -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = frrk8sv1beta1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - -}) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) diff --git a/internal/frr/config.go b/internal/frr/config.go new file mode 100644 index 00000000..f406ca0d --- /dev/null +++ b/internal/frr/config.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "bytes" + "context" + "embed" + "fmt" + "os" + "reflect" + "strconv" + "syscall" + "text/template" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/metallb/frrk8s/internal/ipfamily" + "github.com/pkg/errors" +) + +var ( + configFileName = "/etc/frr_reloader/frr.conf" + reloaderPidFileName = "/etc/frr_reloader/reloader.pid" + //go:embed templates/* templates/* + templates embed.FS +) + +type Config struct { + Loglevel string + Hostname string + Routers []*RouterConfig + BFDProfiles []BFDProfile + ExtraConfig string +} + +type reloadEvent struct { + config *Config + useOld bool +} + +type RouterConfig struct { + MyASN uint32 + RouterID string + Neighbors []*NeighborConfig + VRF string + IPV4Prefixes []string + IPV6Prefixes []string +} + +type BFDProfile struct { + Name string + ReceiveInterval *uint32 + TransmitInterval *uint32 + DetectMultiplier *uint32 + EchoInterval *uint32 + EchoMode bool + PassiveMode bool + MinimumTTL *uint32 +} + +type NeighborConfig struct { + IPFamily ipfamily.Family + Name string + ASN uint32 + SrcAddr string + Addr string + Port uint16 + HoldTime uint64 + KeepaliveTime uint64 + Password string + Advertisements []*AdvertisementConfig + BFDProfile string + EBGPMultiHop bool + VRFName string + HasV4Advertisements bool + HasV6Advertisements bool +} + +func (n *NeighborConfig) ID() string { + if n.VRFName == "" { + return n.Addr + } + return fmt.Sprintf("%s-%s", n.Addr, n.VRFName) +} + +type AdvertisementConfig struct { + IPFamily ipfamily.Family + Prefix string + Communities []string + LocalPref uint32 +} + +// templateConfig uses the template library to template +// 'globalConfigTemplate' using 'data'. +func templateConfig(data interface{}) (string, error) { + i := 0 + currentCounterName := "" + t, err := template.New("frr.tmpl").Funcs( + template.FuncMap{ + "counter": func(counterName string) int { + if currentCounterName != counterName { + currentCounterName = counterName + i = 0 + } + i++ + return i + }, + "frrIPFamily": func(ipFamily ipfamily.Family) string { + if ipFamily == "ipv6" { + return "ipv6" + } + return "ip" + }, + "localPrefPrefixList": func(neighbor *NeighborConfig, localPreference uint32) string { + return fmt.Sprintf("%s-%d-%s-localpref-prefixes", neighbor.ID(), localPreference, neighbor.IPFamily) + }, + "communityPrefixList": func(neighbor *NeighborConfig, community string) string { + return fmt.Sprintf("%s-%s-%s-community-prefixes", neighbor.ID(), community, neighbor.IPFamily) + }, + "allowedPrefixList": func(neighbor *NeighborConfig) string { + return fmt.Sprintf("%s-pl-%s", neighbor.ID(), neighbor.IPFamily) + }, + "mustDisableConnectedCheck": func(ipFamily ipfamily.Family, myASN, asn uint32, eBGPMultiHop bool) bool { + // return true only for IPv6 eBGP sessions + if ipFamily == "ipv6" && myASN != asn && !eBGPMultiHop { + return true + } + return false + }, + "dict": func(values ...interface{}) (map[string]interface{}, error) { + if len(values)%2 != 0 { + return nil, errors.New("invalid dict call, expecting even number of args") + } + dict := make(map[string]interface{}, len(values)/2) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].(string) + if !ok { + return nil, fmt.Errorf("dict keys must be strings, got %v %T", values[i], values[i]) + } + dict[key] = values[i+1] + } + return dict, nil + }, + }).ParseFS(templates, "templates/*") + if err != nil { + return "", err + } + + var b bytes.Buffer + err = t.Execute(&b, data) + return b.String(), err +} + +// writeConfigFile writes the FRR configuration file (represented as a string) +// to 'filename'. +func writeConfig(config string, filename string) error { + return os.WriteFile(filename, []byte(config), 0600) +} + +// reloadConfig requests that FRR reloads the configuration file. This is +// called after updating the configuration. +var reloadConfig = func() error { + pidFile, found := os.LookupEnv("FRR_RELOADER_PID_FILE") + if found { + reloaderPidFileName = pidFile + } + + pid, err := os.ReadFile(reloaderPidFileName) + if err != nil { + return err + } + + pidInt, err := strconv.Atoi(string(pid)) + if err != nil { + return err + } + + // send HUP signal to FRR reloader + err = syscall.Kill(pidInt, syscall.SIGHUP) + if err != nil { + return err + } + + return nil +} + +// generateAndReloadConfigFile takes a 'struct Config' and, using a template, +// generates and writes a valid FRR configuration file. If this completes +// successfully it will also force FRR to reload that configuration file. +func generateAndReloadConfigFile(config *Config, l log.Logger) error { + filename, found := os.LookupEnv("FRR_CONFIG_FILE") + if found { + configFileName = filename + } + + configString, err := templateConfig(config) + if err != nil { + level.Error(l).Log("op", "reload", "error", err, "cause", "template", "config", config) + return err + } + err = writeConfig(configString, configFileName) + if err != nil { + level.Error(l).Log("op", "reload", "error", err, "cause", "writeConfig", "config", config) + return err + } + + err = reloadConfig() + if err != nil { + level.Error(l).Log("op", "reload", "error", err, "cause", "reload", "config", config) + return err + } + return nil +} + +// debouncer takes a function that processes an Config, a channel where +// the update requests are sent, and squashes any requests coming in a given timeframe +// as a single request. +func debouncer(ctx context.Context, body func(config *Config) error, + reload <-chan reloadEvent, + reloadInterval time.Duration, + failureRetryInterval time.Duration, + l log.Logger) { + go func() { + var config *Config + var timeOut <-chan time.Time + timerSet := false + for { + select { + case newCfg, ok := <-reload: + if !ok { // the channel was closed + return + } + if newCfg.useOld && config == nil { + level.Debug(l).Log("op", "reload", "action", "ignore config", "reason", "nil config") + continue // just ignore the event + } + if !newCfg.useOld && reflect.DeepEqual(newCfg.config, config) { + level.Debug(l).Log("op", "reload", "action", "ignore config", "reason", "same config") + continue // config hasn't changed + } + if !newCfg.useOld { + config = newCfg.config + } + if !timerSet { + timeOut = time.After(reloadInterval) + timerSet = true + } + case <-timeOut: + err := body(config) + if err != nil { + timeOut = time.After(failureRetryInterval) + timerSet = true + continue + } + timerSet = false + case <-ctx.Done(): + return + } + } + }() +} diff --git a/internal/frr/debounce_test.go b/internal/frr/debounce_test.go new file mode 100644 index 00000000..421adce1 --- /dev/null +++ b/internal/frr/debounce_test.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/go-kit/log" +) + +const timer = 10 * time.Millisecond +const failureTimer = 10 * time.Millisecond + +func TestDebounce(t *testing.T) { + result := make(chan *Config, 10) // buffered to accommodate spurious rewrites + dummyUpdate := func(config *Config) error { + result <- config + return nil + } + + reload := make(chan reloadEvent) + defer close(reload) + debouncer(context.Background(), dummyUpdate, reload, timer, failureTimer, log.NewNopLogger()) + reload <- reloadEvent{config: &Config{Hostname: "1"}} + reload <- reloadEvent{config: &Config{Hostname: "2"}} + reload <- reloadEvent{config: &Config{Hostname: "3"}} + if len(result) != 0 { + t.Fatal("received update before time") + } + time.Sleep(3 * timer) + if len(result) != 1 { + t.Fatal("received extra updates", len(result)) + } + updated := <-result + if updated.Hostname != "3" { + t.Fatal("Config was not updated") + } + + reload <- reloadEvent{config: &Config{Hostname: "3"}} + reload <- reloadEvent{config: &Config{Hostname: "4"}} + reload <- reloadEvent{config: &Config{Hostname: "5"}} + time.Sleep(3 * timer) + if len(result) != 1 { + t.Fatal("received extra updates", len(result)) + } + updated = <-result + if updated.Hostname != "5" { + t.Fatal("Config was not updated") + } +} + +func TestDebounceRetry(t *testing.T) { + result := make(chan *Config, 10) // buffered to accommodate spurious rewrites + count := 0 + dummyUpdate := func(config *Config) error { + count++ + if count <= 3 { + return fmt.Errorf("error") + } + result <- config + return nil + } + + reload := make(chan reloadEvent) + defer close(reload) + debouncer(context.Background(), dummyUpdate, reload, timer, failureTimer, log.NewNopLogger()) + + reload <- reloadEvent{config: &Config{Hostname: "1"}} + reload <- reloadEvent{config: &Config{Hostname: "2"}} + if len(result) != 0 { + t.Fatal("received update before time") + } + time.Sleep(10 * failureTimer) + if len(result) != 1 { + t.Fatal("received extra updates", len(result)) + } + updated := <-result + if updated.Hostname != "2" { + t.Fatal("Config was not updated") + } +} + +func TestDebounceReuseOld(t *testing.T) { + result := make(chan *Config, 10) // buffered to accommodate spurious rewrites + dummyUpdate := func(config *Config) error { + result <- config + return nil + } + + reload := make(chan reloadEvent) + defer close(reload) + debouncer(context.Background(), dummyUpdate, reload, timer, failureTimer, log.NewNopLogger()) + + reload <- reloadEvent{config: &Config{Hostname: "1"}} + if len(result) != 0 { + t.Fatal("received update before time") + } + time.Sleep(3 * timer) + if len(result) != 1 { + t.Fatal("received extra updates", len(result)) + } + updated := <-result + if updated.Hostname != "1" { + t.Fatal("Config was not updated") + } + // reload to see if the debouncer uses the old config + reload <- reloadEvent{useOld: true} + time.Sleep(3 * timer) + if len(result) != 1 { + t.Fatal("received extra updates", len(result)) + } + updated = <-result + if updated.Hostname != "1" { + t.Fatal("Config was not updated") + } +} + +func TestDebounceSameConfig(t *testing.T) { + result := make(chan *Config, 10) // buffered to accommodate spurious rewrites + dummyUpdate := func(config *Config) error { + result <- config + return nil + } + + reload := make(chan reloadEvent) + defer close(reload) + debouncer(context.Background(), dummyUpdate, reload, timer, failureTimer, log.NewNopLogger()) + reload <- reloadEvent{config: &Config{Hostname: "1"}} + reload <- reloadEvent{config: &Config{Hostname: "2"}} + reload <- reloadEvent{config: &Config{Hostname: "3", Routers: []*RouterConfig{{MyASN: 23}}}} + if len(result) != 0 { + t.Fatal("received update before time") + } + time.Sleep(3 * timer) + if len(result) != 1 { + t.Fatal("received extra updates", len(result)) + } + updated := <-result + if updated.Hostname != "3" { + t.Fatal("Config was not updated") + } + + reload <- reloadEvent{config: &Config{Hostname: "3", Routers: []*RouterConfig{{MyASN: 23}}}} + reload <- reloadEvent{config: &Config{Hostname: "3", Routers: []*RouterConfig{{MyASN: 23}}}} + + time.Sleep(3 * timer) + if len(result) != 0 { + updated := <-result + t.Fatalf("received extra updates: %d %s", len(result), updated.Hostname) + } +} diff --git a/internal/frr/docker_test.go b/internal/frr/docker_test.go new file mode 100644 index 00000000..23e6ff45 --- /dev/null +++ b/internal/frr/docker_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "bytes" + "flag" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/ory/dockertest/v3" + "github.com/pkg/errors" +) + +var ( + containerHandle *dockertest.Resource + frrDir string +) + +const ( + frrImageTag = "8.4.2" +) + +func TestMain(m *testing.M) { + // override reloadConfig so it doesn't try to reload it. + debounceTimeout = time.Millisecond + reloadConfig = func() error { return nil } + + flag.Parse() + if !testing.Short() { + testWithDocker(m) + return + } + m.Run() +} + +func testWithDocker(m *testing.M) { + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("failed to create dockertest pool %s", err) + } + + frrDir, err = os.MkdirTemp("/tmp", "frr_integration") + if err != nil { + log.Fatalf("failed to create temp dir %s", err) + } + + containerHandle, err = pool.RunWithOptions( + &dockertest.RunOptions{ + Name: "frrtest", + Repository: "quay.io/frrouting/frr", + Tag: frrImageTag, + Mounts: []string{fmt.Sprintf("%s:/etc/tempfrr", frrDir)}, + }, + ) + if err != nil { + log.Fatalf("failed to run container %s", err) + } + + cmd := exec.Command("cp", "testdata/vtysh.conf", filepath.Join(frrDir, "vtysh.conf")) + res, err := cmd.CombinedOutput() + if err != nil { + log.Fatalf("failed to move vtysh.conf to %s - %s - %s", frrDir, err, res) + } + buf := new(bytes.Buffer) + resCode, err := containerHandle.Exec([]string{"cp", "/etc/tempfrr/vtysh.conf", "/etc/frr/vtysh.conf"}, + dockertest.ExecOptions{ + StdErr: buf, + }) + if err != nil || resCode != 0 { + log.Fatalf("failed to move vtysh.conf inside the container - res %d %s %s", resCode, err, buf.String()) + } + + retCode := m.Run() + // You can't defer this because os.Exit doesn't care for defer + if err := pool.Purge(containerHandle); err != nil { + log.Fatalf("failed to purge %s - %s", containerHandle.Container.Name, err) + } + os.RemoveAll(frrDir) + + os.Exit(retCode) +} + +type invalidFileErr struct { + Reason string +} + +func (e invalidFileErr) Error() string { + return e.Reason +} + +func testFileIsValid(fileName string) error { + if testing.Short() { + return nil + } + cmd := exec.Command("cp", fileName, filepath.Join(frrDir, "frr.conf")) + res, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "failed to copy %s to %s: %s", fileName, frrDir, string(res)) + } + _, err = containerHandle.Exec([]string{"cp", "/etc/tempfrr/frr.conf", "/etc/frr/frr.conf"}, + dockertest.ExecOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to copy frr.conf inside the container") + } + buf := new(bytes.Buffer) + code, err := containerHandle.Exec([]string{"python3", "/usr/lib/frr/frr-reload.py", "--test", "--stdout", "/etc/frr/frr.conf"}, + dockertest.ExecOptions{ + StdErr: buf, + }) + if err != nil { + return errors.Wrapf(err, "failed to exec reloader into the container") + } + + if code != 0 { + return invalidFileErr{Reason: buf.String()} + } + return nil +} + +func TestDockerFRRFails(t *testing.T) { + if testing.Short() { + t.Skip("skipping FRR integration") + } + + badFile := filepath.Join(testData, "TestDockerTestfails.golden") + err := testFileIsValid(badFile) + if !errors.As(err, &invalidFileErr{}) { + t.Fatalf("Validity check of invalid file passed") + } +} diff --git a/internal/frr/frr.go b/internal/frr/frr.go new file mode 100644 index 00000000..667aae00 --- /dev/null +++ b/internal/frr/frr.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/metallb/frrk8s/internal/logging" +) + +type ConfigHandler interface { + ApplyConfig(config *Config) error +} + +type FRR struct { + reloadConfig chan reloadEvent + logLevel string + sync.Mutex +} + +// Create a variable for os.Hostname() in order to make it easy to mock out +// in unit tests. +var osHostname = os.Hostname + +func (f *FRR) ApplyConfig(config *Config) error { + hostname, err := osHostname() + if err != nil { + return err + } + + // TODO add internal wrapper + config.Loglevel = f.logLevel + config.Hostname = hostname + f.reloadConfig <- reloadEvent{config: config} + return nil +} + +var debounceTimeout = 3 * time.Second +var failureTimeout = time.Second * 5 + +func NewFRR(ctx context.Context, logger log.Logger, logLevel logging.Level) *FRR { + res := &FRR{ + reloadConfig: make(chan reloadEvent), + logLevel: logLevelToFRR(logLevel), + } + reload := func(config *Config) error { + return generateAndReloadConfigFile(config, logger) + } + + debouncer(ctx, reload, res.reloadConfig, debounceTimeout, failureTimeout, logger) + reloadValidator(ctx, logger, res.reloadConfig) + return res +} + +func reloadValidator(ctx context.Context, l log.Logger, reload chan<- reloadEvent) { + var tickerIntervals = 30 * time.Second + var prevReloadTimeStamp string + + ticker := time.NewTicker(tickerIntervals) + go func() { + select { + case <-ticker.C: + validateReload(l, &prevReloadTimeStamp, reload) + case <-ctx.Done(): + return + } + }() +} + +const statusFileName = "/etc/frr_reloader/.status" + +func validateReload(l log.Logger, prevReloadTimeStamp *string, reload chan<- reloadEvent) { + bytes, err := os.ReadFile(statusFileName) + if err != nil { + if !os.IsNotExist(err) { + level.Error(l).Log("op", "reload-validate", "error", err, "cause", "readFile", "fileName", statusFileName) + } + return + } + + lastReloadStatus := strings.Fields(string(bytes)) + if len(lastReloadStatus) != 2 { + level.Error(l).Log("op", "reload-validate", "cause", "Fields", "bytes", string(bytes)) + return + } + + timeStamp, status := lastReloadStatus[0], lastReloadStatus[1] + if timeStamp == *prevReloadTimeStamp { + return + } + + *prevReloadTimeStamp = timeStamp + + if strings.Compare(status, "failure") == 0 { + level.Error(l).Log("op", "reload-validate", "error", fmt.Errorf("reload failure"), + "cause", "frr reload failed", "status", status) + reload <- reloadEvent{useOld: true} + return + } + + level.Info(l).Log("op", "reload-validate", "success", "reloaded config") +} + +func logLevelToFRR(level logging.Level) string { + // Allowed frr log levels are: emergencies, alerts, critical, + // errors, warnings, notifications, informational, or debugging + switch level { + case logging.LevelAll, logging.LevelDebug: + return "debugging" + case logging.LevelInfo: + return "informational" + case logging.LevelWarn: + return "warnings" + case logging.LevelError: + return "error" + case logging.LevelNone: + return "emergencies" + } + + return "informational" +} diff --git a/internal/frr/frr_test.go b/internal/frr/frr_test.go new file mode 100644 index 00000000..50c4f8df --- /dev/null +++ b/internal/frr/frr_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "context" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/metallb/frrk8s/internal/ipfamily" + "github.com/metallb/frrk8s/internal/logging" + "k8s.io/apimachinery/pkg/util/wait" +) + +const testData = "testdata/" + +var update = flag.Bool("update", false, "update .golden files") + +func testOsHostname() (string, error) { + return "dummyhostname", nil +} + +func testCompareFiles(t *testing.T, configFile, goldenFile string) { + var lastError error + + // Try comparing files multiple times because tests can generate more than one configuration + err := wait.PollImmediate(10*time.Millisecond, 2*time.Second, func() (bool, error) { + lastError = nil + cmd := exec.Command("diff", configFile, goldenFile) + output, err := cmd.Output() + + if err != nil { + lastError = fmt.Errorf("command %s returned error: %s\n%s", cmd.String(), err, output) + return false, nil + } + + return true, nil + }) + + // err can only be a ErrWaitTimeout, as the check function always return nil errors. + // So lastError is always set + if err != nil { + t.Fatalf("failed to compare configfiles %s, %s using poll interval\nlast error: %v", configFile, goldenFile, lastError) + } +} + +func testUpdateGoldenFile(t *testing.T, configFile, goldenFile string) { + t.Log("update golden file") + + // Sleep to be sure the sessionManager has produced all configuration the test + // has triggered and no config is still waiting in the debouncer() local variables. + // No other conditions can be checked, so sleeping is our best option. + time.Sleep(100 * time.Millisecond) + + cmd := exec.Command("cp", "-a", configFile, goldenFile) + output, err := cmd.Output() + if err != nil { + t.Fatalf("command %s returned %s and error: %s", cmd.String(), output, err) + } +} + +func testGenerateFileNames(t *testing.T) (string, string) { + return filepath.Join(testData, filepath.FromSlash(t.Name())), filepath.Join(testData, filepath.FromSlash(t.Name())+".golden") +} + +func testSetup(t *testing.T) { + configFile, _ := testGenerateFileNames(t) + os.Setenv("FRR_CONFIG_FILE", configFile) + _ = os.Remove(configFile) // removing leftovers from previous runs + osHostname = testOsHostname +} + +func testCheckConfigFile(t *testing.T) { + configFile, goldenFile := testGenerateFileNames(t) + + if *update { + testUpdateGoldenFile(t, configFile, goldenFile) + } + + testCompareFiles(t, configFile, goldenFile) + + if !strings.Contains(configFile, "Invalid") { + err := testFileIsValid(configFile) + if err != nil { + t.Fatalf("Failed to verify the file %s", err) + } + } +} + +func TestSingleSession(t *testing.T) { + testSetup(t) + ctx, cancel := context.WithCancel(context.Background()) + frr := NewFRR(ctx, log.NewNopLogger(), logging.LevelInfo) + defer cancel() + + config := Config{ + Routers: []*RouterConfig{ + { + MyASN: 65000, + Neighbors: []*NeighborConfig{ + { + IPFamily: ipfamily.IPv4, + ASN: 65001, + Addr: "192.168.1.2", + Port: 4567, + Advertisements: []*AdvertisementConfig{ + { + IPFamily: ipfamily.IPv4, + Prefix: "192.169.1.0/24", + }, + { + IPFamily: ipfamily.IPv4, + Prefix: "192.170.1.0/22", + }, + }, + }, + }, + }, + }, + } + err := frr.ApplyConfig(&config) + if err != nil { + t.Fatalf("Failed to apply config: %s", err) + } + + testCheckConfigFile(t) +} diff --git a/internal/frr/parse.go b/internal/frr/parse.go new file mode 100644 index 00000000..81c793e0 --- /dev/null +++ b/internal/frr/parse.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "encoding/json" + "fmt" + "net" + "sort" + "strconv" + + "github.com/pkg/errors" +) + +type Neighbor struct { + IP net.IP + VRF string + Connected bool + LocalAS string + RemoteAS string + PrefixSent int + Port int + RemoteRouterID string + MsgStats MessageStats +} + +type Route struct { + Destination *net.IPNet + NextHops []net.IP + LocalPref uint32 + Origin string +} + +const bgpConnected = "Established" + +type FRRNeighbor struct { + RemoteAs int `json:"remoteAs"` + LocalAs int `json:"localAs"` + RemoteRouterID string `json:"remoteRouterId"` + BgpVersion int `json:"bgpVersion"` + BgpState string `json:"bgpState"` + PortForeign int `json:"portForeign"` + MsgStats MessageStats `json:"messageStats"` + VRFName string `json:"vrf"` + AddressFamilyInfo map[string]struct { + SentPrefixCounter int `json:"sentPrefixCounter"` + } `json:"addressFamilyInfo"` +} + +type MessageStats struct { + OpensSent int `json:"opensSent"` + OpensReceived int `json:"opensRecv"` + NotificationsSent int `json:"notificationsSent"` + UpdatesSent int `json:"updatesSent"` + UpdatesReceived int `json:"updatesRecv"` + KeepalivesSent int `json:"keepalivesSent"` + KeepalivesReceived int `json:"keepalivesRecv"` + RouteRefreshSent int `json:"routeRefreshSent"` + TotalSent int `json:"totalSent"` + TotalReceived int `json:"totalRecv"` +} + +type IPInfo struct { + Routes map[string][]FRRRoute `json:"routes"` +} + +type FRRRoute struct { + Valid bool `json:"valid"` + PeerID string `json:"peerId"` + LocalPref uint32 `json:"locPrf"` + Origin string `json:"origin"` + Nexthops []struct { + IP string `json:"ip"` + Scope string `json:"scope"` + } `json:"nexthops"` +} + +type BFDPeer struct { + Multihop bool `json:"multihop"` + Peer string `json:"peer"` + Local string `json:"local"` + Vrf string `json:"vrf"` + Interface string `json:"interface"` + ID int `json:"id"` + RemoteID int64 `json:"remote-id"` + PassiveMode bool `json:"passive-mode"` + Status string `json:"status"` + Uptime int `json:"uptime"` + Diagnostic string `json:"diagnostic"` + RemoteDiagnostic string `json:"remote-diagnostic"` + ReceiveInterval int `json:"receive-interval"` + TransmitInterval int `json:"transmit-interval"` + EchoReceiveInterval int `json:"echo-receive-interval"` + EchoTransmitInterval int `json:"echo-transmit-interval"` + DetectMultiplier int `json:"detect-multiplier"` + RemoteReceiveInterval int `json:"remote-receive-interval"` + RemoteTransmitInterval int `json:"remote-transmit-interval"` + RemoteEchoInterval int `json:"remote-echo-interval"` + RemoteEchoReceiveInterval int `json:"remote-echo-receive-interval"` + RemoteDetectMultiplier int `json:"remote-detect-multiplier"` +} + +// parseNeighbour takes the result of a show bgp neighbor x.y.w.z +// and parses the informations related to the neighbour. +func ParseNeighbour(vtyshRes string) (*Neighbor, error) { + res := map[string]FRRNeighbor{} + err := json.Unmarshal([]byte(vtyshRes), &res) + if err != nil { + return nil, errors.Wrap(err, "failed to parse vtysh response") + } + if len(res) > 1 { + return nil, errors.New("more than one peer were returned") + } + if len(res) == 0 { + return nil, errors.New("no peers were returned") + } + for k, n := range res { + ip := net.ParseIP(k) + if ip == nil { + return nil, fmt.Errorf("failed to parse %s as ip", ip) + } + connected := true + if n.BgpState != bgpConnected { + connected = false + } + prefixSent := 0 + for _, s := range n.AddressFamilyInfo { + prefixSent += s.SentPrefixCounter + } + return &Neighbor{ + IP: ip, + Connected: connected, + LocalAS: strconv.Itoa(n.LocalAs), + RemoteAS: strconv.Itoa(n.RemoteAs), + PrefixSent: prefixSent, + Port: n.PortForeign, + RemoteRouterID: n.RemoteRouterID, + MsgStats: n.MsgStats, + }, nil + } + return nil, errors.New("no peers were returned") +} + +// parseNeighbour takes the result of a show bgp neighbor +// and parses the informations related to all the neighbours. +func ParseNeighbours(vtyshRes string) ([]*Neighbor, error) { + toParse := map[string]FRRNeighbor{} + err := json.Unmarshal([]byte(vtyshRes), &toParse) + if err != nil { + return nil, errors.Wrap(err, "failed to parse vtysh response") + } + + res := make([]*Neighbor, 0) + for k, n := range toParse { + ip := net.ParseIP(k) + if ip == nil { + return nil, fmt.Errorf("failed to parse %s as ip", ip) + } + connected := true + if n.BgpState != bgpConnected { + connected = false + } + prefixSent := 0 + for _, s := range n.AddressFamilyInfo { + prefixSent += s.SentPrefixCounter + } + res = append(res, &Neighbor{ + IP: ip, + Connected: connected, + LocalAS: strconv.Itoa(n.LocalAs), + RemoteAS: strconv.Itoa(n.RemoteAs), + PrefixSent: prefixSent, + Port: n.PortForeign, + RemoteRouterID: n.RemoteRouterID, + MsgStats: n.MsgStats, + }) + } + return res, nil +} + +// parseRoute takes the result of a show bgp ipv4 / ipv6 +// and parses the informations related to all the routes. +func ParseRoutes(vtyshRes string) (map[string]Route, error) { + toParse := IPInfo{} + err := json.Unmarshal([]byte(vtyshRes), &toParse) + if err != nil { + return nil, errors.Wrap(err, "failed to parse vtysh response") + } + + res := make(map[string]Route) + for k, frrRoutes := range toParse.Routes { + destIP, dest, err := net.ParseCIDR(k) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse cidr for %s", k) + } + + r := Route{ + Destination: dest, + NextHops: make([]net.IP, 0), + } + for _, n := range frrRoutes { + r.LocalPref = n.LocalPref + r.Origin = n.Origin + out: + for _, h := range n.Nexthops { + ip := net.ParseIP(h.IP) + if ip == nil { + return nil, fmt.Errorf("failed to parse ip %s", h.IP) + } + if ip.To4() == nil && h.Scope == "link-local" { + continue + } + for _, current := range r.NextHops { + if ip.Equal(current) { + continue out + } + } + r.NextHops = append(r.NextHops, ip) + } + } + res[destIP.String()] = r + } + return res, nil +} + +func ParseBFDPeers(vtyshRes string) ([]BFDPeer, error) { + parseRes := []BFDPeer{} + err := json.Unmarshal([]byte(vtyshRes), &parseRes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse vtysh response") + } + return parseRes, nil +} + +func ParseVRFs(vtyshRes string) ([]string, error) { + vrfs := map[string]interface{}{} + err := json.Unmarshal([]byte(vtyshRes), &vrfs) + if err != nil { + return nil, errors.Wrap(err, "parseVRFs: failed to parse vtysh response") + } + res := make([]string, 0) + for v := range vrfs { + res = append(res, v) + } + sort.Strings(res) + return res, nil +} diff --git a/internal/frr/parse_test.go b/internal/frr/parse_test.go new file mode 100644 index 00000000..66c99955 --- /dev/null +++ b/internal/frr/parse_test.go @@ -0,0 +1,784 @@ +// SPDX-License-Identifier:Apache-2.0 + +package frr + +import ( + "bytes" + "fmt" + "net" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" +) + +var expectedStats = MessageStats{ + OpensSent: 1, + OpensReceived: 2, + NotificationsSent: 3, + UpdatesSent: 4, + UpdatesReceived: 5, + KeepalivesSent: 6, + KeepalivesReceived: 7, + RouteRefreshSent: 8, + TotalSent: 9, + TotalReceived: 10, +} + +func TestNeighbour(t *testing.T) { + sample := `{ + "%s":{ + "remoteAs":%s, + "localAs":%s, + "nbrInternalLink":true, + "bgpVersion":4, + "remoteRouterId":"0.0.0.0", + "localRouterId":"172.18.0.5", + "bgpState":"%s", + "bgpTimerLastRead":253000, + "bgpTimerLastWrite":3405000, + "bgpInUpdateElapsedTimeMsecs":3405000, + "bgpTimerHoldTimeMsecs":180000, + "bgpTimerKeepAliveIntervalMsecs":60000, + "gracefulRestartInfo":{ + "endOfRibSend":{ + }, + "endOfRibRecv":{ + }, + "localGrMode":"Helper*", + "remoteGrMode":"NotApplicable", + "rBit":false, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":0 + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":2, + "notificationsSent":3, + "notificationsRecv":0, + "updatesSent":4, + "updatesRecv":5, + "keepalivesSent":6, + "keepalivesRecv":7, + "routeRefreshSent":8, + "routeRefreshRecv":0, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":9, + "totalRecv":10 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv4Unicast":{ + "routerAlwaysNextHop":true, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0, + "sentPrefixCounter":%d + }, + "ipv6Unicast":{ + "routerAlwaysNextHop":true, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0, + "sentPrefixCounter":%d + } + }, + "connectionsEstablished":0, + "connectionsDropped":0, + "lastResetTimerMsecs":253000, + "lastResetDueTo":"Waiting for peer OPEN", + "lastResetCode":32, + "portForeign":%d, + "connectRetryTimer":120, + "nextConnectTimerDueInMsecs":107000, + "readThread":"off", + "writeThread":"off" + } + }` + + tests := []struct { + name string + neighborIP string + remoteAS string + localAS string + status string + ipv4PrefixSent int + ipv6PrefixSent int + port int + expectedError string + }{ + { + "ipv4, connected", + "172.18.0.5", + "64512", + "64512", + "Established", + 1, + 0, + 179, + "", + }, + { + "ipv4, connected", + "172.18.0.5", + "64512", + "64512", + "Active", + 0, + 0, + 180, + "", + }, + { + "ipv6, connected", + "2620:52:0:1302::8af5", + "64512", + "64512", + "Established", + 2, + 1, + 181, + "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n, err := ParseNeighbour(fmt.Sprintf(sample, tt.neighborIP, tt.remoteAS, tt.localAS, tt.status, tt.ipv4PrefixSent, tt.ipv6PrefixSent, tt.port)) + if err != nil { + t.Fatal("Failed to parse ", err) + } + if !n.IP.Equal(net.ParseIP(tt.neighborIP)) { + t.Fatal("Expected neighbour ip", tt.neighborIP, "got", n.IP.String()) + } + if n.RemoteAS != tt.remoteAS { + t.Fatal("Expected remote as", tt.remoteAS, "got", n.RemoteAS) + } + if n.LocalAS != tt.localAS { + t.Fatal("Expected local as", tt.localAS, "got", n.LocalAS) + } + if tt.status == "Established" && n.Connected != true { + t.Fatal("Expected connected", true, "got", n.Connected) + } + if tt.status != "Established" && n.Connected == true { + t.Fatal("Expected connected", false, "got", n.Connected) + } + if tt.ipv4PrefixSent+tt.ipv6PrefixSent != n.PrefixSent { + t.Fatal("Expected prefix sent", tt.ipv4PrefixSent+tt.ipv6PrefixSent, "got", n.PrefixSent) + } + if tt.port != n.Port { + t.Fatal("Expected port", tt.port, "got", n.Port) + } + if n.RemoteRouterID != "0.0.0.0" { + t.Fatal("Expected remote routerid 0.0.0.0") + } + if !cmp.Equal(expectedStats, n.MsgStats) { + t.Fatal("unexpected BGP messages stats (-want +got)\n", cmp.Diff(expectedStats, n.MsgStats)) + } + }) + } +} + +const threeNeighbours = ` +{ + "172.18.0.2":{ + "remoteAs":64512, + "localAs":64512, + "nbrInternalLink":true, + "bgpVersion":4, + "remoteRouterId":"0.0.0.0", + "localRouterId":"172.18.0.5", + "bgpState":"Active", + "bgpTimerLastRead":14000, + "bgpTimerLastWrite":3166000, + "bgpInUpdateElapsedTimeMsecs":3166000, + "bgpTimerHoldTimeMsecs":180000, + "bgpTimerKeepAliveIntervalMsecs":60000, + "gracefulRestartInfo":{ + "endOfRibSend":{ + }, + "endOfRibRecv":{ + }, + "localGrMode":"Helper*", + "remoteGrMode":"NotApplicable", + "rBit":false, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":0 + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":2, + "notificationsSent":3, + "notificationsRecv":0, + "updatesSent":4, + "updatesRecv":5, + "keepalivesSent":6, + "keepalivesRecv":7, + "routeRefreshSent":8, + "routeRefreshRecv":0, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":9, + "totalRecv":10 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv4Unicast":{ + "routerAlwaysNextHop":true, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0 + } + }, + "connectionsEstablished":0, + "connectionsDropped":0, + "lastResetTimerMsecs":14000, + "lastResetDueTo":"Waiting for peer OPEN", + "lastResetCode":32, + "connectRetryTimer":120, + "nextConnectTimerDueInMsecs":107000, + "readThread":"off", + "writeThread":"off" + }, + "172.18.0.3":{ + "remoteAs":64512, + "localAs":64512, + "nbrInternalLink":true, + "bgpVersion":4, + "remoteRouterId":"0.0.0.0", + "localRouterId":"172.18.0.5", + "bgpState":"Active", + "bgpTimerLastRead":14000, + "bgpTimerLastWrite":3166000, + "bgpInUpdateElapsedTimeMsecs":3166000, + "bgpTimerHoldTimeMsecs":180000, + "bgpTimerKeepAliveIntervalMsecs":60000, + "gracefulRestartInfo":{ + "endOfRibSend":{ + }, + "endOfRibRecv":{ + }, + "localGrMode":"Helper*", + "remoteGrMode":"NotApplicable", + "rBit":false, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":0 + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":2, + "notificationsSent":3, + "notificationsRecv":0, + "updatesSent":4, + "updatesRecv":5, + "keepalivesSent":6, + "keepalivesRecv":7, + "routeRefreshSent":8, + "routeRefreshRecv":0, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":9, + "totalRecv":10 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv4Unicast":{ + "routerAlwaysNextHop":true, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0 + } + }, + "connectionsEstablished":0, + "connectionsDropped":0, + "lastResetTimerMsecs":14000, + "lastResetDueTo":"Waiting for peer OPEN", + "lastResetCode":32, + "connectRetryTimer":120, + "nextConnectTimerDueInMsecs":107000, + "readThread":"off", + "writeThread":"off" + }, + "172.18.0.4":{ + "remoteAs":64512, + "localAs":64512, + "nbrInternalLink":true, + "bgpVersion":4, + "remoteRouterId":"0.0.0.0", + "localRouterId":"172.18.0.5", + "bgpState":"Active", + "bgpTimerLastRead":14000, + "bgpTimerLastWrite":3166000, + "bgpInUpdateElapsedTimeMsecs":3166000, + "bgpTimerHoldTimeMsecs":180000, + "bgpTimerKeepAliveIntervalMsecs":60000, + "gracefulRestartInfo":{ + "endOfRibSend":{ + }, + "endOfRibRecv":{ + }, + "localGrMode":"Helper*", + "remoteGrMode":"NotApplicable", + "rBit":false, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":0 + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":2, + "notificationsSent":3, + "notificationsRecv":0, + "updatesSent":4, + "updatesRecv":5, + "keepalivesSent":6, + "keepalivesRecv":7, + "routeRefreshSent":8, + "routeRefreshRecv":0, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":9, + "totalRecv":10 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv4Unicast":{ + "routerAlwaysNextHop":true, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0 + } + }, + "connectionsEstablished":0, + "connectionsDropped":0, + "lastResetTimerMsecs":14000, + "lastResetDueTo":"Waiting for peer OPEN", + "lastResetCode":32, + "connectRetryTimer":120, + "nextConnectTimerDueInMsecs":107000, + "readThread":"off", + "writeThread":"off" + }, + "fc00:f853:ccd:e793::4":{ + "remoteAs":64512, + "localAs":64513, + "nbrExternalLink":true, + "hostname":"kind-control-plane", + "bgpVersion":4, + "remoteRouterId":"11.11.11.11", + "localRouterId":"172.18.0.5", + "bgpState":"Established", + "bgpTimerUpMsec":0, + "bgpTimerUpString":"00:00:00", + "bgpTimerUpEstablishedEpoch":1636386709, + "bgpTimerLastRead":4000, + "bgpTimerLastWrite":0, + "bgpInUpdateElapsedTimeMsecs":78272000, + "bgpTimerHoldTimeMsecs":90000, + "bgpTimerKeepAliveIntervalMsecs":30000, + "neighborCapabilities":{ + "4byteAs":"advertisedAndReceived", + "extendedMessage":"advertisedAndReceived", + "addPath":{ + "ipv6Unicast":{ + "rxAdvertisedAndReceived":true + } + }, + "routeRefresh":"advertisedAndReceivedOldNew", + "enhancedRouteRefresh":"advertisedAndReceived", + "multiprotocolExtensions":{ + "ipv4Unicast":{ + "received":true + }, + "ipv6Unicast":{ + "advertisedAndReceived":true + } + }, + "hostName":{ + "advHostName":"85e811e29230", + "advDomainName":"n\/a", + "rcvHostName":"kind-control-plane", + "rcvDomainName":"n\/a" + }, + "gracefulRestart":"advertisedAndReceived", + "gracefulRestartRemoteTimerMsecs":120000, + "addressFamiliesByPeer":"none" + }, + "gracefulRestartInfo":{ + "endOfRibSend":{ + "ipv6Unicast":true + }, + "endOfRibRecv":{ + }, + "localGrMode":"Helper*", + "remoteGrMode":"Helper", + "rBit":true, + "timers":{ + "configuredRestartTimer":120, + "receivedRestartTimer":120 + }, + "ipv6Unicast":{ + "fBit":false, + "endOfRibStatus":{ + "endOfRibSend":true, + "endOfRibSentAfterUpdate":true, + "endOfRibRecv":false + }, + "timers":{ + "stalePathTimer":360 + } + } + }, + "messageStats":{ + "depthInq":0, + "depthOutq":0, + "opensSent":1, + "opensRecv":2, + "notificationsSent":3, + "notificationsRecv":0, + "updatesSent":4, + "updatesRecv":5, + "keepalivesSent":6, + "keepalivesRecv":7, + "routeRefreshSent":8, + "routeRefreshRecv":0, + "capabilitySent":0, + "capabilityRecv":0, + "totalSent":9, + "totalRecv":10 + }, + "minBtwnAdvertisementRunsTimerMsecs":0, + "addressFamilyInfo":{ + "ipv6Unicast":{ + "updateGroupId":1, + "subGroupId":1, + "packetQueueLength":0, + "routerAlwaysNextHop":true, + "commAttriSentToNbr":"extendedAndStandard", + "acceptedPrefixCounter":0, + "sentPrefixCounter":0 + } + }, + "connectionsEstablished":1, + "connectionsDropped":0, + "lastResetTimerMsecs":4000, + "lastResetDueTo":"No AFI\/SAFI activated for peer", + "lastResetCode":30, + "hostLocal":"fc00:f853:ccd:e793::5", + "portLocal":180, + "hostForeign":"fc00:f853:ccd:e793::4", + "portForeign":53568, + "nexthop":"172.18.0.5", + "nexthopGlobal":"fc00:f853:ccd:e793::5", + "nexthopLocal":"fe80::42:acff:fe12:5", + "bgpConnection":"sharedNetwork", + "connectRetryTimer":120, + "authenticationEnabled":1, + "readThread":"on", + "writeThread":"on" + } +}` + +func TestNeighbours(t *testing.T) { + nn, err := ParseNeighbours(threeNeighbours) + if err != nil { + t.Fatalf("Failed to parse %s", err) + } + if len(nn) != 4 { + t.Fatalf("Expected 4 neighbours, got %d", len(nn)) + } + sort.Slice(nn, func(i, j int) bool { + return (bytes.Compare(nn[i].IP, nn[j].IP) < 0) + }) + + if !nn[0].IP.Equal(net.ParseIP("172.18.0.2")) { + t.Fatal("neighbour ip not matching") + } + if !nn[1].IP.Equal(net.ParseIP("172.18.0.3")) { + t.Fatal("neighbour ip not matching") + } + if !nn[2].IP.Equal(net.ParseIP("172.18.0.4")) { + t.Fatal("neighbour ip not matching") + } + + for i, n := range nn { + if !cmp.Equal(expectedStats, n.MsgStats) { + t.Fatal("unexpected BGP messages stats for neightbor", i, "(-want +got)\n", cmp.Diff(expectedStats, n.MsgStats)) + } + } +} + +const routes = `{ + "vrfId": 0, + "vrfName": "default", + "tableVersion": 7, + "routerId": "172.18.0.5", + "defaultLocPrf": 100, + "localAS": 64512, + "routes": { "192.168.10.0/32": [ + { + "valid":true, + "multipath":true, + "pathFrom":"internal", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "locPrf":0, + "weight":0, + "peerId":"172.18.0.4", + "path":"", + "origin":"incomplete", + "nexthops":[ + { + "ip":"172.18.0.4", + "afi":"ipv4", + "used":true + } + ] + }, + { + "valid":true, + "bestpath":true, + "pathFrom":"internal", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "locPrf":0, + "weight":0, + "peerId":"172.18.0.2", + "path":"", + "origin":"incomplete", + "nexthops":[ + { + "ip":"172.18.0.2", + "afi":"ipv4", + "used":true + } + ] + }, + { + "valid":true, + "multipath":true, + "pathFrom":"internal", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "locPrf":0, + "weight":0, + "peerId":"172.18.0.3", + "path":"", + "origin":"incomplete", + "nexthops":[ + { + "ip":"172.18.0.3", + "afi":"ipv4", + "used":true + } + ] + } + ] } }` + +func TestRoutes(t *testing.T) { + rr, err := ParseRoutes(routes) + if err != nil { + t.Fatalf("Failed to parse %s", err) + } + + ipRoutes, ok := rr["192.168.10.0"] + if !ok { + t.Fatalf("Routes for 192.168.10.0/32 not found") + } + + ips := make([]net.IP, 0) + ips = append(ips, ipRoutes.NextHops...) + + sort.Slice(ips, func(i, j int) bool { + return (bytes.Compare(ips[i], ips[j]) < 0) + }) + if !ips[0].Equal(net.ParseIP("172.18.0.2")) { + t.Fatal("neighbour ip not matching") + } + if !ips[1].Equal(net.ParseIP("172.18.0.3")) { + t.Fatal("neighbour ip not matching") + } + if !ips[2].Equal(net.ParseIP("172.18.0.4")) { + t.Fatal("neighbour ip not matching") + } +} + +const bfdPeers = `[ + { + "multihop":false, + "peer":"172.18.0.4", + "local":"172.18.0.5", + "vrf":"default", + "interface":"eth0", + "id":632314921, + "remote-id":2999817552, + "passive-mode":false, + "status":"up", + "uptime":52, + "diagnostic":"ok", + "remote-diagnostic":"ok", + "receive-interval":300, + "transmit-interval":300, + "echo-receive-interval":50, + "echo-transmit-interval":0, + "detect-multiplier":3, + "remote-receive-interval":300, + "remote-transmit-interval":300, + "remote-echo-receive-interval":50, + "remote-detect-multiplier":3 + }, + { + "multihop":false, + "peer":"172.18.0.2", + "local":"172.18.0.5", + "vrf":"default", + "interface":"eth0", + "id":3048501273, + "remote-id":2977557242, + "passive-mode":false, + "status":"up", + "uptime":52, + "diagnostic":"ok", + "remote-diagnostic":"ok", + "receive-interval":300, + "transmit-interval":300, + "echo-receive-interval":50, + "echo-transmit-interval":0, + "detect-multiplier":3, + "remote-receive-interval":300, + "remote-transmit-interval":300, + "remote-echo-receive-interval":50, + "remote-detect-multiplier":3 + }, + { + "multihop":false, + "peer":"172.18.0.3", + "local":"172.18.0.5", + "vrf":"default", + "interface":"eth0", + "id":2114932580, + "remote-id":493597049, + "passive-mode":false, + "status":"up", + "uptime":52, + "diagnostic":"ok", + "remote-diagnostic":"ok", + "receive-interval":300, + "transmit-interval":300, + "echo-receive-interval":50, + "echo-transmit-interval":0, + "detect-multiplier":3, + "remote-receive-interval":300, + "remote-transmit-interval":300, + "remote-echo-interval":50, + "remote-detect-multiplier":3 + } +]` + +func TestBFDPeers(t *testing.T) { + peers, err := ParseBFDPeers(bfdPeers) + if err != nil { + t.Fatalf("Failed to parse %s", err) + } + if len(peers) != 3 { + t.Fatal("Unexpected peer number", len(peers)) + } + if peers[2].Peer != "172.18.0.3" { + t.Fatal("Peer not found") + } + if peers[2].Status != "up" { + t.Fatal("wrong status") + } + if peers[2].RemoteEchoInterval != 50 { + t.Fatal("wrong echo interval") + } +} + +const vrfs = `{ +"default":{ + "vrfId": 0, + "vrfName": "default", + "tableVersion": 1, + "routerId": "172.18.0.3", + "defaultLocPrf": 100, + "localAS": 64512, + "routes": { "192.168.10.0/32": [ + { + "valid":true, + "bestpath":true, + "pathFrom":"external", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "metric":0, + "weight":32768, + "peerId":"(unspec)", + "path":"", + "origin":"IGP", + "nexthops":[ + { + "ip":"0.0.0.0", + "hostname":"kind-control-plane", + "afi":"ipv4", + "used":true + } + ] + } +] } } +, +"red":{ + "vrfId": 5, + "vrfName": "red", + "tableVersion": 1, + "routerId": "172.31.0.4", + "defaultLocPrf": 100, + "localAS": 64512, + "routes": { "192.168.10.0/32": [ + { + "valid":true, + "bestpath":true, + "pathFrom":"external", + "prefix":"192.168.10.0", + "prefixLen":32, + "network":"192.168.10.0\/32", + "metric":0, + "weight":32768, + "peerId":"(unspec)", + "path":"", + "origin":"IGP", + "nexthops":[ + { + "ip":"0.0.0.0", + "hostname":"kind-control-plane", + "afi":"ipv4", + "used":true + } + ] + } +] } } +}` + +func TestVRFs(t *testing.T) { + parsed, err := ParseVRFs(vrfs) + if err != nil { + t.Fatalf("Failed to parse %s", err) + } + expected := []string{"default", "red"} + if !cmp.Equal(parsed, expected) { + t.Fatalf("unexpected vrf list: %s", cmp.Diff(parsed, expected)) + } +} diff --git a/internal/frr/templates/bfdprofile.tmpl b/internal/frr/templates/bfdprofile.tmpl new file mode 100644 index 00000000..b208eb0b --- /dev/null +++ b/internal/frr/templates/bfdprofile.tmpl @@ -0,0 +1,24 @@ +{{- define "bfdprofile" }} + profile {{.profile.Name}} + {{ if .profile.ReceiveInterval -}} + receive-interval {{.profile.ReceiveInterval}} + {{end -}} + {{ if .profile.TransmitInterval -}} + transmit-interval {{.profile.TransmitInterval}} + {{end -}} + {{ if .profile.DetectMultiplier -}} + detect-multiplier {{.profile.DetectMultiplier}} + {{end -}} + {{ if .profile.EchoMode -}} + echo-mode + {{end -}} + {{ if .profile.EchoInterval -}} + echo-interval {{.profile.EchoInterval}} + {{end -}} + {{ if .profile.PassiveMode -}} + passive-mode + {{end -}} + {{ if .profile.MinimumTTL -}} + minimum-ttl {{ .profile.MinimumTTL }} + {{end -}} +{{- end -}} \ No newline at end of file diff --git a/internal/frr/templates/filters.tmpl b/internal/frr/templates/filters.tmpl new file mode 100644 index 00000000..7cf35a07 --- /dev/null +++ b/internal/frr/templates/filters.tmpl @@ -0,0 +1,51 @@ +{{- define "localpreffilter" -}} +{{frrIPFamily .advertisement.IPFamily}} prefix-list {{localPrefPrefixList .neighbor .advertisement.LocalPref}} permit {{.advertisement.Prefix}} +route-map {{.neighbor.ID}}-out permit {{counter .neighbor.ID}} + match {{frrIPFamily .advertisement.IPFamily}} address prefix-list {{localPrefPrefixList .neighbor .advertisement.LocalPref}} + set local-preference {{.advertisement.LocalPref}} + on-match next +{{- end -}} + +{{- define "communityfilter" -}} +{{frrIPFamily .advertisement.IPFamily}} prefix-list {{communityPrefixList .neighbor .community}} permit {{.advertisement.Prefix}} +route-map {{.neighbor.ID}}-out permit {{counter .neighbor.ID}} + match {{frrIPFamily .advertisement.IPFamily}} address prefix-list {{communityPrefixList .neighbor .community}} + set community {{.community}} additive + on-match next +{{- end -}} + +{{- /* The prefixes are per router in FRR, but MetalLB api allows to associate a given BGPAdvertisement to a service IP, + and a given advertisement contains both the properties of the announcement (i.e. community) and the list of peers + we may want to advertise to. Because of this, for each neighbor we must opt-in and allow the advertisement, and + deny all the others.*/ -}} +{{- define "neighborfilters" -}} + +route-map {{.neighbor.ID}}-in deny 20 +{{- range $a := .neighbor.Advertisements }} +{{/* Advertisements for which we must enable set the local pref */}} +{{- if not (eq $a.LocalPref 0)}} +{{template "localpreffilter" dict "advertisement" $a "neighbor" $.neighbor}} +{{- end -}} + +{{/* Advertisements for which we must enable the community property */}} +{{- range $c := $a.Communities }} +{{template "communityfilter" dict "advertisement" $a "neighbor" $.neighbor "community" $c}} +{{- end }} +{{/* this advertisement is allowed to the specific neighbor */}} +{{frrIPFamily $a.IPFamily}} prefix-list {{allowedPrefixList $.neighbor}} permit {{$a.Prefix}} +{{- end }} + +route-map {{$.neighbor.ID}}-out permit {{counter $.neighbor.ID}} + match ip address prefix-list {{allowedPrefixList $.neighbor}} +route-map {{$.neighbor.ID}}-out permit {{counter $.neighbor.ID}} + match ipv6 address prefix-list {{allowedPrefixList $.neighbor}} + +{{/* If the neighbor does not have an advertisement, we need to add a prefix to deny +for when we have a prefix but a given peer is not selected for any prefixes */}} +{{- if not .neighbor.HasV4Advertisements}} +ip prefix-list {{allowedPrefixList $.neighbor }} deny any +{{- end }} +{{- if not .neighbor.HasV6Advertisements}} +ipv6 prefix-list {{allowedPrefixList $.neighbor}} deny any +{{- end -}} +{{- end -}} diff --git a/internal/frr/templates/frr.tmpl b/internal/frr/templates/frr.tmpl new file mode 100644 index 00000000..f732de91 --- /dev/null +++ b/internal/frr/templates/frr.tmpl @@ -0,0 +1,70 @@ +log file /etc/frr/frr.log {{.Loglevel}} +log timestamp precision 3 +{{- if eq .Loglevel "debugging" }} +debug zebra events +debug zebra nht +debug zebra kernel +debug zebra rib +debug zebra nexthop +debug bgp neighbor-events +debug bgp updates +debug bgp keepalives +debug bgp nht +debug bgp zebra +debug bfd network +debug bfd peer +debug bfd zebra +{{- end }} +hostname {{.Hostname}} +ip nht resolve-via-default +ipv6 nht resolve-via-default + +{{- range $r := .Routers }} +{{- range .Neighbors }} +{{template "neighborfilters" dict "neighbor" . "router" $r}} +{{- end }} +{{- end }} + +{{range $r := .Routers -}} +router bgp {{$r.MyASN}}{{ if $r.VRF }} vrf {{$r.VRF}}{{end}} + no bgp ebgp-requires-policy + no bgp network import-check + no bgp default ipv4-unicast +{{ if $r.RouterID }} + bgp router-id {{$r.RouterID}} +{{- end }} + +{{- range .Neighbors }} +{{- template "neighborsession" dict "neighbor" . "routerASN" $r.MyASN -}} +{{- end }} + +{{- range $n := .Neighbors -}} +{{- template "neighborenableipfamily" . -}} +{{end -}} + +{{- if gt (len .IPV4Prefixes) 0}} + address-family ipv4 unicast +{{- range .IPV4Prefixes }} + network {{.}} +{{- end}} + exit-address-family +{{end }} + +{{- if gt (len .IPV6Prefixes) 0}} + address-family ipv6 unicast +{{- range .IPV6Prefixes }} + network {{.}} +{{- end}} + exit-address-family +{{end }} +{{end }} +{{- if gt (len .BFDProfiles) 0}} +bfd +{{- range .BFDProfiles }} +{{- template "bfdprofile" dict "profile" . -}} +{{- end }} +{{- end }} + +{{- if .ExtraConfig }} +{{ .ExtraConfig }} +{{- end }} diff --git a/internal/frr/templates/neighboripfamily.tmpl b/internal/frr/templates/neighboripfamily.tmpl new file mode 100644 index 00000000..7a56f800 --- /dev/null +++ b/internal/frr/templates/neighboripfamily.tmpl @@ -0,0 +1,13 @@ +{{- define "neighborenableipfamily"}} +{{/* no bgp default ipv4-unicast prevents peering if no address families are defined. We declare an ipv4 one for the peer to make the pairing happen */}} + address-family ipv4 unicast + neighbor {{.Addr}} activate + neighbor {{.Addr}} route-map {{.ID}}-in in + neighbor {{.Addr}} route-map {{.ID}}-out out + exit-address-family + address-family ipv6 unicast + neighbor {{.Addr}} activate + neighbor {{.Addr}} route-map {{.ID}}-in in + neighbor {{.Addr}} route-map {{.ID}}-out out + exit-address-family +{{- end -}} diff --git a/internal/frr/templates/neighborsession.tmpl b/internal/frr/templates/neighborsession.tmpl new file mode 100644 index 00000000..eea97592 --- /dev/null +++ b/internal/frr/templates/neighborsession.tmpl @@ -0,0 +1,22 @@ +{{- define "neighborsession"}} + neighbor {{.neighbor.Addr}} remote-as {{.neighbor.ASN}} + {{- if .neighbor.EBGPMultiHop }} + neighbor {{.neighbor.Addr}} ebgp-multihop + {{- end }} + {{ if .neighbor.Port -}} + neighbor {{.neighbor.Addr}} port {{.neighbor.Port}} + {{- end }} + neighbor {{.neighbor.Addr}} timers {{.neighbor.KeepaliveTime}} {{.neighbor.HoldTime}} + {{ if .neighbor.Password -}} + neighbor {{.neighbor.Addr}} password {{.neighbor.Password}} + {{- end }} + {{ if .neighbor.SrcAddr -}} + neighbor {{.neighbor.Addr}} update-source {{.neighbor.SrcAddr}} + {{- end }} +{{- if ne .neighbor.BFDProfile ""}} + neighbor {{.neighbor.Addr}} bfd profile {{.neighbor.BFDProfile}} +{{- end }} +{{- if mustDisableConnectedCheck .neighbor.IPFamily .routerASN .neighbor.ASN .neighbor.EBGPMultiHop }} + neighbor {{.neighbor.Addr}} disable-connected-check +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/internal/frr/testdata/.gitignore b/internal/frr/testdata/.gitignore new file mode 100644 index 00000000..64c24ad5 --- /dev/null +++ b/internal/frr/testdata/.gitignore @@ -0,0 +1,2 @@ +Test* +!*.golden diff --git a/internal/frr/testdata/TestDockerTestfails.golden b/internal/frr/testdata/TestDockerTestfails.golden new file mode 100644 index 00000000..fe80dfdf --- /dev/null +++ b/internal/frr/testdata/TestDockerTestfails.golden @@ -0,0 +1,20 @@ + +log stdout +hostname dummyhostname + +router bgp 100 + no bgp network import-check + no bgp default ipv4-unicast + + bgp router-id 10.1.1.254 + + neighbor 10.2.2.254 remote-as 200 + neighbor 10.2.2.254 port 179 + + address-family ipv4 unicast + neighbor 10.2.2.254 activate + network 172.16.1.10/24 + exit-address-family +wrong line + + diff --git a/internal/frr/testdata/TestSingleSession b/internal/frr/testdata/TestSingleSession new file mode 100644 index 00000000..8a068dbd --- /dev/null +++ b/internal/frr/testdata/TestSingleSession @@ -0,0 +1,44 @@ +log file /etc/frr/frr.log informational +log timestamp precision 3 +hostname dummyhostname +ip nht resolve-via-default +ipv6 nht resolve-via-default +route-map 192.168.1.2-in deny 20 + + +ip prefix-list 192.168.1.2-pl-ipv4 permit 192.169.1.0/24 + + +ip prefix-list 192.168.1.2-pl-ipv4 permit 192.170.1.0/22 + +route-map 192.168.1.2-out permit 1 + match ip address prefix-list 192.168.1.2-pl-ipv4 +route-map 192.168.1.2-out permit 2 + match ipv6 address prefix-list 192.168.1.2-pl-ipv4 + + +ip prefix-list 192.168.1.2-pl-ipv4 deny any +ipv6 prefix-list 192.168.1.2-pl-ipv4 deny any + +router bgp 65000 + no bgp ebgp-requires-policy + no bgp network import-check + no bgp default ipv4-unicast + + neighbor 192.168.1.2 remote-as 65001 + neighbor 192.168.1.2 port 4567 + neighbor 192.168.1.2 timers 0 0 + + + + address-family ipv4 unicast + neighbor 192.168.1.2 activate + neighbor 192.168.1.2 route-map 192.168.1.2-in in + neighbor 192.168.1.2 route-map 192.168.1.2-out out + exit-address-family + address-family ipv6 unicast + neighbor 192.168.1.2 activate + neighbor 192.168.1.2 route-map 192.168.1.2-in in + neighbor 192.168.1.2 route-map 192.168.1.2-out out + exit-address-family + diff --git a/internal/frr/testdata/TestSingleSession.golden b/internal/frr/testdata/TestSingleSession.golden new file mode 100644 index 00000000..8a068dbd --- /dev/null +++ b/internal/frr/testdata/TestSingleSession.golden @@ -0,0 +1,44 @@ +log file /etc/frr/frr.log informational +log timestamp precision 3 +hostname dummyhostname +ip nht resolve-via-default +ipv6 nht resolve-via-default +route-map 192.168.1.2-in deny 20 + + +ip prefix-list 192.168.1.2-pl-ipv4 permit 192.169.1.0/24 + + +ip prefix-list 192.168.1.2-pl-ipv4 permit 192.170.1.0/22 + +route-map 192.168.1.2-out permit 1 + match ip address prefix-list 192.168.1.2-pl-ipv4 +route-map 192.168.1.2-out permit 2 + match ipv6 address prefix-list 192.168.1.2-pl-ipv4 + + +ip prefix-list 192.168.1.2-pl-ipv4 deny any +ipv6 prefix-list 192.168.1.2-pl-ipv4 deny any + +router bgp 65000 + no bgp ebgp-requires-policy + no bgp network import-check + no bgp default ipv4-unicast + + neighbor 192.168.1.2 remote-as 65001 + neighbor 192.168.1.2 port 4567 + neighbor 192.168.1.2 timers 0 0 + + + + address-family ipv4 unicast + neighbor 192.168.1.2 activate + neighbor 192.168.1.2 route-map 192.168.1.2-in in + neighbor 192.168.1.2 route-map 192.168.1.2-out out + exit-address-family + address-family ipv6 unicast + neighbor 192.168.1.2 activate + neighbor 192.168.1.2 route-map 192.168.1.2-in in + neighbor 192.168.1.2 route-map 192.168.1.2-out out + exit-address-family + diff --git a/internal/frr/testdata/vtysh.conf b/internal/frr/testdata/vtysh.conf new file mode 100644 index 00000000..e4be3657 --- /dev/null +++ b/internal/frr/testdata/vtysh.conf @@ -0,0 +1,2 @@ +service integrated-vtysh-config + diff --git a/internal/ipfamily/ipfamily.go b/internal/ipfamily/ipfamily.go new file mode 100644 index 00000000..bb7f4daf --- /dev/null +++ b/internal/ipfamily/ipfamily.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier:Apache-2.0 + +package ipfamily // import "go.universe.tf/metallb/internal/ipfamily" + +import ( + "fmt" + "net" + + v1 "k8s.io/api/core/v1" +) + +// IP family helps identifying single stack IPv4/IPv6 vs Dual-stack ["IPv4", "IPv6"] or ["IPv6", "Ipv4"]. +type Family string + +const ( + IPv4 Family = "ipv4" + IPv6 Family = "ipv6" + DualStack Family = "dual" + Unknown Family = "unknown" +) + +// ForAddresses returns the address family given list of addresses strings. +func ForAddresses(ips ...string) (Family, error) { + switch len(ips) { + case 1: + ip := net.ParseIP(ips[0]) + res := ForAddress(ip) + return res, nil + case 2: + ip1 := net.ParseIP(ips[0]) + ip2 := net.ParseIP(ips[1]) + if ip1 == nil || ip2 == nil { + return Unknown, fmt.Errorf("IPFamilyForAddresses: Invalid address %q", ips) + } + if (ip1.To4() == nil) == (ip2.To4() == nil) { + return Unknown, fmt.Errorf("IPFamilyForAddresses: same address family %q", ips) + } + return DualStack, nil + default: + return Unknown, fmt.Errorf("IPFamilyForAddresses: invalid ips length %d %q", len(ips), ips) + } +} + +// ForAddressesIPs returns the address family from a given list of addresses IPs. +func ForAddressesIPs(ips []net.IP) (Family, error) { + ipsStrings := []string{} + + for _, ip := range ips { + ipsStrings = append(ipsStrings, ip.String()) + } + return ForAddresses(ipsStrings...) +} + +// ForCIDRString returns the address family from a given CIDR in string format. +func ForCIDRString(cidr string) Family { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return Unknown + } + if ip.To4() == nil { + return IPv6 + } + return IPv4 +} + +// ForCIDR returns the address family from a given CIDR. +func ForCIDR(cidr *net.IPNet) Family { + if cidr.IP.To4() == nil { + return IPv6 + } + return IPv4 +} + +// ForAddress returns the address family for a given address. +func ForAddress(ip net.IP) Family { + if ip.To4() == nil { + return IPv6 + } + return IPv4 +} + +// ForService returns the address family of a given service. +func ForService(svc *v1.Service) (Family, error) { + if len(svc.Spec.ClusterIPs) > 0 { + return ForAddresses(svc.Spec.ClusterIPs...) + } + // fallback to clusterip if clusterips are not set + addresses := []string{svc.Spec.ClusterIP} + return ForAddresses(addresses...) +} diff --git a/internal/ipfamily/ipfamily_test.go b/internal/ipfamily/ipfamily_test.go new file mode 100644 index 00000000..f48a80f9 --- /dev/null +++ b/internal/ipfamily/ipfamily_test.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier:Apache-2.0 + +package ipfamily + +import ( + "net" + "testing" +) + +func TestIPFamilyForAddresses(t *testing.T) { + tests := []struct { + desc string + ips []string + family Family + wantErr bool + }{ + { + desc: "ipv4 address", + ips: []string{"1.1.1.1"}, + family: IPv4, + }, + { + desc: "ipv6 address", + ips: []string{"100::1"}, + family: IPv6, + }, + { + desc: "ipv4 and ipv6 addresses", + ips: []string{"1.2.3.4", "100::1"}, + family: DualStack, + }, + { + desc: "dual stack with same address family", + ips: []string{"1.2.3.4", "5.6.7.8"}, + family: Unknown, + wantErr: true, + }, + { + desc: "dual stack with empty address", + ips: []string{"", ""}, + family: Unknown, + wantErr: true, + }, + { + desc: "more than 2 addresses", + ips: []string{"1.1.1.1", "100::1", "2.2.2.2"}, + family: Unknown, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + family, err := ForAddresses(test.ips...) + if test.wantErr && err == nil { + t.Fatalf("Expected error for %s", test.desc) + } + if !test.wantErr && err != nil { + t.Fatalf("Not expected error %s for %s", err, test.desc) + } + if family != test.family { + t.Fatalf("Incorrect IPFamily returned %s expected %s", family, test.family) + } + }) + } +} + +func TestIPFamilyForAddressesIPs(t *testing.T) { + tests := []struct { + desc string + ips []net.IP + family Family + wantErr bool + }{ + { + desc: "ipv4 address", + ips: []net.IP{net.ParseIP("1.2.4.0")}, + family: IPv4, + }, + { + desc: "ipv6 address", + ips: []net.IP{net.ParseIP("100::1")}, + family: IPv6, + }, + { + desc: "ipv4 and ipv6 addresses", + ips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("100::1")}, + family: DualStack, + }, + { + desc: "dual stack with same address family", + ips: []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("5.6.7.8")}, + family: Unknown, + wantErr: true, + }, + { + desc: "dual stack with empty address", + ips: []net.IP{net.ParseIP(""), net.ParseIP("")}, + family: Unknown, + wantErr: true, + }, + { + desc: "more than 2 addresses", + ips: []net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("100::1"), net.ParseIP("2.2.2.2")}, + family: Unknown, + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + family, err := ForAddressesIPs(test.ips) + if test.wantErr && err == nil { + t.Fatalf("Expected error for %s", test.desc) + } + if !test.wantErr && err != nil { + t.Fatalf("Not expected error %s for %s", err, test.desc) + } + if family != test.family { + t.Fatalf("Incorrect IPFamily returned %s expected %s", family, test.family) + } + }) + } +} + +func TestIPFamilyForCIDR(t *testing.T) { + tests := []struct { + desc string + cidr *net.IPNet + family Family + }{ + { + desc: "ipv4 cidr", + cidr: ipnet("1.2.3.4/30"), + family: IPv4, + }, + { + desc: "ipv6 cidr", + cidr: ipnet("100::/96"), + family: IPv6, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + family := ForCIDR(test.cidr) + if family != test.family { + t.Fatalf("Incorrect IPFamily returned %s expected %s", family, test.family) + } + }) + } +} + +func TestIPFamilyForAddress(t *testing.T) { + tests := []struct { + desc string + ip net.IP + family Family + }{ + { + desc: "ipv4 address", + ip: net.ParseIP("1.2.3.4"), + family: IPv4, + }, + { + desc: "ipv6 address", + ip: net.ParseIP("100::"), + family: IPv6, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + family := ForAddress(test.ip) + if family != test.family { + t.Fatalf("Incorrect IPFamily returned %s expected %s", family, test.family) + } + }) + } +} + +func ipnet(s string) *net.IPNet { + _, n, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + return n +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 00000000..f6b94c8c --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Package logging sets up structured logging in a uniform way, and +// redirects glog statements into the structured log. +package logging + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "k8s.io/klog" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +const ( + LevelAll = "all" + LevelDebug = "debug" + LevelInfo = "info" + LevelWarn = "warn" + LevelError = "error" + LevelNone = "none" +) + +type Level string +type levelSlice []Level + +var ( + // Levels returns an array of valid log levels. + Levels = levelSlice{LevelAll, LevelDebug, LevelInfo, LevelWarn, LevelError, LevelNone} +) + +func (l levelSlice) String() string { + strs := make([]string, len(l)) + for i, v := range l { + strs[i] = string(v) + } + return strings.Join(strs, ", ") +} + +// Init returns a logger configured with common settings like +// timestamping and source code locations. Both the stdlib logger and +// glog are reconfigured to push logs into this logger. +// +// Init must be called as early as possible in main(), before any +// application-specific flag parsing or logging occurs, because it +// mutates the contents of the flag package as well as os.Stderr. +func Init(lvl string) (log.Logger, error) { + l := log.NewJSONLogger(log.NewSyncWriter(os.Stdout)) + + r, w, err := os.Pipe() + if err != nil { + return nil, fmt.Errorf("creating pipe for glog redirection: %s", err) + } + klog.InitFlags(flag.NewFlagSet("klog", flag.ExitOnError)) + klog.SetOutput(w) + go collectGlogs(r, l) + + opt, err := parseLevel(lvl) + if err != nil { + return nil, err + } + + timeStampValuer := log.TimestampFormat(time.Now, time.RFC3339) + l = log.With(l, "ts", timeStampValuer) + l = level.NewFilter(l, opt) + + // Note: caller must be added after everything else that decorates the + // logger (otherwise we get incorrect caller reference). + l = log.With(l, "caller", log.DefaultCaller) + + // Setting a controller-runtime logger is required to + // get any log created by it. + ctrl.SetLogger(zap.New()) + + return l, nil +} + +func collectGlogs(f *os.File, logger log.Logger) { + defer func() { + if err := f.Close(); err != nil { + // cant log here, as this is the logger + errorString := fmt.Sprintf("Error closing file: %s", err) + panic(errorString) + } + }() + + r := bufio.NewReader(f) + for { + var buf []byte + l, pfx, err := r.ReadLine() + if err != nil { + // TODO: log + return + } + buf = append(buf, l...) + for pfx { + l, pfx, err = r.ReadLine() + if err != nil { + // TODO: log + return + } + buf = append(buf, l...) + } + + leveledLogger, ts, caller, msg := deformat(logger, buf) + leveledLogger.Log("ts", ts.Format(time.RFC3339Nano), "caller", caller, "msg", msg) + } +} + +var logPrefix = regexp.MustCompile(`^(.)(\d{2})(\d{2}) (\d{2}):(\d{2}):(\d{2}).(\d{6})\s+\d+ ([^:]+:\d+)] (.*)$`) + +func deformat(logger log.Logger, b []byte) (leveledLogger log.Logger, ts time.Time, caller, msg string) { + // Default deconstruction used when anything goes wrong. + leveledLogger = level.Info(logger) + ts = time.Now() + caller = "" + msg = string(b) + + if len(b) < 30 { + return + } + + ms := logPrefix.FindSubmatch(b) + if ms == nil { + return + } + + month, err := strconv.Atoi(string(ms[2])) + if err != nil { + return + } + day, err := strconv.Atoi(string(ms[3])) + if err != nil { + return + } + hour, err := strconv.Atoi(string(ms[4])) + if err != nil { + return + } + minute, err := strconv.Atoi(string(ms[5])) + if err != nil { + return + } + second, err := strconv.Atoi(string(ms[6])) + if err != nil { + return + } + micros, err := strconv.Atoi(string(ms[7])) + if err != nil { + return + } + ts = time.Date(ts.Year(), time.Month(month), day, hour, minute, second, micros*1000, time.Local).UTC() + + switch ms[1][0] { + case 'I': + leveledLogger = level.Info(logger) + case 'W': + leveledLogger = level.Warn(logger) + case 'E', 'F': + leveledLogger = level.Error(logger) + } + + caller = string(ms[8]) + msg = string(ms[9]) + + return +} + +func parseLevel(lvl string) (level.Option, error) { + switch lvl { + case LevelAll: + return level.AllowAll(), nil + case LevelDebug: + return level.AllowDebug(), nil + case LevelInfo: + return level.AllowInfo(), nil + case LevelWarn: + return level.AllowWarn(), nil + case LevelError: + return level.AllowError(), nil + case LevelNone: + return level.AllowNone(), nil + } + + return nil, fmt.Errorf("failed to parse log level: %s", lvl) +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 00000000..b9d5e708 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier:Apache-2.0 + +package version + +import ( + "fmt" + "runtime" +) + +var ( + version = "" // Filled out during release cutting + gitCommit string // Provided by ldflags during build + gitBranch string // Provided by ldflags during build +) + +// String returns a human-readable version string. +func String() string { + hasVersion := version != "" + hasBuildInfo := gitCommit != "" + + switch { + case hasVersion && hasBuildInfo: + return fmt.Sprintf("version %s (commit %s, branch %s)", version, gitCommit, gitBranch) + case !hasVersion && hasBuildInfo: + return fmt.Sprintf("(commit %s, branch %s)", gitCommit, gitBranch) + case hasVersion && !hasBuildInfo: + return fmt.Sprintf("version %s (no build information)", version) + default: + return "(no version or build info)" + } +} + +// Version returns the version string. +func Version() string { return version } + +// CommitHash returns the commit hash at which the binary was built. +func CommitHash() string { return gitCommit } + +// Branch returns the branch at which the binary was built. +func Branch() string { return gitBranch } + +// GoString returns the compiler, compiler version and architecture of the build. +func GoString() string { + return fmt.Sprintf("%s / %s / %s", runtime.Compiler, runtime.Version(), runtime.GOARCH) +}