Add kvisor agent debug command line tool

There is now a new kvisor agent debug CLI tool, that allows to dump the
current state of socket to process associations. The idea is that this
tool can be executed on the node to give a better insights into how
kvisor reports network related metrics.

Additionally the eBPF code has been refactored a bit, to better fit
having multiple C source locations in the repository.

Makefile

@@ -98,6 +98,7 @@ gen-args-types:
 .PHONY: gen-bpf
 gen-bpf:
 	go generate ./pkg/ebpftracer
+	go generate ./pkg/ebpftracer/debug
 
 .PHONY: gen-bpf-docker
 gen-bpf-docker: builder-image

cmd/agent/daemon/debug.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"fmt"
+	"net/netip"
+	"os"
+	"strconv"
+
+	"github.com/castai/kvisor/pkg/ebpftracer/debug"
+	"github.com/castai/kvisor/pkg/logging"
+	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/spf13/cobra"
+	"golang.org/x/sys/unix"
+)
+
+func NewDebugCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "debug",
+	}
+
+	cmd.AddCommand(newSocketDebugCommand())
+
+	return cmd
+}
+
+func newSocketDebugCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "sockets",
+	}
+
+	var (
+		targetPID = cmd.Flags().Uint32("kvisor-pid", 0, "PID of kvisor process to get data from")
+	)
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		log := logging.New(&logging.Config{})
+		d, err := debug.New(log, debug.DebugCfg{
+			TargetPID: *targetPID,
+		})
+		if err != nil {
+			return err
+		}
+
+		if err := d.Load(); err != nil {
+			return err
+		}
+
+		sockIter := d.SocketInfoIterator()
+
+		t := table.NewWriter()
+
+		t.AppendHeader(table.Row{"Process", "Process", "Process", "Process", "Socket", "Socket", "Socket", "Socket"}, table.RowConfig{AutoMerge: true})
+		t.AppendHeader(table.Row{"Cgroup", "PID", "Host PID", "Comm", "Protocol", "From", "To", "State", "Inode"})
+		t.SetAutoIndex(true)
+		t.SetStyle(table.StyleLight)
+		t.Style().Options.SeparateRows = true
+		t.SetColumnConfigs([]table.ColumnConfig{
+			{Number: 1, AutoMerge: true},
+			{Number: 2, AutoMerge: true},
+		})
+
+		displayWarning := false
+
+		for _, d := range sockIter.Iter() {
+			if d.Netctx.Taskctx.Pid == 0 {
+				displayWarning = true
+				continue
+			}
+
+			t.AppendRow(table.Row{
+				d.Netctx.Taskctx.CgroupId,
+				d.Netctx.Taskctx.Pid,
+				d.Netctx.Taskctx.HostPid,
+				toComm(d.Netctx.Taskctx.Comm[:]),
+				formatProto(d.SockInfo.Proto),
+				formatAddr(d.SockInfo.Tuple.Saddr.Raw, d.SockInfo.Tuple.Sport, d.SockInfo.Family),
+				formatAddr(d.SockInfo.Tuple.Daddr.Raw, d.SockInfo.Tuple.Dport, d.SockInfo.Family),
+				formatSockState(d.SockInfo.State, d.SockInfo.Proto),
+				d.SockInfo.Ino,
+			})
+		}
+
+		fmt.Println(t.Render())
+
+		if displayWarning {
+			fmt.Fprintln(os.Stderr, "")
+			fmt.Fprintln(os.Stderr, "warning: some sockets could not be associated with any processes and might be missing")
+		}
+
+		return nil
+	}
+
+	return cmd
+}
+
+func formatSockState(state, proto uint8) string {
+	if proto == unix.IPPROTO_TCP || proto == unix.IPPROTO_UDP {
+		switch state {
+		case 1:
+			return "ESTABLISHED"
+		case 2:
+			return "SYN_SENT"
+		case 3:
+			return "SYN_RECV"
+		case 4:
+			return "FIN_WAIT1"
+		case 5:
+			return "FIN_WAIT2"
+		case 6:
+			return "TIME_WAIT"
+		case 7:
+			return "CLOSE"
+		case 8:
+			return "CLOSE_WAIT"
+		case 9:
+			return "LAST_ACK"
+		case 10:
+			return "LISTEN"
+		case 11:
+			return "CLOSING"
+		case 12:
+			return "NEW_SYN_RECV"
+		case 13:
+			return "MAX_STATES"
+		}
+	}
+
+	return strconv.FormatUint(uint64(state), 10)
+}
+
+func formatProto(proto uint8) string {
+	switch proto {
+	case unix.IPPROTO_ICMP:
+		return "ICMP"
+	case unix.IPPROTO_ICMPV6:
+		return "ICMP6"
+	case unix.IPPROTO_TCP:
+		return "TCP"
+	case unix.IPPROTO_UDP:
+		return "UDP"
+	}
+
+	return strconv.FormatUint(uint64(proto), 10)
+}
+
+func toComm(data []int8) string {
+	result := make([]byte, len(data))
+	b := 0
+	for i, v := range data {
+		if v == 0 {
+			b = i
+			break
+		}
+		result[i] = byte(v)
+	}
+
+	return string(result[:b])
+}
+
+func formatAddr(addr [16]byte, port uint16, family uint16) string {
+	ip := formatIP(addr, family)
+	return fmt.Sprintf("%s:%d", ip, port)
+}
+
+func formatIP(addr [16]byte, family uint16) string {
+	switch family {
+	case unix.AF_INET:
+		addr, _ := netip.AddrFromSlice(addr[:4])
+		return addr.String()
+	case unix.AF_INET6:
+		addr := netip.AddrFrom16(addr)
+		return addr.String()
+	}
+
+	return "<unknown>"
+}

cmd/agent/daemon/main.go

@@ -19,6 +19,7 @@ func main() {
 	root.AddCommand(
 		NewRunCommand(Version),
 		NewClickhouseInitCommand(),
+    NewDebugCommand(),
 	)
 
 	if err := root.Execute(); err != nil {

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/grafana/pyroscope-go v1.2.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/iancoleman/strcase v0.3.0
+	github.com/jedib0t/go-pretty/v6 v6.6.1
 	github.com/json-iterator/go v1.1.12
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/labstack/echo/v4 v4.12.0

go.sum

@@ -857,6 +857,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jedib0t/go-pretty/v6 v6.6.1 h1:iJ65Xjb680rHcikRj6DSIbzCex2huitmc7bDtxYVWyc=
+github.com/jedib0t/go-pretty/v6 v6.6.1/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY=
 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=

pkg/ebpftracer/c/headers/common/cgroups.h

@@ -58,4 +58,10 @@ statfunc const u64 get_cgroup_v1_subsys0_id(struct task_struct *task)
     return get_cgroup_id(cgroup);
 }
 
+statfunc const u64 get_default_cgroup_id(struct task_struct *task)
+{
+    struct cgroup *cgroup = BPF_CORE_READ(task, cgroups, dfl_cgrp);
+    return get_cgroup_id(cgroup);
+}
+
 #endif