From 483b9e24f8be2afb99b6cf9e5634fe0bcf682800 Mon Sep 17 00:00:00 2001 From: Anastasios Papagiannis Date: Wed, 11 Dec 2024 10:38:24 +0200 Subject: [PATCH] Add tetra policyfilter listpolicies command It is useful to have a debug command to indentify which Kubernetes Identity Aware policies should be applied on a specific container. An example can be found here: Create a pod with "app: ubuntu" and "usage: dev" labels. $ cat << EOF | kubectl apply -f - apiVersion: v1 kind: Pod metadata: name: ubuntu labels: app: ubuntu usage: dev spec: containers: - name: ubuntu image: ubuntu:24.10 command: ["/bin/sleep", "3650d"] imagePullPolicy: IfNotPresent restartPolicy: Always EOF And apply several policies where some of them match while others don't. $ cat << EOF | kubectl apply -f - apiVersion: cilium.io/v1alpha1 kind: TracingPolicy metadata: name: "lseek-podfilter-app" spec: podSelector: matchLabels: app: "ubuntu" kprobes: [...] EOF $ cat << EOF | kubectl apply -f - apiVersion: cilium.io/v1alpha1 kind: TracingPolicy metadata: name: "lseek-podfilter-usage" spec: podSelector: matchLabels: usage: "dev" kprobes: [...] EOF $ cat << EOF | kubectl apply -f - apiVersion: cilium.io/v1alpha1 kind: TracingPolicy metadata: name: "lseek-podfilter-prod" spec: podSelector: matchLabels: prod: "true" kprobes: [...] EOF $ cat << EOF | kubectl apply -f - apiVersion: cilium.io/v1alpha1 kind: TracingPolicy metadata: name: "lseek-podfilter-info" spec: podSelector: matchLabels: info: "broken" kprobes: [...] EOF $ cat << EOF | kubectl apply -f - apiVersion: cilium.io/v1alpha1 kind: TracingPolicy metadata: name: "lseek-podfilter-global" spec: kprobes: [...] EOF Based on the labels we expect that policies lseek-podfilter-app and lseek-podfilter-usage to match on that pod. lseek-podfilter-global is not a Kubernetes Identity Aware policy so this will be applied in all cases and we do not report that. First step is to find the container ID that we care about. $ kubectl describe pod/ubuntu | grep containerd Container ID: containerd://ff433e9e16467787a60ac853d9b313150091968731f620776d6d7c514b1e8d6c And then use it to report all Kubernetes Identity Aware policies that match. $ kubectl exec -it ds/tetragon -n kube-system -c tetragon -- tetra policyfilter -r "unix:///procRoot/1/root/run/containerd/containerd.sock" listpolicies ff433e9e16467787a60ac853d9b313150091968731f620776d6d7c514b1e8d6c ID NAME STATE FILTERID NAMESPACE SENSORS KERNELMEMORY 5 lseek-podfilter-usage enabled 5 (global) generic_kprobe 1.72 MB 1 lseek-podfilter-app enabled 1 (global) generic_kprobe 1.72 MB We also provide --debug flag to provide more details i.e.: $ kubectl exec -it ds/tetragon -n kube-system -c tetragon -- tetra policyfilter -r "unix:///procRoot/1/root/run/containerd/containerd.sock" listpolicies ff433e9e16467787a60ac853d9b313150091968731f620776d6d7c514b1e8d6c --debug time="2024-12-13T09:47:38Z" level=info msg=cgroup path=/run/tetragon/cgroup2/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod189a8053_9f36_4250_bcae_9ed167172920.slice/cri-containerd-ff433e9e16467787a60ac853d9b313150091968731f620776d6d7c514b1e8d6c.scope time="2024-12-13T09:47:38Z" level=info msg=cgroup id=5695 time="2024-12-13T09:47:39Z" level=debug msg="resolved server address using info file" InitInfoFile=/var/run/tetragon/tetragon-info.json ServerAddress="localhost:54321" ID NAME STATE FILTERID NAMESPACE SENSORS KERNELMEMORY 1 lseek-podfilter-app enabled 1 (global) generic_kprobe 1.72 MB 5 lseek-podfilter-usage enabled 5 (global) generic_kprobe 1.72 MB This uses the cgroup-based policy filter map that introduced in a previous commit that maps cgroupIds to policyIds. Signed-off-by: Anastasios Papagiannis --- cmd/tetra/common/utils.go | 63 ++++++++++++++++- cmd/tetra/policyfilter/policyfilter.go | 88 ++++++++++++++++++++++++ cmd/tetra/tracingpolicy/tracingpolicy.go | 50 +------------- 3 files changed, 151 insertions(+), 50 deletions(-) diff --git a/cmd/tetra/common/utils.go b/cmd/tetra/common/utils.go index 6e6ad47b8ac..0abf1a1a109 100644 --- a/cmd/tetra/common/utils.go +++ b/cmd/tetra/common/utils.go @@ -3,7 +3,14 @@ package common -import "fmt" +import ( + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/cilium/tetragon/api/v1/tetragon" +) // HumanizeByteCount transforms bytes count into a quickly-readable version, for // example it transforms 4458824 into "4.46 MB". I copied this code from @@ -21,3 +28,57 @@ func HumanizeByteCount(b int) string { return fmt.Sprintf("%.2f %cB", float64(b)/float64(div), "kMGTPE"[exp]) } + +func PrintTracingPolicies(output io.Writer, policies []*tetragon.TracingPolicyStatus, skipPolicy func(pol *tetragon.TracingPolicyStatus) bool) { + // tabwriter config imitates kubectl default output, i.e. 3 spaces padding + w := tabwriter.NewWriter(output, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "ID\tNAME\tSTATE\tFILTERID\tNAMESPACE\tSENSORS\tKERNELMEMORY") + + for _, pol := range policies { + if skipPolicy != nil && skipPolicy(pol) { + continue + } + + namespace := pol.Namespace + if namespace == "" { + namespace = "(global)" + } + + sensors := strings.Join(pol.Sensors, ",") + + // From v0.11 and before, enabled, filterID and error were + // bundled in a string. To have a retro-compatible tetra + // command, we scan the string. If the scan fails, it means + // something else might be in Info and we print it. + // + // we can drop the following block (and comment) when we + // feel tetra should support only version after v0.11 + if pol.Info != "" { + var parsedEnabled bool + var parsedFilterID uint64 + var parsedError string + var parsedName string + str := strings.NewReader(pol.Info) + _, err := fmt.Fscanf(str, "%253s enabled:%t filterID:%d error:%512s", &parsedName, &parsedEnabled, &parsedFilterID, &parsedError) + if err == nil { + if parsedEnabled { + pol.State = tetragon.TracingPolicyState_TP_STATE_ENABLED + } + pol.FilterId = parsedFilterID + pol.Error = parsedError + pol.Info = "" + } + } + + fmt.Fprintf(w, "%d\t%s\t%s\t%d\t%s\t%s\t%s\t\n", + pol.Id, + pol.Name, + strings.TrimPrefix(strings.ToLower(pol.State.String()), "tp_state_"), + pol.FilterId, + namespace, + sensors, + HumanizeByteCount(int(pol.KernelMemoryBytes)), + ) + } + w.Flush() +} diff --git a/cmd/tetra/policyfilter/policyfilter.go b/cmd/tetra/policyfilter/policyfilter.go index 6c4790e35ec..8ad329380c5 100644 --- a/cmd/tetra/policyfilter/policyfilter.go +++ b/cmd/tetra/policyfilter/policyfilter.go @@ -4,12 +4,18 @@ package policyfilter import ( + "context" "fmt" + "os" + "path" "path/filepath" "strconv" + "github.com/cilium/tetragon/api/v1/tetragon" + "github.com/cilium/tetragon/cmd/tetra/common" "github.com/cilium/tetragon/cmd/tetra/debug" "github.com/cilium/tetragon/pkg/cgroups" + "github.com/cilium/tetragon/pkg/cri" "github.com/cilium/tetragon/pkg/defaults" "github.com/cilium/tetragon/pkg/logger" "github.com/cilium/tetragon/pkg/policyfilter" @@ -29,6 +35,7 @@ func New() *cobra.Command { addCommand(), cgroupGetIDCommand(), dumpDebugCmd(), + listPoliciesForContainer(), ) return ret @@ -138,3 +145,84 @@ func addCgroup(fname string, polID policyfilter.PolicyID, cgID policyfilter.Cgro } } + +func listPoliciesForContainer() *cobra.Command { + var endpoint, cgroupMnt string + mapFname := filepath.Join(defaults.DefaultMapRoot, defaults.DefaultMapPrefix, policyfilter.MapName) + ret := &cobra.Command{ + Use: "listpolicies [container id]", + Short: "list all Kubernetes Identity Aware policies that apply to a specific container", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + ctx := context.Background() + client, err := cri.NewClient(ctx, endpoint) + if err != nil { + return err + } + + cgroupPath, err := cri.CgroupPath(ctx, client, args[0]) + if err != nil { + return err + } + + if cgroupMnt == "" { + cgroupMnt = defaults.Cgroup2Dir + } + fullCgroupPath := path.Join(cgroupMnt, cgroupPath) + if common.Debug { + logger.GetLogger().WithField("path", fullCgroupPath).Info("cgroup") + } + + cgID, err := cgroups.GetCgroupIdFromPath(fullCgroupPath) + if err != nil { + logger.GetLogger().WithError(err).Fatal("Failed to parse cgroup") + } + + if common.Debug { + logger.GetLogger().WithField("id", cgID).Info("cgroup") + } + + m, err := policyfilter.OpenMap(mapFname) + if err != nil { + logger.GetLogger().WithError(err).Fatal("Failed to open policyfilter map") + return err + } + defer m.Close() + + data, err := m.Dump() + if err != nil { + logger.GetLogger().WithError(err).Fatal("Failed to open policyfilter map") + return err + } + + policyIds, ok := data.Cgroup[policyfilter.CgroupID(cgID)] + if !ok { + return nil + } + + c, err := common.NewClientWithDefaultContextAndAddress() + if err != nil { + return fmt.Errorf("failed create gRPC client: %w", err) + } + defer c.Close() + + res, err := c.Client.ListTracingPolicies(c.Ctx, &tetragon.ListTracingPoliciesRequest{}) + if err != nil || res == nil { + return fmt.Errorf("failed to list tracing policies: %w", err) + } + + common.PrintTracingPolicies(os.Stdout, res.Policies, func(pol *tetragon.TracingPolicyStatus) bool { + _, ok := policyIds[policyfilter.PolicyID(pol.FilterId)] + return !ok + }) + + return nil + }, + } + + flags := ret.Flags() + flags.StringVarP(&endpoint, "runtime-endpoint", "r", "", "CRI endpoint") + flags.StringVar(&mapFname, "map-fname", mapFname, "policyfilter map filename") + flags.StringVar(&cgroupMnt, "cgroup-mount", cgroupMnt, "cgroupFS mount point") + return ret +} diff --git a/cmd/tetra/tracingpolicy/tracingpolicy.go b/cmd/tetra/tracingpolicy/tracingpolicy.go index 380d3735535..2c7024a79c9 100644 --- a/cmd/tetra/tracingpolicy/tracingpolicy.go +++ b/cmd/tetra/tracingpolicy/tracingpolicy.go @@ -6,8 +6,6 @@ package tracingpolicy import ( "fmt" "os" - "strings" - "text/tabwriter" "github.com/cilium/tetragon/api/v1/tetragon" "github.com/cilium/tetragon/cmd/tetra/common" @@ -161,53 +159,7 @@ func New() *cobra.Command { } cmd.Println(string(b)) case "text": - // tabwriter config imitates kubectl default output, i.e. 3 spaces padding - w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ID\tNAME\tSTATE\tFILTERID\tNAMESPACE\tSENSORS\tKERNELMEMORY") - - for _, pol := range res.Policies { - namespace := pol.Namespace - if namespace == "" { - namespace = "(global)" - } - - sensors := strings.Join(pol.Sensors, ",") - - // From v0.11 and before, enabled, filterID and error were - // bundled in a string. To have a retro-compatible tetra - // command, we scan the string. If the scan fails, it means - // something else might be in Info and we print it. - // - // we can drop the following block (and comment) when we - // feel tetra should support only version after v0.11 - if pol.Info != "" { - var parsedEnabled bool - var parsedFilterID uint64 - var parsedError string - var parsedName string - str := strings.NewReader(pol.Info) - _, err := fmt.Fscanf(str, "%253s enabled:%t filterID:%d error:%512s", &parsedName, &parsedEnabled, &parsedFilterID, &parsedError) - if err == nil { - if parsedEnabled { - pol.State = tetragon.TracingPolicyState_TP_STATE_ENABLED - } - pol.FilterId = parsedFilterID - pol.Error = parsedError - pol.Info = "" - } - } - - fmt.Fprintf(w, "%d\t%s\t%s\t%d\t%s\t%s\t%s\t\n", - pol.Id, - pol.Name, - strings.TrimPrefix(strings.ToLower(pol.State.String()), "tp_state_"), - pol.FilterId, - namespace, - sensors, - common.HumanizeByteCount(int(pol.KernelMemoryBytes)), - ) - } - w.Flush() + common.PrintTracingPolicies(cmd.OutOrStdout(), res.Policies, nil) } return nil