From 7619a8d92374b41156036cfab44016701aa8a4c2 Mon Sep 17 00:00:00 2001 From: Barun Acharya Date: Tue, 14 Nov 2023 17:12:32 +0530 Subject: [PATCH] feat(bpf,core,feeder,monitor): improve performance and owner only handling (#1491) --- KubeArmor/BPF/system_monitor.c | 19 +++++++++++---- KubeArmor/core/containerdHandler.go | 20 ---------------- KubeArmor/core/crioHandler.go | 3 --- KubeArmor/core/dockerHandler.go | 2 -- KubeArmor/feeder/feeder.go | 3 --- KubeArmor/feeder/policyMatcher.go | 8 +++---- KubeArmor/monitor/logUpdate.go | 4 +--- KubeArmor/monitor/systemMonitor.go | 31 ++++++++++++++++++------- KubeArmor/types/types.go | 6 +---- tests/k8s_env/syscalls/syscalls_test.go | 12 ++++------ 10 files changed, 49 insertions(+), 59 deletions(-) diff --git a/KubeArmor/BPF/system_monitor.c b/KubeArmor/BPF/system_monitor.c index 576953f3fd..1fa7541fb0 100644 --- a/KubeArmor/BPF/system_monitor.c +++ b/KubeArmor/BPF/system_monitor.c @@ -219,6 +219,7 @@ typedef struct __attribute__((__packed__)) sys_context char comm[TASK_COMM_LEN]; char cwd[CWD_LEN]; + u32 oid; // owner id } sys_context_t; #define BPF_MAP(_name, _type, _key_type, _value_type, _max_entries) \ @@ -1026,23 +1027,22 @@ static __always_inline u32 init_context(sys_context_t *context) bufs_t *string_p = get_buffer(CWD_BUF_TYPE); if (string_p == NULL) - return 0; + return 0; if (!prepend_path(&path, string_p, CWD_BUF_TYPE)) { - return 0; + return 0; } u32 *off = get_buffer_offset(CWD_BUF_TYPE); if (off == NULL) return 0; - bpf_probe_read_str(&context->cwd, CWD_LEN,(void *)&string_p->buf[*off]); + bpf_probe_read_str(&context->cwd, CWD_LEN, (void *)&string_p->buf[*off]); return 0; } - SEC("kprobe/security_path_mknod") int kprobe__security_path_mknod(struct pt_regs *ctx) { @@ -1456,6 +1456,17 @@ static __always_inline int trace_ret_generic(u32 id, struct pt_regs *ctx, u64 ty return 0; } + u64 pid_tgid = bpf_get_current_pid_tgid(); + + struct path *p = bpf_map_lookup_elem(&file_map, &pid_tgid); + if (p) + { + struct dentry *dent = READ_KERN(p->dentry); + struct inode *ino = READ_KERN(dent->d_inode); + kuid_t owner = READ_KERN(ino->i_uid); + context.oid = owner.val; + } + set_buffer_offset(DATA_BUF_TYPE, sizeof(sys_context_t)); bufs_t *bufs_p = get_buffer(DATA_BUF_TYPE); diff --git a/KubeArmor/core/containerdHandler.go b/KubeArmor/core/containerdHandler.go index aad8d0703f..c0886d635a 100644 --- a/KubeArmor/core/containerdHandler.go +++ b/KubeArmor/core/containerdHandler.go @@ -9,7 +9,6 @@ import ( "fmt" "os" "strconv" - "strings" "time" kl "github.com/kubearmor/KubeArmor/KubeArmor/common" @@ -60,9 +59,6 @@ type ContainerdHandler struct { containerd context.Context docker context.Context - // storage path - StoragePath string - // active containers containers map[string]context.Context } @@ -71,14 +67,6 @@ type ContainerdHandler struct { func NewContainerdHandler() *ContainerdHandler { ch := &ContainerdHandler{} - if strings.Contains(cfg.GlobalCfg.CRISocket, "microk8s") { // microk8s - ch.StoragePath = "/var/snap/microk8s/common/run/containerd" - } else if strings.Contains(cfg.GlobalCfg.CRISocket, "k3s") { // k3s - ch.StoragePath = "/run/k3s/containerd" - } else { // vanilla containerd - ch.StoragePath = "/run/containerd" - } - conn, err := grpc.Dial(cfg.GlobalCfg.CRISocket, grpc.WithInsecure()) if err != nil { return nil @@ -154,14 +142,6 @@ func (ch *ContainerdHandler) GetContainerInfo(ctx context.Context, containerID s spec := iface.(*specs.Spec) container.AppArmorProfile = spec.Process.ApparmorProfile - if spec.Root.Path == "rootfs" { // containerd - preMergedDir := ch.StoragePath + "/io.containerd.runtime.v2.task/k8s.io/" - postMergedDir := "/rootfs" - container.MergedDir = preMergedDir + container.ContainerID + postMergedDir - } else { // docker - container.MergedDir = spec.Root.Path - } - // == // taskReq := pt.ListPidsRequest{ContainerID: container.ContainerID} diff --git a/KubeArmor/core/crioHandler.go b/KubeArmor/core/crioHandler.go index 464e324f92..988bfdd1e1 100644 --- a/KubeArmor/core/crioHandler.go +++ b/KubeArmor/core/crioHandler.go @@ -120,9 +120,6 @@ func (ch *CrioHandler) GetContainerInfo(ctx context.Context, containerID string) // path to container's root storage container.AppArmorProfile = containerInfo.RuntimeSpec.Process.ApparmorProfile - // path to the rootfs - container.MergedDir = containerInfo.RuntimeSpec.Root.Path - pid := strconv.Itoa(containerInfo.Pid) if data, err := os.Readlink("/proc/" + pid + "/ns/pid"); err == nil { diff --git a/KubeArmor/core/dockerHandler.go b/KubeArmor/core/dockerHandler.go index 3c569aba97..0ffeea9e3b 100644 --- a/KubeArmor/core/dockerHandler.go +++ b/KubeArmor/core/dockerHandler.go @@ -116,8 +116,6 @@ func (dh *DockerHandler) GetContainerInfo(containerID string) (tp.Container, err container.AppArmorProfile = inspect.AppArmorProfile - container.MergedDir = inspect.GraphDriver.Data["MergedDir"] - // == // pid := strconv.Itoa(inspect.State.Pid) diff --git a/KubeArmor/feeder/feeder.go b/KubeArmor/feeder/feeder.go index d117248f28..6b57cdd8fd 100644 --- a/KubeArmor/feeder/feeder.go +++ b/KubeArmor/feeder/feeder.go @@ -596,9 +596,6 @@ func (fd *Feeder) PushLog(log tp.Log) { // set hostname log.HostName = cfg.GlobalCfg.Host - // remove MergedDir - log.MergedDir = "" - // remove flags log.PolicyEnabled = 0 log.ProcessVisibilityEnabled = false diff --git a/KubeArmor/feeder/policyMatcher.go b/KubeArmor/feeder/policyMatcher.go index a6f12df03e..91b3ee857e 100644 --- a/KubeArmor/feeder/policyMatcher.go +++ b/KubeArmor/feeder/policyMatcher.go @@ -1014,9 +1014,9 @@ func (fd *Feeder) UpdateMatchedPolicy(log tp.Log) tp.Log { matchedFlags := false - if secPolicy.ReadOnly && log.Resource != "" && secPolicy.OwnerOnly && log.MergedDir != "" { + if secPolicy.ReadOnly && log.Resource != "" && secPolicy.OwnerOnly { // read only && owner only - if strings.Contains(log.Data, "O_RDONLY") && strconv.Itoa(int(log.UID)) == getFileProcessUID(log.MergedDir+log.Resource) { + if strings.Contains(log.Data, "O_RDONLY") && log.UID == log.OID { matchedFlags = true } } else if secPolicy.ReadOnly && log.Resource != "" { @@ -1024,9 +1024,9 @@ func (fd *Feeder) UpdateMatchedPolicy(log tp.Log) tp.Log { if strings.Contains(log.Data, "O_RDONLY") { matchedFlags = true } - } else if secPolicy.OwnerOnly && log.MergedDir != "" { + } else if secPolicy.OwnerOnly { // owner only - if strconv.Itoa(int(log.UID)) == getFileProcessUID(log.MergedDir+log.Resource) { + if log.UID == log.OID { matchedFlags = true } } else { diff --git a/KubeArmor/monitor/logUpdate.go b/KubeArmor/monitor/logUpdate.go index 80942960ef..73ed451483 100644 --- a/KubeArmor/monitor/logUpdate.go +++ b/KubeArmor/monitor/logUpdate.go @@ -35,9 +35,6 @@ func (mon *SystemMonitor) UpdateContainerInfoByContainerID(log tp.Log) tp.Log { log.ContainerName = val.ContainerName log.ContainerImage = val.ContainerImage - // get merged directory - log.MergedDir = val.MergedDir - // update policy flag log.PolicyEnabled = val.PolicyEnabled @@ -88,6 +85,7 @@ func (mon *SystemMonitor) BuildLogBase(eventID int32, msg ContextCombined) tp.Lo } log.Cwd = strings.TrimRight(string(msg.ContextSys.Cwd[:]), "\x00") + "/" + log.OID = int32(msg.ContextSys.OID) log.ParentProcessName = mon.GetExecPath(msg.ContainerID, msg.ContextSys.HostPPID) log.ProcessName = mon.GetExecPath(msg.ContainerID, msg.ContextSys.HostPID) diff --git a/KubeArmor/monitor/systemMonitor.go b/KubeArmor/monitor/systemMonitor.go index 4ed945fd73..7c2a43ba31 100644 --- a/KubeArmor/monitor/systemMonitor.go +++ b/KubeArmor/monitor/systemMonitor.go @@ -82,6 +82,7 @@ type SyscallContext struct { Comm [16]byte Cwd [80]byte + OID uint32 } // ContextCombined Structure @@ -563,21 +564,35 @@ func (mon *SystemMonitor) TraceSyscall() { } now := time.Now() - if now.After(time.Unix(int64(ctx.Ts), 0).Add(5 * time.Second)) { + if now.After(time.Unix(int64(ctx.Ts), 0).Add(10 * time.Second)) { mon.Logger.Warn("Event dropped due to replay timeout") continue } // Best effort replay go func() { - time.Sleep(1 * time.Second) - select { - case mon.SyscallChannel <- dataRaw: - default: - // channel is full, wait for a short time before retrying - time.Sleep(1 * time.Second) - mon.Logger.Warn("Event droped due to busy event channel") + for i := 0; i < 10; i++ { + containerID := "" + + if ctx.PidID != 0 && ctx.MntID != 0 { + containerID = mon.LookupContainerID(ctx.PidID, ctx.MntID, ctx.HostPPID, ctx.HostPID) + + if containerID == "" { + time.Sleep(1 * time.Second) + continue + } + } + + select { + case mon.SyscallChannel <- dataRaw: + default: + // channel is full, wait for a short time before retrying + time.Sleep(1 * time.Second) + mon.Logger.Warn("Event droped due to busy event channel") + } + } + mon.Logger.Warn("Event dropped due to replay timeout") }() } }() diff --git a/KubeArmor/types/types.go b/KubeArmor/types/types.go index 3303a1f308..49f47c95e0 100644 --- a/KubeArmor/types/types.go +++ b/KubeArmor/types/types.go @@ -35,8 +35,6 @@ type Container struct { PidNS uint32 `json:"pidns"` MntNS uint32 `json:"mntns"` - MergedDir string `json:"mergedDir"` - // == // PolicyEnabled int `json:"policyEnabled"` @@ -202,9 +200,6 @@ type Log struct { ContainerName string `json:"containerName,omitempty"` ContainerImage string `json:"containerImage,omitempty"` - // container merged directory - MergedDir string `json:"mergedDir,omitempty"` - // common HostPPID int32 `json:"hostPPid"` HostPID int32 `json:"hostPid"` @@ -234,6 +229,7 @@ type Log struct { Operation string `json:"operation"` Resource string `json:"resource"` Cwd string `json:"cwd"` + OID int32 `json:"oid"` Data string `json:"data,omitempty"` Action string `json:"action,omitempty"` Result string `json:"result"` diff --git a/tests/k8s_env/syscalls/syscalls_test.go b/tests/k8s_env/syscalls/syscalls_test.go index 8b8e2cdfbe..63cf22c43a 100644 --- a/tests/k8s_env/syscalls/syscalls_test.go +++ b/tests/k8s_env/syscalls/syscalls_test.go @@ -5,7 +5,6 @@ package syscalls import ( "fmt" - "strings" "time" "github.com/kubearmor/KubeArmor/protobuf" @@ -482,9 +481,8 @@ var _ = Describe("Syscalls", func() { }) It("mount will be blocked by default for a pod", func() { - if strings.Contains(K8sRuntimeEnforcer(), "bpf") { - Skip("Skipping due to alerts being generated when something is blocked by kubearmor") - } + Skip("Skipping due to alerts only being generated when something is blocked by kubearmor") + // Start KubeArmor Logs err := KarmorLogStart("policy", "syscalls", "Syscall", ubuntu) Expect(err).To(BeNil()) @@ -511,9 +509,9 @@ var _ = Describe("Syscalls", func() { }) It("umount will be blocked by default for a pod as the capability not added", func() { - if strings.Contains(K8sRuntimeEnforcer(), "bpf") { - Skip("Skipping due to alerts being generated when something is blocked by kubearmor") - } + + Skip("Skipping due to alerts only being generated when something is blocked by kubearmor") + // Start KubeArmor Logs err := KarmorLogStart("policy", "syscalls", "Syscall", ubuntu) Expect(err).To(BeNil())