tetragon: Add overhead metrics test for kprobe/uprobe/tracepoint

Adding test for overhead metrics on top of kprobe/uprobe/tracepoint
tracing policies.

Signed-off-by: Jiri Olsa <[email protected]>

pkg/sensors/tracing/overhead_test.go

@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/tetragon/pkg/arch"
+	"github.com/cilium/tetragon/pkg/observer/observertesthelper"
+	"github.com/cilium/tetragon/pkg/testutils"
+	tus "github.com/cilium/tetragon/pkg/testutils/sensors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+type metricResult struct {
+	label string
+	value int
+	ok    bool
+}
+
+func checkMetric(t *testing.T, serverURL, metric string, res []*metricResult) int {
+	resp, err := http.Get(serverURL + "/metrics")
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, resp.Body.Close())
+	}()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	tmp := fmt.Sprintf(`%s{(.+)} (\d+)`, metric)
+	re := regexp.MustCompile(tmp)
+
+	fmt.Printf("DEBUG body %s\n", string(body[:]))
+
+	matches := re.FindAllStringSubmatch(string(body), -1)
+
+	for _, v := range matches {
+		label := v[1]
+		value, _ := strconv.Atoi(v[2])
+
+		for _, r := range res {
+			if label == r.label && value >= r.value {
+				r.ok = true
+			}
+		}
+	}
+
+	for _, r := range res {
+		assert.NoError(t, err)
+		if !assert.Equal(t, true, r.ok) {
+			t.Logf("failed to match '%s'\n", r.label)
+		}
+	}
+
+	return 0
+}
+
+func testOverheadStats(t *testing.T, policy, testBin string, res []*metricResult) {
+	var doneWG, readyWG sync.WaitGroup
+	defer doneWG.Wait()
+
+	ctx, cancel := context.WithTimeout(context.Background(), tus.Conf().CmdWaitTime)
+	defer cancel()
+
+	// enable bpf stats
+	stats, err := ebpf.EnableStats(uint32(unix.BPF_STATS_RUN_TIME))
+	if err != nil {
+		t.Skip("stats not supported")
+	}
+	defer stats.Close()
+
+	createCrdFile(t, policy)
+
+	obs, err := observertesthelper.GetDefaultObserverWithFile(t, ctx, testConfigFile, tus.Conf().TetragonLib)
+	if err != nil {
+		t.Fatalf("GetDefaultObserverWithFile error: %s", err)
+	}
+	observertesthelper.LoopEvents(ctx, t, &doneWG, &readyWG, obs)
+	readyWG.Wait()
+
+	if err := exec.Command(testBin).Run(); err != nil {
+		fmt.Printf("Failed to execute test binary: %s\n", err)
+	}
+
+	checkMetric(t, "http://localhost:2112", "tetragon_overhead_cnt_program_total", res)
+	checkMetric(t, "http://localhost:2112", "tetragon_overhead_time_program_total", res)
+}
+
+func TestUprobeOverheadStats(t *testing.T) {
+	testNop := testutils.RepoRootPath("contrib/tester-progs/nop")
+
+	policy := `
+apiVersion: cilium.io/v1alpha1
+kind: TracingPolicy
+metadata:
+  name: "overhead"
+spec:
+  options:
+    - name: "disable-uprobe-multi"
+      value: "1"
+  uprobes:
+  - path: ` + testNop + `
+    symbols:
+    - "main"
+`
+	res := []*metricResult{
+		{label: fmt.Sprintf("attach=\"%s main\",policy=\"overhead\"", testNop), value: 1},
+		{label: "attach=\"sched/sched_process_exec\",policy=\"__base__\"", value: 1},
+		{label: "attach=\"wake_up_new_task\",policy=\"__base__\"", value: 1},
+	}
+
+	testOverheadStats(t, policy, testNop, res)
+}
+
+func TestKprobeOverheadStats(t *testing.T) {
+	testNop := testutils.RepoRootPath("contrib/tester-progs/nop")
+
+	policy := `
+apiVersion: cilium.io/v1alpha1
+kind: TracingPolicy
+metadata:
+  name: "overhead"
+spec:
+  options:
+    - name: "disable-kprobe-multi"
+      value: "1"
+  kprobes:
+  - call: "sys_read"
+    syscall: true
+    selectors:
+    - matchBinaries:
+      - operator: "In"
+        values:
+        - "` + testNop + `"
+`
+	syscall, _ := arch.AddSyscallPrefix("sys_read")
+	res := []*metricResult{
+		{label: fmt.Sprintf("attach=\"%s\",policy=\"overhead\"", syscall), value: 1},
+		{label: "attach=\"sched/sched_process_exec\",policy=\"__base__\"", value: 1},
+		{label: "attach=\"wake_up_new_task\",policy=\"__base__\"", value: 1},
+	}
+
+	testOverheadStats(t, policy, testNop, res)
+}
+
+func TestTracepointOverheadStats(t *testing.T) {
+	testNop := testutils.RepoRootPath("contrib/tester-progs/nop")
+
+	policy := `
+apiVersion: cilium.io/v1alpha1
+kind: TracingPolicy
+metadata:
+  name: "overhead"
+spec:
+  tracepoints:
+  - subsystem: "raw_syscalls"
+    event: "sys_enter"
+    args:
+    - index: 4
+      type: "syscall64"
+    - index: 5
+      type: "uint64"
+    selectors:
+    - matchBinaries:
+      - operator: "In"
+        values:
+        - "` + testNop + `"
+`
+
+	res := []*metricResult{
+		{label: "attach=\"raw_syscalls/sys_enter\",policy=\"overhead\"", value: 1},
+		{label: "attach=\"sched/sched_process_exec\",policy=\"__base__\"", value: 1},
+		{label: "attach=\"wake_up_new_task\",policy=\"__base__\"", value: 1},
+	}
+
+	testOverheadStats(t, policy, testNop, res)
+}