Add perf_event_open test (#1363)

Fix and an integration test for perf tracepoints blocking

Co-authored-by: Mauro Ezequiel Moltrasio <[email protected]>

integration-tests/container/QA_TAG

@@ -1 +1 @@
-1.1.0
+1.1.1

integration-tests/container/perf_event_open/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:jammy
+
+COPY sched_process_exit.c /sched_process_exit.c
+COPY entrypoint.sh /entrypoint.sh
+
+RUN apt update -y && apt install gcc -y && \
+    gcc sched_process_exit.c -o sched_process_exit
+
+ENTRYPOINT /entrypoint.sh

integration-tests/container/perf_event_open/Makefile

@@ -0,0 +1,24 @@
+BASE_PAT = .
+include ../Makefile-constants.mk
+
+.DEFAULT_GOAL = all
+
+COLLECTOR_QA_PERF_EVENT_OPEN := collector-perf-event-open
+
+ifneq ($(COLLECTOR_QA_TAG),)
+COLLECTOR_QA_PERF_EVENT_OPEN=collector-perf-event-open-$(COLLECTOR_QA_TAG)
+endif
+
+.PHONY: all
+all: build
+
+.PHONY: build
+build:
+	@docker buildx build --load --platform $(PLATFORM) \
+                -t quay.io/rhacs-eng/qa-multi-arch:$(COLLECTOR_QA_PERF_EVENT_OPEN) .
+
+.PHONY: build-and-push
+build-and-push:
+	@docker buildx build --push --platform $(PLATFORM) \
+		-t quay.io/rhacs-eng/qa-multi-arch:$(COLLECTOR_QA_PERF_EVENT_OPEN) .
+

integration-tests/container/perf_event_open/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Need debugfs to get the tracepoint id
+mount -t debugfs none /sys/kernel/debug
+
+# start workload to generate sched:sched_process_exit on cpu 0
+while true; do taskset -c 0 ls &> /dev/null; done &
+
+# start capturing
+export SCHED_PROCESS_EXIT_ID
+
+SCHED_PROCESS_EXIT_ID=$(cat /sys/kernel/debug/tracing/events/sched/sched_process_exit/id)
+/sched_process_exit 10 "${SCHED_PROCESS_EXIT_ID}"

integration-tests/container/perf_event_open/sched_process_exit.c

@@ -0,0 +1,64 @@
+// Loosely based on the example from man 2 perf_event_open
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <asm/unistd.h>
+#include <linux/perf_event.h>
+#include <sys/ioctl.h>
+
+static long
+perf_event_open(struct perf_event_attr* hw_event, pid_t pid,
+                int cpu, int group_fd, unsigned long flags) {
+  int ret;
+
+  ret = syscall(__NR_perf_event_open, hw_event, pid, cpu,
+                group_fd, flags);
+  return ret;
+}
+
+int main(int argc, char** argv) {
+  struct perf_event_attr pe;
+  long long count;
+  int fd;
+
+  // How long to wait for events, in seconds
+  int wait_interval = 10;
+
+  // /sys/kernel/debug/tracing/events/sched/sched_process_exit/id
+  int tracepoint_code = 310;
+
+  if (argc != 3) {
+    fprintf(stderr, "Expected 2 arguments, got: %d", argc - 1);
+    exit(EXIT_FAILURE);
+  }
+
+  wait_interval = atoi(argv[1]);
+  tracepoint_code = atoi(argv[2]);
+
+  memset(&pe, 0, sizeof(struct perf_event_attr));
+  pe.type = PERF_TYPE_TRACEPOINT;
+  pe.size = sizeof(struct perf_event_attr);
+  // /sys/kernel/debug/tracing/events/sched/sched_process_exit/id
+  pe.config = tracepoint_code;
+  pe.disabled = 1;
+
+  fd = perf_event_open(&pe, -1, 0, -1, PERF_FLAG_FD_CLOEXEC);
+  if (fd == -1) {
+    fprintf(stderr, "Error opening leader %llx\n", pe.config);
+    exit(EXIT_FAILURE);
+  }
+
+  ioctl(fd, PERF_EVENT_IOC_RESET, 0);
+  ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
+
+  sleep(wait_interval);
+  ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
+  read(fd, &count, sizeof(long long));
+
+  printf("%lld\n", count);
+
+  close(fd);
+}

integration-tests/images.yml

@@ -11,6 +11,7 @@ qa:
   qa-plop: quay.io/rhacs-eng/qa-multi-arch:collector-processes-listening-on-ports
   qa-schedule-curls: quay.io/rhacs-eng/qa-multi-arch:collector-schedule-curls
   qa-alpine-curl: quay.io/rhacs-eng/qa-multi-arch:alpine-curl
+  qa-perf-event-open: quay.io/rhacs-eng/qa-multi-arch:collector-perf-event-open
 
 non_qa:
   nginx: nginx:1.14-alpine

integration-tests/integration_test.go

@@ -459,3 +459,7 @@ func TestAsyncConnectionSuccessWithDisableTracking(t *testing.T) {
 func TestCollectorStartup(t *testing.T) {
 	suite.Run(t, new(suites.CollectorStartupTestSuite))
 }
+
+func TestPerfEvent(t *testing.T) {
+	suite.Run(t, new(suites.PerfEventOpenTestSuite))
+}

integration-tests/suites/perf_event_open.go

@@ -0,0 +1,49 @@
+package suites
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/stackrox/collector/integration-tests/suites/config"
+	"github.com/stretchr/testify/assert"
+)
+
+type PerfEventOpenTestSuite struct {
+	IntegrationTestSuiteBase
+}
+
+func (s *PerfEventOpenTestSuite) SetupSuite() {
+	s.RegisterCleanup("perf-event-open")
+	s.StartCollector(false, nil)
+}
+
+func (s *PerfEventOpenTestSuite) TearDownSuite() {
+	s.StopCollector()
+	s.cleanupContainers("perf-event-open")
+}
+
+// Verify that Collector probe doesn't block tracepoints
+func (s *PerfEventOpenTestSuite) TestReadingTracepoints() {
+	image := config.Images().QaImageByKey("qa-perf-event-open")
+	// attach to sched:sched_process_exit and count events
+	containerID, err := s.launchContainer("perf-event-open", "--privileged", image, "", "STDOUT")
+	s.Require().NoError(err)
+
+	if finished, _ := s.waitForContainerToExit("perf-event-open", containerID, 5*time.Second); finished {
+		logs, err := s.containerLogs("perf-event-open")
+		if err != nil {
+			fmt.Println(logs)
+			assert.FailNow(s.T(), "Failed to initialize host for performance testing")
+		}
+
+		count, err := strconv.Atoi(logs)
+		if err != nil {
+			fmt.Println(logs)
+			assert.FailNow(s.T(), "Cannot convert result to the integer type")
+		}
+
+		s.Assert().Greater(count, 0, "Number of captured tracepoint events is zero")
+	}
+	s.cleanupContainers(containerID)
+}

kernel-modules/MODULE_VERSION

@@ -1 +1 @@
-2.6.0
+2.7.0