Proposal: support for third party OSS to use otel-ebpf-profiler

.gitignore

@@ -3,5 +3,7 @@
 .cache
 /.idea
 /go
+.vscode
 ebpf-profiler
 ci-kernels
+ebpf-profiler

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "typeinfo": "cpp"
+    }
+}

Dockerfile

@@ -44,4 +44,6 @@ RUN
 # Append to /etc/profile for login shells
 RUN echo 'export PATH="/usr/local/go/bin:$PATH"' >> /etc/profile
 
+RUN apt-get install -y libbpf-dev
+
 ENTRYPOINT ["/bin/bash", "-l", "-c"]

Makefile

@@ -132,4 +132,4 @@ legal:
 	@./legal/add-non-go.sh legal/non-go-dependencies.json LICENSES
 
 codespell:
-	@codespell
+	@codespell

host/host.go

@@ -46,6 +46,30 @@ type Frame struct {
 	ReturnAddress bool
 }
 
+type Regs struct {
+	R15    uint64
+	R14    uint64
+	R13    uint64
+	R12    uint64
+	Bp     uint64
+	Bx     uint64
+	R11    uint64
+	R10    uint64
+	R9     uint64
+	R8     uint64
+	Ax     uint64
+	Cx     uint64
+	Dx     uint64
+	Si     uint64
+	Di     uint64
+	OrigAx uint64
+	Ip     uint64
+	Cs     uint64
+	Flags  uint64
+	Sp     uint64
+	Ss     uint64
+}
+
 type Trace struct {
 	Comm             string
 	Frames           []Frame
@@ -55,4 +79,5 @@ type Trace struct {
 	TID              libpf.PID
 	APMTraceID       libpf.APMTraceID
 	APMTransactionID libpf.APMTransactionID
+	Registers        Regs
 }

reporter/iface.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"time"
 
+	"go.opentelemetry.io/ebpf-profiler/host"
 	"go.opentelemetry.io/ebpf-profiler/libpf"
 	"go.opentelemetry.io/ebpf-profiler/process"
 )
@@ -29,6 +30,7 @@ type TraceEventMeta struct {
 	Comm           string
 	APMServiceName string
 	PID, TID       libpf.PID
+	Registers      host.Regs
 }
 
 type TraceReporter interface {

support/ebpf/Makefile

@@ -87,4 +87,4 @@ bloatcheck: $(TRACER_NAME)
 	python3 bloat-o-meter $(TRACER_NAME).baseline $(TRACER_NAME)
 
 clean:
-	rm -f *.o
+	rm -f *.o

support/ebpf/dotnet_tracer.ebpf.c

@@ -8,6 +8,7 @@
 #include "bpfdefs.h"
 #include "tracemgmt.h"
 #include "types.h"
+#include "helpers.h"
 
 // The number of dotnet frames to unwind per frame-unwinding eBPF program.
 #define DOTNET_FRAMES_PER_PROGRAM   5
@@ -244,8 +245,9 @@ ErrorCode unwind_one_dotnet_frame(PerCPURecord *record, DotnetProcInfo *vi, bool
 // unwind_dotnet is the entry point for tracing when invoked from the native tracer
 // or interpreter dispatcher. It does not reset the trace object and will append the
 // dotnet stack frames to the trace object for the current CPU.
-SEC("perf_event/unwind_dotnet")
-int unwind_dotnet(struct pt_regs *ctx) {
+static inline __attribute__((__always_inline__))
+int unwind_dotnet(struct pt_regs *ctx, bpf_map_def *prog_map)
+{
   PerCPURecord *record = get_per_cpu_record();
   if (!record) {
     return -1;
@@ -285,7 +287,9 @@ int unwind_dotnet(struct pt_regs *ctx) {
 
 exit:
   record->state.unwind_error = error;
-  tail_call(ctx, unwinder);
+  tail_call(ctx, unwinder, prog_map);
   DEBUG_PRINT("dotnet: tail call for next frame unwinder (%d) failed", unwinder);
   return -1;
 }
+
+DEFINE_DUAL_PROGRAM(unwind_dotnet, unwind_dotnet);

support/ebpf/extmaps.h

@@ -6,7 +6,8 @@
 #include "bpf_map.h"
 
 // References to map definitions in *.ebpf.c.
-extern bpf_map_def progs;
+extern bpf_map_def perf_progs;
+extern bpf_map_def kprobe_progs;
 extern bpf_map_def per_cpu_records;
 extern bpf_map_def pid_page_to_mapping_info;
 extern bpf_map_def metrics;

support/ebpf/helpers.h

@@ -0,0 +1,20 @@
+// Provide helpers for the eBPF code.
+
+#ifndef OPTI_HELPERS_H
+#define OPTI_HELPERS_H
+
+// Macros for BPF program type and context handling.
+#define DEFINE_DUAL_PROGRAM(name, func)                    \
+SEC("perf_event/" #name)                                         \
+int name##_perf(struct pt_regs *ctx)                         \
+{                                                                        \
+    return func(ctx, &perf_progs);                            \
+}                                                                        \
+                                                                         \
+SEC("kprobe/" #name)                                             \
+int name##_kprobe(struct pt_regs *ctx)                                   \
+{                                                                        \
+    return func(ctx, &kprobe_progs);                                                    \
+}
+
+#endif // OPTI_HELPERS_H

support/ebpf/hotspot_tracer.ebpf.c

@@ -8,6 +8,7 @@
 #include "tracemgmt.h"
 #include "types.h"
 #include "errors.h"
+#include "helpers.h"
 
 // Information extracted from a JDK `CodeBlob` instance.
 typedef struct CodeBlobInfo {
@@ -890,8 +891,9 @@ static ErrorCode hotspot_unwind_one_frame(PerCPURecord *record, HotspotProcInfo
 // unwind_hotspot is the entry point for tracing when invoked from the native tracer
 // and it recursive unwinds all HotSpot frames and then jumps back to unwind further
 // native frames that follow.
-SEC("perf_event/unwind_hotspot")
-int unwind_hotspot(struct pt_regs *ctx) {
+static inline __attribute__((__always_inline__))
+int unwind_hotspot(struct pt_regs *ctx, bpf_map_def *prog_map)
+{
   PerCPURecord *record = get_per_cpu_record();
   if (!record)
     return -1;
@@ -923,7 +925,9 @@ int unwind_hotspot(struct pt_regs *ctx) {
   }
 
   record->state.unwind_error = error;
-  tail_call(ctx, unwinder);
+  tail_call(ctx, unwinder, prog_map);
   DEBUG_PRINT("jvm: tail call for next frame unwinder (%d) failed", unwinder);
   return -1;
 }
+
+DEFINE_DUAL_PROGRAM(unwind_hotspot, unwind_hotspot);

support/ebpf/interpreter_dispatcher.ebpf.c

@@ -6,6 +6,7 @@
 #include "types.h"
 #include "tracemgmt.h"
 #include "tsd.h"
+#include "helpers.h"
 
 // Begin shared maps
 
@@ -25,8 +26,16 @@ bpf_map_def SEC("maps") metrics = {
   .max_entries = metricID_Max,
 };
 
-// progs maps from a program ID to an eBPF program
-bpf_map_def SEC("maps") progs = {
+// perf progs maps from a program ID to an eBPF program
+bpf_map_def SEC("maps") perf_progs = {
+  .type = BPF_MAP_TYPE_PROG_ARRAY,
+  .key_size = sizeof(u32),
+  .value_size = sizeof(u32),
+  .max_entries = NUM_TRACER_PROGS,
+};
+
+// kprobe progs maps from a program ID to an eBPF program
+bpf_map_def SEC("maps") kprobe_progs = {
   .type = BPF_MAP_TYPE_PROG_ARRAY,
   .key_size = sizeof(u32),
   .value_size = sizeof(u32),
@@ -171,9 +180,9 @@ void maybe_add_apm_info(Trace *trace) {
   DEBUG_PRINT("APM transaction ID: %016llX, flags: 0x%02X",
               trace->apm_transaction_id.as_int, corr_buf.trace_flags);
 }
-
-SEC("perf_event/unwind_stop")
-int unwind_stop(struct pt_regs *ctx) {
+static inline __attribute__((__always_inline__))
+int unwind_stop(struct pt_regs *ctx, bpf_map_def *prog_map)
+{
   PerCPURecord *record = get_per_cpu_record();
   if (!record)
     return -1;
@@ -234,11 +243,21 @@ int unwind_stop(struct pt_regs *ctx) {
   }
   // TEMPORARY HACK END
 
+
+  struct pt_regs regs = {0}; 
+  __builtin_memcpy(&regs, ctx, sizeof(struct pt_regs));
+  DEBUG_PRINT("OTEL registers ctx:  sp %lx", regs.sp);
+  trace->registers = regs;
+  DEBUG_PRINT("OTEL2 registers ctx:  sp %lx", trace->registers.sp);
+  DEBUG_PRINT("Now sending")
   send_trace(ctx, trace);
+  DEBUG_PRINT("Sent")
 
   return 0;
 }
 
+DEFINE_DUAL_PROGRAM(unwind_stop, unwind_stop);
+
 char _license[] SEC("license") = "GPL";
 // this number will be interpreted by the elf loader
 // to set the current running kernel version

support/ebpf/native_stack_trace.ebpf.c

@@ -3,6 +3,7 @@
 #include "types.h"
 #include "tracemgmt.h"
 #include "stackdeltatypes.h"
+#include "helpers.h"
 
 #ifndef __USER32_CS
   // defined in arch/x86/include/asm/segment.h
@@ -747,8 +748,9 @@ static inline ErrorCode get_usermode_regs(struct pt_regs *ctx,
 
 #endif
 
-SEC("perf_event/unwind_native")
-int unwind_native(struct pt_regs *ctx) {
+static inline __attribute__((__always_inline__))
+int unwind_native(struct pt_regs *ctx, bpf_map_def *prog_map)
+{
   PerCPURecord *record = get_per_cpu_record();
   if (!record)
     return -1;
@@ -794,13 +796,15 @@ int unwind_native(struct pt_regs *ctx) {
   // Tail call needed for recursion, switching to interpreter unwinder, or reporting
   // trace due to end-of-trace or error. The unwinder program index is set accordingly.
   record->state.unwind_error = error;
-  tail_call(ctx, unwinder);
+  tail_call(ctx, unwinder, prog_map);
   DEBUG_PRINT("bpf_tail call failed for %d in unwind_native", unwinder);
   return -1;
 }
 
-static inline
-int collect_trace(struct pt_regs *ctx) {
+DEFINE_DUAL_PROGRAM(unwind_native, unwind_native);
+
+static inline __attribute__((__always_inline__))
+int collect_trace(struct pt_regs *ctx, bpf_map_def *prog_map) {
   // Get the PID and TGID register.
   u64 id = bpf_get_current_pid_tgid();
   u32 pid = id >> 32;
@@ -833,7 +837,6 @@ int collect_trace(struct pt_regs *ctx) {
   // Get the kernel mode stack trace first
   trace->kernel_stack_id = bpf_get_stackid(ctx, &kernel_stackmap, BPF_F_REUSE_STACKID);
   DEBUG_PRINT("kernel stack id = %d", trace->kernel_stack_id);
-
   // Recursive unwind frames
   int unwinder = PROG_UNWIND_STOP;
   bool has_usermode_regs = false;
@@ -852,12 +855,19 @@ int collect_trace(struct pt_regs *ctx) {
 
 exit:
   record->state.unwind_error = error;
-  tail_call(ctx, unwinder);
+  tail_call(ctx, unwinder, prog_map);
   DEBUG_PRINT("bpf_tail call failed for %d in native_tracer_entry", unwinder);
   return -1;
 }
 
 SEC("perf_event/native_tracer_entry")
-int native_tracer_entry(struct bpf_perf_event_data *ctx) {
-  return collect_trace((struct pt_regs*) &ctx->regs);
+int native_tracer_entry_perf(struct bpf_perf_event_data *ctx)
+{
+  return collect_trace((struct pt_regs*) &ctx->regs, &perf_progs);
+}
+
+SEC("kprobe/native_tracer_entry")
+int native_tracer_entry_kprobe(struct pt_regs *ctx)
+{
+  return collect_trace(ctx, &kprobe_progs);
 }

support/ebpf/perl_tracer.ebpf.c

@@ -32,6 +32,7 @@
 #include "tracemgmt.h"
 #include "types.h"
 #include "tsd.h"
+#include "helpers.h"
 
 // The number of Perl frames to unwind per frame-unwinding eBPF program.
 #define PERL_FRAMES_PER_PROGRAM 12
@@ -356,8 +357,9 @@ int walk_perl_stack(PerCPURecord *record, const PerlProcInfo *perlinfo) {
 // unwind_perl is the entry point for tracing when invoked from the native tracer
 // or interpreter dispatcher. It does not reset the trace object and will append the
 // Perl stack frames to the trace object for the current CPU.
-SEC("perf_event/unwind_perl")
-int unwind_perl(struct pt_regs *ctx) {
+static inline __attribute__((__always_inline__))
+int unwind_perl(struct pt_regs *ctx, bpf_map_def *prog_map)
+{
   PerCPURecord *record = get_per_cpu_record();
   if (!record) {
     return -1;
@@ -423,6 +425,8 @@ int unwind_perl(struct pt_regs *ctx) {
   unwinder = walk_perl_stack(record, perlinfo);
 
 exit:
-  tail_call(ctx, unwinder);
+  tail_call(ctx, unwinder, prog_map);
   return -1;
 }
+
+DEFINE_DUAL_PROGRAM(unwind_perl, unwind_perl);

support/ebpf/php_tracer.ebpf.c

@@ -3,6 +3,7 @@
 #include "bpfdefs.h"
 #include "tracemgmt.h"
 #include "types.h"
+#include "helpers.h"
 
 // The number of PHP frames to unwind per frame-unwinding eBPF program. If
 // we start running out of instructions in the walk_php_stack program, one
@@ -182,8 +183,9 @@ int walk_php_stack(PerCPURecord *record, PHPProcInfo *phpinfo, bool is_jitted) {
   return unwinder;
 }
 
-SEC("perf_event/unwind_php")
-int unwind_php(struct pt_regs *ctx) {
+static inline __attribute__((__always_inline__))
+int unwind_php(struct pt_regs *ctx, bpf_map_def *prog_map)
+{
   PerCPURecord *record = get_per_cpu_record();
   if (!record)
     return -1;
@@ -236,6 +238,8 @@ int unwind_php(struct pt_regs *ctx) {
   unwinder = walk_php_stack(record, phpinfo, is_jitted);
 
 exit:
-  tail_call(ctx, unwinder);
+  tail_call(ctx, unwinder, prog_map);
   return -1;
 }
+
+DEFINE_DUAL_PROGRAM(unwind_php, unwind_php)