From ee997ff3b2e80e9ab91fb2bb8ffd6ed50c6f7e35 Mon Sep 17 00:00:00 2001 From: Kyle Huey Date: Sun, 26 May 2024 18:23:10 -0700 Subject: [PATCH] tranche 2 --- src/PerfCounters.cc | 202 +++++++++++++++++++++++------------ src/PerfCounters.h | 7 ++ src/ReplaySession.cc | 2 + src/bpf/async_event_filter.c | 13 +++ 4 files changed, 158 insertions(+), 66 deletions(-) diff --git a/src/PerfCounters.cc b/src/PerfCounters.cc index 2c003e67b2e..ecbd195d9dd 100644 --- a/src/PerfCounters.cc +++ b/src/PerfCounters.cc @@ -1125,96 +1125,166 @@ Ticks PerfCounters::read_ticks(Task* t, Error* error) { return ret; } + +class BpfAccelerator { +public: + static std::shared_ptr get_or_create(); + + ScopedFd create_counter(pid_t tid); + void match_regs_and_open_counter(Registers regs, ScopedFd& counter); + uint64_t skips() const { + return *bpf_skips; + } + + // Can't be private because of make_shared. + BpfAccelerator(ScopedFd&& bpf_prog_fd, user_regs_struct* bpf_regs, uint64_t* bpf_skips) + : bpf_prog_fd(std::move(bpf_prog_fd)), bpf_regs(bpf_regs), bpf_skips(bpf_skips) + {} + +private: + static std::shared_ptr singleton; + + struct perf_event_attr attr; + ScopedFd bpf_prog_fd; + user_regs_struct* bpf_regs; + uint64_t* bpf_skips; +}; + +std::shared_ptr BpfAccelerator::singleton; + #ifdef BPF -bool PerfCounters::accelerate_async_signal(const Registers& regs) { +/* static */ std::shared_ptr BpfAccelerator::get_or_create() { static int initialized; - static struct perf_event_attr attr; - static int bpf_prog_fd; - static struct user_regs_struct* bpf_regs; + if (BpfAccelerator::singleton) { + return BpfAccelerator::singleton; + } - if (!fd_async_signal_accelerator.is_open()) { - if (!initialized) { - initialized = -1; - - attr.type = PERF_TYPE_BREAKPOINT; - attr.size = sizeof(attr); - attr.bp_type = HW_BREAKPOINT_X; - attr.bp_len = sizeof(long); - attr.sample_period = 1; - attr.sample_type = PERF_SAMPLE_IP; - attr.pinned = 1; - attr.exclude_kernel = 1; - attr.exclude_hv = 1; - attr.wakeup_events = 1; - attr.precise_ip = 3; - attr.disabled = 1; - - libbpf_set_strict_mode(LIBBPF_STRICT_DIRECT_ERRS); - string path = resource_path() + "share/rr/async_event_filter.o"; - struct bpf_object* obj = bpf_object__open(path.c_str()); - if ((intptr_t)obj <= 0) { - return false; - } - if (bpf_object__load(obj) < 0) { - return false; - } - int bpf_map_fd = bpf_object__find_map_fd_by_name(obj, "registers"); - if (bpf_map_fd < 0) { - return false; - } - struct bpf_program* prog = bpf_program__next(NULL, obj); - if (!prog) { - return false; - } - bpf_prog_fd = bpf_program__fd(prog); - if (bpf_prog_fd < 0) { - return false; - } + if (!initialized) { + initialized = -1; - bpf_regs = (struct user_regs_struct*) - mmap(NULL, 4096, PROT_READ | PROT_WRITE, - MAP_SHARED, bpf_map_fd, 0); - if (bpf_regs == MAP_FAILED) { - return false; - } + libbpf_set_strict_mode(LIBBPF_STRICT_DIRECT_ERRS); + string path = resource_path() + "share/rr/async_event_filter.o"; + struct bpf_object* obj = bpf_object__open(path.c_str()); + if ((intptr_t)obj <= 0) { + return nullptr; + } + if (bpf_object__load(obj) < 0) { + return nullptr; + } + int bpf_map_fd = bpf_object__find_map_fd_by_name(obj, "registers"); + if (bpf_map_fd < 0) { + return nullptr; + } + struct bpf_program* prog = bpf_program__next(NULL, obj); + if (!prog) { + return nullptr; + } + ScopedFd bpf_prog_fd = ScopedFd(bpf_program__fd(prog)); + if (bpf_prog_fd < 0) { + return nullptr; + } - initialized = 1; - } else if (initialized < 0) { - return false; + auto bpf_regs = (struct user_regs_struct*) + mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED, bpf_map_fd, 0); + if (bpf_regs == MAP_FAILED) { + return nullptr; } - attr.bp_addr = 0; - fd_async_signal_accelerator = start_counter(tid, -1, &attr); + bpf_map_fd = bpf_object__find_map_fd_by_name(obj, "skips"); + if (bpf_map_fd < 0) { + return nullptr; + } - struct f_owner_ex own; - own.type = F_OWNER_TID; - own.pid = tid; - if (fcntl(fd_async_signal_accelerator, F_SETOWN_EX, &own)) { - FATAL() << "Failed to SETOWN_EX bpf-accelerated breakpoint fd"; + auto bpf_skips = (uint64_t*) + mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED, bpf_map_fd, 0); + if (bpf_regs == MAP_FAILED) { + return nullptr; } - make_counter_async(fd_async_signal_accelerator, SIGTRAP); + BpfAccelerator::singleton = + std::make_shared(std::move(bpf_prog_fd), bpf_regs, bpf_skips); + memset(&singleton->attr, 0, sizeof(singleton->attr)); + singleton->attr.type = PERF_TYPE_BREAKPOINT; + singleton->attr.size = sizeof(attr); + singleton->attr.bp_type = HW_BREAKPOINT_X; + singleton->attr.bp_len = sizeof(long); + singleton->attr.sample_period = 1; + singleton->attr.sample_type = PERF_SAMPLE_IP; + singleton->attr.pinned = 1; + singleton->attr.exclude_kernel = 1; + singleton->attr.exclude_hv = 1; + singleton->attr.wakeup_events = 1; + singleton->attr.precise_ip = 3; + singleton->attr.disabled = 1; + initialized = 1; + } + + return BpfAccelerator::singleton; +} - if (ioctl(fd_async_signal_accelerator, PERF_EVENT_IOC_SET_BPF, bpf_prog_fd)) { - FATAL() << "Failed PERF_EVENT_IOC_SET_BPF"; - } +ScopedFd BpfAccelerator::create_counter(pid_t tid) { + attr.bp_addr = 0; + ScopedFd fd = start_counter(tid, -1, &attr); + + struct f_owner_ex own; + own.type = F_OWNER_TID; + own.pid = tid; + if (fcntl(fd, F_SETOWN_EX, &own)) { + FATAL() << "Failed to SETOWN_EX bpf-accelerated breakpoint fd"; } - if (!fd_async_signal_accelerator.is_open()) { - return false; + make_counter_async(fd, SIGTRAP); + + if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, bpf_prog_fd.get())) { + FATAL() << "Failed PERF_EVENT_IOC_SET_BPF"; } + return fd; +} + +void BpfAccelerator::match_regs_and_open_counter(Registers regs, ScopedFd& fd) { attr.bp_addr = regs.ip().register_value(); - if (ioctl(fd_async_signal_accelerator, PERF_EVENT_IOC_MODIFY_ATTRIBUTES, &attr)) { + if (ioctl(fd, PERF_EVENT_IOC_MODIFY_ATTRIBUTES, &attr)) { FATAL() << "Failed PERF_EVENT_IOC_MODIFY_ATTRIBUTES"; } auto r = regs.get_ptrace(); memcpy(bpf_regs, &r, sizeof(struct user_regs_struct)); + *bpf_skips = 0; - infallible_perf_event_enable_if_open(fd_async_signal_accelerator); + infallible_perf_event_enable_if_open(fd); +} + +bool PerfCounters::accelerate_async_signal(const Registers& regs) { + if (!fd_async_signal_accelerator.is_open()) { + if (!bpf) { + bpf = BpfAccelerator::get_or_create(); + } + + if (!bpf) { + return false; + } + + fd_async_signal_accelerator = bpf->create_counter(tid); + } + + if (!fd_async_signal_accelerator.is_open()) { + return false; + } + + bpf->match_regs_and_open_counter(regs, fd_async_signal_accelerator); return true; } + +uint64_t PerfCounters::bpf_skips() const { + if (!bpf) { + return 0; + } + + return bpf->skips(); +} #endif } // namespace rr diff --git a/src/PerfCounters.h b/src/PerfCounters.h index 179e2bb42f0..c9982f6a1dd 100644 --- a/src/PerfCounters.h +++ b/src/PerfCounters.h @@ -23,6 +23,7 @@ namespace rr { class Registers; class Task; +class BpfAccelerator; enum TicksSemantics { TICKS_RETIRED_CONDITIONAL_BRANCHES, @@ -181,10 +182,14 @@ class PerfCounters { */ #ifdef BPF bool accelerate_async_signal(const Registers& regs); + uint64_t bpf_skips() const; #else bool accelerate_async_signal(const Registers&) { return false; } + uint64_t bpf_skips() const { + return 0; + } #endif private: @@ -227,6 +232,8 @@ class PerfCounters { // BPF-enabled hardware breakpoint for fast async signal emulation. ScopedFd fd_async_signal_accelerator; + std::shared_ptr bpf; + std::unique_ptr pt_state; TicksSemantics ticks_semantics_; diff --git a/src/ReplaySession.cc b/src/ReplaySession.cc index 7fa20683e9e..8333948d0c9 100644 --- a/src/ReplaySession.cc +++ b/src/ReplaySession.cc @@ -1055,6 +1055,8 @@ Completion ReplaySession::emulate_async_signal( return COMPLETE; } ASSERT(t, regs.ip() == t->ip().undo_executed_bkpt(t->arch())); + } else { + LOG(debug) << " fast-forwarded through " << t->hpc.bpf_skips() << " breakpoint hits with bpf"; } /* Case (1) above: cover the tracks of * our internal breakpoint, and go diff --git a/src/bpf/async_event_filter.c b/src/bpf/async_event_filter.c index 611042744b9..ad4cd638c1d 100644 --- a/src/bpf/async_event_filter.c +++ b/src/bpf/async_event_filter.c @@ -13,6 +13,14 @@ struct { __type(value, uint64_t); } registers SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_MMAPABLE); + __type(key, uint32_t); + __type(value, uint64_t); +} skips SEC(".maps"); + SEC("perf_event") int match_registers(struct bpf_perf_event_data* event) { #define CHECK_REG(name) \ @@ -23,6 +31,11 @@ int match_registers(struct bpf_perf_event_data* event) { return 1; \ } \ if (event->regs.name != *reg) { \ + const uint32_t j = 0; \ + uint64_t* s = bpf_map_lookup_elem(&skips, &j); \ + if (s) { \ + *s += 1; \ + } \ return 0; \ } \ }