diff --git a/src/PerfCounters.cc b/src/PerfCounters.cc index ecbd195d9dd..a46cec4df74 100644 --- a/src/PerfCounters.cc +++ b/src/PerfCounters.cc @@ -1131,21 +1131,30 @@ class BpfAccelerator { static std::shared_ptr get_or_create(); ScopedFd create_counter(pid_t tid); - void match_regs_and_open_counter(Registers regs, ScopedFd& counter); + void match_regs_and_open_counter(const Registers& regs, ScopedFd& counter); uint64_t skips() const { return *bpf_skips; } // Can't be private because of make_shared. - BpfAccelerator(ScopedFd&& bpf_prog_fd, user_regs_struct* bpf_regs, uint64_t* bpf_skips) - : bpf_prog_fd(std::move(bpf_prog_fd)), bpf_regs(bpf_regs), bpf_skips(bpf_skips) + BpfAccelerator(struct bpf_object* bpf_obj, int bpf_prog_fd, + user_regs_struct* bpf_regs, uint64_t* bpf_skips) + : bpf_obj(bpf_obj), bpf_prog_fd(bpf_prog_fd), bpf_regs(bpf_regs), bpf_skips(bpf_skips) {} + ~BpfAccelerator() { + munmap(bpf_skips, 4096); + munmap(bpf_regs, 4096); + bpf_object__close(bpf_obj); + } + private: static std::shared_ptr singleton; struct perf_event_attr attr; - ScopedFd bpf_prog_fd; + struct bpf_object* bpf_obj; + // Not a ScopedFd because the bpf_object maintains ownership. + int bpf_prog_fd; user_regs_struct* bpf_regs; uint64_t* bpf_skips; }; @@ -1166,21 +1175,27 @@ std::shared_ptr BpfAccelerator::singleton; string path = resource_path() + "share/rr/async_event_filter.o"; struct bpf_object* obj = bpf_object__open(path.c_str()); if ((intptr_t)obj <= 0) { + LOG(error) << "Failed to find bpf at " << path; return nullptr; } if (bpf_object__load(obj) < 0) { + LOG(error) << "Failed to load bpf at " << path << " into the kernel. Do we have permissions?"; + bpf_object__close(obj); return nullptr; } int bpf_map_fd = bpf_object__find_map_fd_by_name(obj, "registers"); if (bpf_map_fd < 0) { + CLEAN_FATAL() << "rr's bpf at " << path << " is corrupt"; return nullptr; } struct bpf_program* prog = bpf_program__next(NULL, obj); if (!prog) { + CLEAN_FATAL() << "rr's bpf at " << path << " is corrupt"; return nullptr; } - ScopedFd bpf_prog_fd = ScopedFd(bpf_program__fd(prog)); + int bpf_prog_fd = bpf_program__fd(prog); if (bpf_prog_fd < 0) { + CLEAN_FATAL() << "rr's bpf at " << path << " is corrupt"; return nullptr; } @@ -1188,11 +1203,13 @@ std::shared_ptr BpfAccelerator::singleton; mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, bpf_map_fd, 0); if (bpf_regs == MAP_FAILED) { + CLEAN_FATAL() << "Failed to mmap bpf maps"; return nullptr; } bpf_map_fd = bpf_object__find_map_fd_by_name(obj, "skips"); if (bpf_map_fd < 0) { + CLEAN_FATAL() << "rr's bpf at " << path << " is corrupt"; return nullptr; } @@ -1200,11 +1217,12 @@ std::shared_ptr BpfAccelerator::singleton; mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, bpf_map_fd, 0); if (bpf_regs == MAP_FAILED) { + CLEAN_FATAL() << "Failed to mmap bpf maps"; return nullptr; } BpfAccelerator::singleton = - std::make_shared(std::move(bpf_prog_fd), bpf_regs, bpf_skips); + std::make_shared(obj, bpf_prog_fd, bpf_regs, bpf_skips); memset(&singleton->attr, 0, sizeof(singleton->attr)); singleton->attr.type = PERF_TYPE_BREAKPOINT; singleton->attr.size = sizeof(attr); @@ -1237,14 +1255,14 @@ ScopedFd BpfAccelerator::create_counter(pid_t tid) { make_counter_async(fd, SIGTRAP); - if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, bpf_prog_fd.get())) { + if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, bpf_prog_fd)) { FATAL() << "Failed PERF_EVENT_IOC_SET_BPF"; } return fd; } -void BpfAccelerator::match_regs_and_open_counter(Registers regs, ScopedFd& fd) { +void BpfAccelerator::match_regs_and_open_counter(const Registers& regs, ScopedFd& fd) { attr.bp_addr = regs.ip().register_value(); if (ioctl(fd, PERF_EVENT_IOC_MODIFY_ATTRIBUTES, &attr)) { FATAL() << "Failed PERF_EVENT_IOC_MODIFY_ATTRIBUTES"; diff --git a/src/bpf/async_event_filter.c b/src/bpf/async_event_filter.c index ad4cd638c1d..7ee2f997b49 100644 --- a/src/bpf/async_event_filter.c +++ b/src/bpf/async_event_filter.c @@ -1,3 +1,5 @@ +/* -*- Mode: C++; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */ + #include #include #include @@ -6,58 +8,58 @@ const uint32_t REGISTER_COUNT = sizeof(struct pt_regs)/sizeof(uint64_t); struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, REGISTER_COUNT); - __uint(map_flags, BPF_F_MMAPABLE); - __type(key, uint32_t); - __type(value, uint64_t); + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, REGISTER_COUNT); + __uint(map_flags, BPF_F_MMAPABLE); + __type(key, uint32_t); + __type(value, uint64_t); } registers SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(max_entries, 1); - __uint(map_flags, BPF_F_MMAPABLE); - __type(key, uint32_t); - __type(value, uint64_t); + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __uint(map_flags, BPF_F_MMAPABLE); + __type(key, uint32_t); + __type(value, uint64_t); } skips SEC(".maps"); SEC("perf_event") int match_registers(struct bpf_perf_event_data* event) { #define CHECK_REG(name) \ - { \ - const uint32_t i = offsetof(struct pt_regs, name) / sizeof(uint64_t); \ - uint64_t* reg = bpf_map_lookup_elem(®isters, &i); \ - if (!reg) { \ - return 1; \ - } \ - if (event->regs.name != *reg) { \ - const uint32_t j = 0; \ - uint64_t* s = bpf_map_lookup_elem(&skips, &j); \ - if (s) { \ - *s += 1; \ - } \ - return 0; \ - } \ - } - - CHECK_REG(r15) - CHECK_REG(r14) - CHECK_REG(r13) - CHECK_REG(r12) - CHECK_REG(rbp) - CHECK_REG(rbx) - CHECK_REG(r11) - CHECK_REG(r10) - CHECK_REG(r9) - CHECK_REG(r8) - CHECK_REG(rax) - CHECK_REG(rcx) - CHECK_REG(rdx) - CHECK_REG(rsi) - CHECK_REG(rdi) - CHECK_REG(rsp) - - return 1; + do { \ + const uint32_t i = offsetof(struct pt_regs, name) / sizeof(uint64_t); \ + uint64_t* reg = bpf_map_lookup_elem(®isters, &i); \ + if (!reg) { \ + return 1; \ + } \ + if (event->regs.name != *reg) { \ + const uint32_t j = 0; \ + uint64_t* s = bpf_map_lookup_elem(&skips, &j); \ + if (s) { \ + *s += 1; \ + } \ + return 0; \ + } \ + } while(0) + + CHECK_REG(r15); + CHECK_REG(r14); + CHECK_REG(r13); + CHECK_REG(r12); + CHECK_REG(rbp); + CHECK_REG(rbx); + CHECK_REG(r11); + CHECK_REG(r10); + CHECK_REG(r9); + CHECK_REG(r8); + CHECK_REG(rax); + CHECK_REG(rcx); + CHECK_REG(rdx); + CHECK_REG(rsi); + CHECK_REG(rdi); + CHECK_REG(rsp); + + return 1; } char _license[] SEC("license") = "Dual MIT/GPL";