Create PerfCounterBuffers to contain the logic to extract packets f…

…rom perf event buffers

CMakeLists.txt

@@ -619,6 +619,7 @@ set(RR_SOURCES
   src/MvCommand.cc
   src/PackCommand.cc
   src/PerfCounters.cc
+  src/PerfCounterBuffers.cc
   src/PidFdMonitor.cc
   src/processor_trace_check.cc
   src/ProcFdDirMonitor.cc

src/PerfCounterBuffers.cc

@@ -0,0 +1,107 @@
+/* -*- Mode: C++; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
+
+#include "PerfCounterBuffers.h"
+
+#include <sys/mman.h>
+
+#include "util.h"
+#include "log.h"
+
+using namespace std;
+
+namespace rr {
+
+void PerfCounterBuffers::destroy() {
+  if (mmap_aux_buffer) {
+    munmap(mmap_aux_buffer, mmap_header->aux_size);
+    mmap_aux_buffer = nullptr;
+  }
+  if (mmap_header) {
+    munmap(mmap_header, page_size() + buffer_size);
+    mmap_header = nullptr;
+  }
+}
+
+void PerfCounterBuffers::allocate(ScopedFd& perf_event_fd,
+    uint64_t buffer_size, uint64_t aux_size) {
+  this->buffer_size = buffer_size;
+
+  void* base = mmap(NULL, page_size() + buffer_size,
+      PROT_READ | PROT_WRITE, MAP_SHARED, perf_event_fd, 0);
+  if (base == MAP_FAILED) {
+    FATAL() << "Can't allocate memory for PT DATA area";
+  }
+  mmap_header = static_cast<struct perf_event_mmap_page*>(base);
+
+  if (aux_size > 0) {
+    mmap_header->aux_offset = mmap_header->data_offset + mmap_header->data_size;
+    mmap_header->aux_size = aux_size;
+
+    void* aux = mmap(NULL, mmap_header->aux_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+        perf_event_fd, mmap_header->aux_offset);
+    if (aux == MAP_FAILED) {
+      FATAL() << "Can't allocate memory for PT AUX area";
+    }
+    mmap_aux_buffer = static_cast<char*>(aux);
+  }
+}
+
+optional<PerfCounterBuffers::Packet> PerfCounterBuffers::next_packet() {
+  if (packet_in_use) {
+    FATAL() << "Can't offer more than one packet at a time";
+  }
+
+  uint64_t data_end = *reinterpret_cast<volatile uint64_t*>(mmap_header->data_head);
+  if (mmap_header->data_tail >= data_end) {
+    return nullopt;
+  }
+  __sync_synchronize();
+
+  char* data_buf = reinterpret_cast<char*>(mmap_header) + mmap_header->data_offset;
+  uint64_t data_start = mmap_header->data_tail;
+  size_t start_offset = data_start % mmap_header->data_size;
+  auto header_ptr = reinterpret_cast<struct perf_event_header*>(data_buf + start_offset);
+  struct perf_event_header header;
+  memcpy(&header, header_ptr, sizeof(header));
+
+  size_t first_chunk_size = min<size_t>(header.size,
+      mmap_header->data_size - start_offset);
+
+  if (first_chunk_size < header.size) {
+    packet_storage.resize(header.size);
+    memcpy(packet_storage.data(), const_cast<perf_event_header*>(header_ptr), first_chunk_size);
+    memcpy(packet_storage.data() + first_chunk_size, const_cast<char*>(data_buf), header.size - first_chunk_size);
+    header_ptr = reinterpret_cast<struct perf_event_header*>(packet_storage.data());
+  }
+
+  void* aux_ptr = nullptr;
+  if (header.type == PERF_RECORD_AUX) {
+    PerfEventAux aux_packet = *reinterpret_cast<PerfEventAux*>(header_ptr);
+    size_t aux_start_offset = aux_packet.aux_offset % mmap_header->aux_size;
+    aux_ptr = mmap_aux_buffer + aux_start_offset;
+    first_chunk_size = min<size_t>(aux_packet.aux_size, mmap_header->aux_size - aux_start_offset);
+    if (first_chunk_size < aux_packet.aux_size) {
+      aux_packet_storage.resize(aux_packet.aux_size);
+      memcpy(aux_packet_storage.data(), aux_ptr, first_chunk_size);
+      memcpy(aux_packet_storage.data() + first_chunk_size, mmap_aux_buffer,
+             aux_packet.aux_size - first_chunk_size);
+      aux_ptr = aux_packet_storage.data();
+    }
+    packet_data_aux_end = mmap_header->aux_tail + aux_packet.aux_size;
+  }
+
+  packet_in_use = true;
+  packet_data_end = data_start + header.size;
+  return Packet(*this, header_ptr, aux_ptr);
+}
+
+void PerfCounterBuffers::release_packet() {
+  if (!packet_in_use) {
+    FATAL() << "No packet!";
+  }
+  mmap_header->data_tail = packet_data_end;
+  mmap_header->aux_tail = packet_data_aux_end;
+  packet_in_use = false;
+}
+
+}

src/PerfCounterBuffers.h

@@ -0,0 +1,85 @@
+/* -*- Mode: C++; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
+
+#ifndef RR_PERF_COUNTER_BUFFERS_H_
+#define RR_PERF_COUNTER_BUFFERS_H_
+
+#include <stdint.h>
+
+#include <optional>
+
+#include <linux/perf_event.h>
+
+#include "ScopedFd.h"
+
+namespace rr {
+
+// I wish I knew why this type isn't defined in perf_event.h but is just
+// commented out there...
+struct PerfEventAux {
+  struct perf_event_header header;
+  uint64_t aux_offset;
+  uint64_t aux_size;
+  uint64_t flags;
+  uint64_t sample_id;
+};
+
+/**
+ * Encapsulates the mmap buffers used for perf events.
+ */
+class PerfCounterBuffers {
+public:
+  PerfCounterBuffers() : mmap_header(nullptr), mmap_aux_buffer(nullptr),
+      buffer_size(0), packet_in_use(false) {}
+  ~PerfCounterBuffers() { destroy(); }
+
+  void allocate(ScopedFd& perf_event_fd, uint64_t buffer_size, uint64_t aux_size);
+  void destroy();
+
+  bool allocated() const { return mmap_header != nullptr; }
+
+  class Packet {
+  public:
+    Packet(Packet&& other) : buffers(other.buffers), data_(other.data_),
+        aux_data_(other.aux_data_) {
+      other.buffers = nullptr;
+    }
+    ~Packet() {
+      if (buffers) {
+        buffers->release_packet();
+      }
+    }
+    struct perf_event_header* data() const { return data_; }
+    void* aux_data() const { return aux_data_; }
+
+  private:
+    friend class PerfCounterBuffers;
+
+    Packet(PerfCounterBuffers& buffers, struct perf_event_header* data,
+        void* aux_data)
+      : buffers(&buffers), data_(data), aux_data_(aux_data) {}
+
+    PerfCounterBuffers* buffers;
+    struct perf_event_header* data_;
+    void* aux_data_;
+  };
+
+  std::optional<Packet> next_packet();
+
+private:
+  friend class Packet;
+
+  void release_packet();
+
+  perf_event_mmap_page* mmap_header;
+  char* mmap_aux_buffer;
+  uint64_t buffer_size;
+  uint64_t packet_data_end;
+  uint64_t packet_data_aux_end;
+  std::vector<uint8_t> packet_storage;
+  std::vector<uint8_t> aux_packet_storage;
+  bool packet_in_use;
+};
+
+} // namespace rr
+
+#endif /* RR_PERF_COUNTER_BUFFERS_H_ */

src/PerfCounters.cc

@@ -778,87 +778,42 @@ void PerfCounters::PTState::open(pid_t tid) {
     return;
   }
 
-  void* base = mmap(NULL, page_size() + PT_PERF_DATA_SIZE,
-      PROT_READ | PROT_WRITE, MAP_SHARED, pt_perf_event_fd, 0);
-  if (base == MAP_FAILED) {
-    FATAL() << "Can't allocate memory for PT DATA area";
-  }
-  auto header = static_cast<struct perf_event_mmap_page*>(base);
-  mmap_header = header;
-
-  header->aux_offset = header->data_offset + header->data_size;
-  header->aux_size = PT_PERF_AUX_SIZE;
-
-  void* aux = mmap(NULL, header->aux_size, PROT_READ | PROT_WRITE, MAP_SHARED,
-      pt_perf_event_fd, header->aux_offset);
-  if (aux == MAP_FAILED) {
-    FATAL() << "Can't allocate memory for PT AUX area";
-  }
-  mmap_aux_buffer = static_cast<char*>(aux);
+  perf_buffers.allocate(pt_perf_event_fd, PT_PERF_DATA_SIZE, PT_PERF_AUX_SIZE);
 }
 
-// I wish I knew why this type isn't defined in perf_event.h but is just
-// commented out there...
-struct PerfEventAux {
-  struct perf_event_header header;
-  uint64_t aux_offset;
-  uint64_t aux_size;
-  uint64_t flags;
-  uint64_t sample_id;
-};
-
 size_t PerfCounters::PTState::flush() {
-  if (!mmap_header) {
+  if (!perf_buffers.allocated()) {
     return 0;
   }
 
-  uint64_t data_end = mmap_header->data_head;
-  __sync_synchronize();
-  volatile char* data_buf = reinterpret_cast<volatile char*>(mmap_header) +
-      mmap_header->data_offset;
-
-  vector<char> packet;
   size_t ret = 0;
-  while (mmap_header->data_tail < data_end) {
-    uint64_t data_start = mmap_header->data_tail;
-    size_t start_offset = data_start % mmap_header->data_size;
-    auto header = reinterpret_cast<volatile perf_event_header*>(data_buf + start_offset);
-    packet.resize(header->size);
-    size_t first_chunk_size = min<size_t>(header->size,
-        mmap_header->data_size - start_offset);
-    memcpy(packet.data(), const_cast<perf_event_header*>(header), first_chunk_size);
-    memcpy(packet.data() + first_chunk_size, const_cast<char*>(data_buf), header->size - first_chunk_size);
-
-    switch (header->type) {
+
+  while (auto packet = perf_buffers.next_packet()) {
+    struct perf_event_header header = *packet->data();
+    switch (header.type) {
       case PERF_RECORD_LOST:
         FATAL() << "PT records lost!";
         break;
       case PERF_RECORD_ITRACE_START:
         break;
       case PERF_RECORD_AUX: {
-        auto aux_packet = reinterpret_cast<PerfEventAux*>(packet.data());
-        if (aux_packet->flags) {
-          FATAL() << "Unexpected AUX packet flags " << aux_packet->flags;
+        auto aux_packet = *reinterpret_cast<PerfEventAux*>(packet->data());
+        if (aux_packet.flags) {
+          FATAL() << "Unexpected AUX packet flags " << aux_packet.flags;
         }
         pt_data.data.emplace_back();
         vector<uint8_t>& data = pt_data.data.back();
-        data.resize(aux_packet->aux_size);
-        ret += aux_packet->aux_size;
-        size_t aux_start_offset = aux_packet->aux_offset % PT_PERF_AUX_SIZE;
-        first_chunk_size = min<size_t>(aux_packet->aux_size, PT_PERF_AUX_SIZE - aux_start_offset);
-        memcpy(data.data(), mmap_aux_buffer + aux_start_offset, first_chunk_size);
-        memcpy(data.data() + first_chunk_size, mmap_aux_buffer,
-               aux_packet->aux_size - first_chunk_size);
-        mmap_header->aux_tail += aux_packet->aux_size;
+        data.resize(aux_packet.aux_size);
+        memcpy(data.data(), packet->aux_data(), aux_packet.aux_size);
+        ret += aux_packet.aux_size;
         break;
       }
       default:
-        FATAL() << "Unknown record " << header->type;
+        FATAL() << "Unknown record " << header.type;
         break;
     }
-
-    mmap_header->data_tail += header->size;
   }
+
   return ret;
 }
 
@@ -872,12 +827,7 @@ PTData PerfCounters::extract_intel_pt_data() {
 
 void PerfCounters::PTState::close() {
   pt_perf_event_fd.close();
-  if (mmap_aux_buffer) {
-    munmap(mmap_aux_buffer, mmap_header->aux_size);
-    mmap_aux_buffer = nullptr;
-    munmap(const_cast<perf_event_mmap_page*>(mmap_header), page_size() + PT_PERF_DATA_SIZE);
-    mmap_header = nullptr;
-  }
+  perf_buffers.destroy();
 }
 
 void PerfCounters::start(Task* t, Ticks ticks_period) {

src/PerfCounters.h

@@ -16,6 +16,7 @@
 #include <utility>
 #include <vector>
 
+#include "PerfCounterBuffers.h"
 #include "ScopedFd.h"
 #include "Ticks.h"
 
@@ -70,10 +71,9 @@ class PerfCounters {
   struct PTState {
     PTData pt_data;
     ScopedFd pt_perf_event_fd;
-    volatile perf_event_mmap_page* mmap_header;
-    char* mmap_aux_buffer;
+    PerfCounterBuffers perf_buffers;
 
-    PTState() : mmap_header(nullptr), mmap_aux_buffer(nullptr) {}
+    PTState() {}
     ~PTState() { close(); }
 
     void open(pid_t tid);