Merge pull request #1053 from palmtenor/bufferepoll

Use epoll in BPFPerfBuffer

src/cc/BPFTable.cc

@@ -14,18 +14,20 @@
  * limitations under the License.
  */
 
-#include <errno.h>
+#include <sys/epoll.h>
 #include <unistd.h>
+#include <cerrno>
 #include <cstring>
 #include <iostream>
+#include <memory>
 
 #include "BPFTable.h"
 
 #include "bcc_exception.h"
 #include "bcc_syms.h"
+#include "common.h"
 #include "libbpf.h"
 #include "perf_reader.h"
-#include "common.h"
 
 namespace ebpf {
 
@@ -70,27 +72,42 @@ StatusTuple BPFPerfBuffer::open_on_cpu(perf_reader_raw_cb cb, int cpu,
  void* cb_cookie, int page_cnt) {
  if (cpu_readers_.find(cpu) != cpu_readers_.end())
  return StatusTuple(-1, "Perf buffer already open on CPU %d", cpu);
- auto reader =
- static_cast<perf_reader*>(bpf_open_perf_buffer(cb, cb_cookie, -1, cpu, page_cnt));
+
+ auto reader = static_cast<perf_reader*>(
+ bpf_open_perf_buffer(cb, cb_cookie, -1, cpu, page_cnt));
  if (reader == nullptr)
  return StatusTuple(-1, "Unable to construct perf reader");
+
  int reader_fd = perf_reader_fd(reader);
  if (!update(&cpu, &reader_fd)) {
  perf_reader_free(static_cast<void*>(reader));
  return StatusTuple(-1, "Unable to open perf buffer on CPU %d: %s", cpu,
- strerror(errno));
+ std::strerror(errno));
+ }
+
+ struct epoll_event event = {};
+ event.events = EPOLLIN;
+ event.data.ptr = static_cast<void*>(reader);
+ if (epoll_ctl(epfd_, EPOLL_CTL_ADD, reader_fd, &event) != 0) {
+ perf_reader_free(static_cast<void*>(reader));
+ return StatusTuple(-1, "Unable to add perf_reader FD to epoll: %s",
+ std::strerror(errno));
  }
+
  cpu_readers_[cpu] = reader;
- readers_.push_back(reader);
  return StatusTuple(0);
 }
 
-StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb,
- void* cb_cookie, int page_cnt) {
- if (cpu_readers_.size() != 0 || readers_.size() != 0)
+StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb, void* cb_cookie,
+ int page_cnt) {
+ if (cpu_readers_.size() != 0 || epfd_ != -1)
  return StatusTuple(-1, "Previously opened perf buffer not cleaned");
 
- for (int i: get_online_cpus()) {
+ std::vector<int> cpus = get_online_cpus();
+ ep_events_.reset(new epoll_event[cpus.size()]);
+ epfd_ = epoll_create1(EPOLL_CLOEXEC);
+
+ for (int i : cpus) {
  auto res = open_on_cpu(cb, i, cb_cookie, page_cnt);
  if (res.code() != 0) {
  TRY2(close_all_cpu());
@@ -114,31 +131,47 @@ StatusTuple BPFPerfBuffer::close_on_cpu(int cpu) {
 StatusTuple BPFPerfBuffer::close_all_cpu() {
  std::string errors;
  bool has_error = false;
- for (int i: get_online_cpus()) {
+
+ int close_res = close(epfd_);
+ epfd_ = -1;
+ ep_events_.reset();
+ if (close_res != 0) {
+ has_error = true;
+ errors += std::string(std::strerror(errno)) + "\n";
+ }
+
+ std::vector<int> opened_cpus;
+ for (auto it : cpu_readers_)
+ opened_cpus.push_back(it.first);
+ for (int i : opened_cpus) {
  auto res = close_on_cpu(i);
  if (res.code() != 0) {
  errors += "Failed to close CPU" + std::to_string(i) + " perf buffer: ";
  errors += res.msg() + "\n";
  has_error = true;
  }
  }
- readers_.clear();
+
  if (has_error)
  return StatusTuple(-1, errors);
  return StatusTuple(0);
 }
 
 void BPFPerfBuffer::poll(int timeout) {
- if (readers_.empty())
+ if (epfd_ < 0)
+ return;
+ int cnt = epoll_wait(epfd_, ep_events_.get(), cpu_readers_.size(), timeout);
+ if (cnt <= 0)
  return;
- perf_reader_poll(readers_.size(), readers_.data(), timeout);
+ for (int i = 0; i < cnt; i++)
+ perf_reader_event_read(static_cast<perf_reader*>(ep_events_[i].data.ptr));
 }
 
 BPFPerfBuffer::~BPFPerfBuffer() {
  auto res = close_all_cpu();
  if (res.code() != 0)
- std::cerr << "Failed to close all perf buffer on destruction: "
- << res.msg() << std::endl;
+ std::cerr << "Failed to close all perf buffer on destruction: " << res.msg()
+ << std::endl;
 }
 
 } // namespace ebpf

src/cc/BPFTable.h

@@ -16,6 +16,7 @@
 
 #pragma once
 
+#include <sys/epoll.h>
 #include <exception>
 #include <map>
 #include <memory>
@@ -123,7 +124,7 @@ class BPFStackTable : protected BPFTableBase<int, stacktrace_t> {
 class BPFPerfBuffer : protected BPFTableBase<int, int> {
 public:
  BPFPerfBuffer(BPFModule* bpf_module, const std::string& name)
- : BPFTableBase<int, int>(bpf_module, name) {}
+ : BPFTableBase<int, int>(bpf_module, name), epfd_(-1) {}
  ~BPFPerfBuffer();
 
  StatusTuple open_all_cpu(perf_reader_raw_cb cb, void* cb_cookie,
@@ -137,7 +138,9 @@ class BPFPerfBuffer : protected BPFTableBase<int, int> {
  StatusTuple close_on_cpu(int cpu);
 
  std::map<int, perf_reader*> cpu_readers_;
- std::vector<perf_reader*> readers_;
+
+ int epfd_;
+ std::unique_ptr<epoll_event[]> ep_events_;
 };
 
 } // namespace ebpf

src/cc/perf_reader.c

@@ -204,7 +204,7 @@ static void write_data_tail(struct perf_event_mmap_page *perf_header, uint64_t d
  perf_header->data_tail = data_tail;
 }
 
-static void event_read(struct perf_reader *reader) {
+void perf_reader_event_read(struct perf_reader *reader) {
  struct perf_event_mmap_page *perf_header = reader->base;
  uint64_t buffer_size = (uint64_t)reader->page_size * reader->page_cnt;
  uint64_t data_head;
@@ -261,7 +261,7 @@ int perf_reader_poll(int num_readers, struct perf_reader **readers, int timeout)
  if (poll(pfds, num_readers, timeout) > 0) {
  for (i = 0; i < num_readers; ++i) {
  if (pfds[i].revents & POLLIN)
- event_read(readers[i]);
+ perf_reader_event_read(readers[i]);
  }
  }
  return 0;

src/cc/perf_reader.h

@@ -29,6 +29,7 @@ struct perf_reader * perf_reader_new(perf_reader_cb cb,
  perf_reader_raw_cb raw_cb, void *cb_cookie, int page_cnt);
 void perf_reader_free(void *ptr);
 int perf_reader_mmap(struct perf_reader *reader, unsigned type, unsigned long sample_type);
+void perf_reader_event_read(struct perf_reader *reader);
 int perf_reader_poll(int num_readers, struct perf_reader **readers, int timeout);
 int perf_reader_fd(struct perf_reader *reader);
 void perf_reader_set_fd(struct perf_reader *reader, int fd);