Merge pull request iovisor#3135 from danobi/uprobe_ref_cnt

uprobe: Add ref_cnt_offset arg to bpf_attach_uprobe

src/cc/api/BPF.cc

@@ -20,9 +20,12 @@
 #include <cstdio>
 #include <cstring>
 #include <exception>
+#include <fcntl.h>
 #include <iostream>
 #include <memory>
 #include <sstream>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <utility>
 #include <vector>
 
@@ -39,6 +42,37 @@
 
 #include "BPF.h"
 
+namespace {
+/*
+ * Kernels ~4.20 and later support specifying the ref_ctr_offset as an argument
+ * to attaching a uprobe, which negates the need to seek to this memory offset
+ * in userspace to manage semaphores, as the kernel will do it for us. This
+ * helper function checks if this support is available by reading the uprobe
+ * format for this value, added in a6ca88b241d5e929e6e60b12ad8cd288f0ffa
+*/
+bool uprobe_ref_ctr_supported() {
+ const char *ref_ctr_pmu_path =
+ "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset";
+ const char *ref_ctr_pmu_expected = "config:32-63\0";
+ char ref_ctr_pmu_fmt[64]; // in Linux source this buffer is compared vs
+ // PAGE_SIZE, but 64 is probably ample
+ int fd = open(ref_ctr_pmu_path, O_RDONLY);
+ if (fd < 0)
+ return false;
+
+ int ret = read(fd, ref_ctr_pmu_fmt, sizeof(ref_ctr_pmu_fmt));
+ close(fd);
+ if (ret < 0) {
+ return false;
+ }
+ if (strncmp(ref_ctr_pmu_expected, ref_ctr_pmu_fmt,
+ strlen(ref_ctr_pmu_expected)) == 0) {
+ return true;
+ }
+ return false;
+}
+} // namespace
+
 namespace ebpf {
 
 std::string uint_to_hex(uint64_t value) {
@@ -225,7 +259,8 @@ StatusTuple BPF::attach_uprobe(const std::string& binary_path,
  const std::string& probe_func,
  uint64_t symbol_addr,
  bpf_probe_attach_type attach_type, pid_t pid,
- uint64_t symbol_offset) {
+ uint64_t symbol_offset,
+ uint32_t ref_ctr_offset) {
 
  if (symbol_addr != 0 && symbol_offset != 0)
  return StatusTuple(-1,
@@ -245,7 +280,8 @@ StatusTuple BPF::attach_uprobe(const std::string& binary_path,
  TRY2(load_func(probe_func, BPF_PROG_TYPE_KPROBE, probe_fd));
 
  int res_fd = bpf_attach_uprobe(probe_fd, attach_type, probe_event.c_str(),
- binary_path.c_str(), offset, pid);
+ binary_path.c_str(), offset, pid,
+ ref_ctr_offset);
 
  if (res_fd < 0) {
  TRY2(unload_func(probe_func));
@@ -266,15 +302,16 @@ StatusTuple BPF::attach_uprobe(const std::string& binary_path,
 
 StatusTuple BPF::attach_usdt_without_validation(const USDT& u, pid_t pid) {
  auto& probe = *static_cast<::USDT::Probe*>(u.probe_.get());
- if (!probe.enable(u.probe_func_))
+ if (!uprobe_ref_ctr_supported() && !probe.enable(u.probe_func_))
  return StatusTuple(-1, "Unable to enable USDT %s" + u.print_name());
 
  bool failed = false;
  std::string err_msg;
  int cnt = 0;
  for (const auto& loc : probe.locations_) {
  auto res = attach_uprobe(loc.bin_path_, std::string(), u.probe_func_,
- loc.address_, BPF_PROBE_ENTRY, pid);
+ loc.address_, BPF_PROBE_ENTRY, pid, 0,
+ probe.semaphore_offset());
  if (!res.ok()) {
  failed = true;
  err_msg += "USDT " + u.print_name() + " at " + loc.bin_path_ +
@@ -505,7 +542,7 @@ StatusTuple BPF::detach_usdt_without_validation(const USDT& u, pid_t pid) {
  }
  }
 
- if (!probe.disable()) {
+ if (!uprobe_ref_ctr_supported() && !probe.disable()) {
  failed = true;
  err_msg += "Unable to disable USDT " + u.print_name();
  }

src/cc/api/BPF.h

@@ -78,7 +78,8 @@ class BPF {
  uint64_t symbol_addr = 0,
  bpf_probe_attach_type attach_type = BPF_PROBE_ENTRY,
  pid_t pid = -1,
- uint64_t symbol_offset = 0);
+ uint64_t symbol_offset = 0,
+ uint32_t ref_ctr_offset = 0);
  StatusTuple detach_uprobe(const std::string& binary_path,
  const std::string& symbol, uint64_t symbol_addr = 0,
  bpf_probe_attach_type attach_type = BPF_PROBE_ENTRY,

src/cc/bcc_elf.c

@@ -58,6 +58,7 @@ static int openelf(const char *path, Elf **elf_out, int *fd_out) {
 }
 
 static const char *parse_stapsdt_note(struct bcc_elf_usdt *probe,
+ GElf_Shdr *probes_shdr,
  const char *desc, int elf_class) {
  if (elf_class == ELFCLASS32) {
  probe->pc = *((uint32_t *)(desc));
@@ -71,6 +72,13 @@ static const char *parse_stapsdt_note(struct bcc_elf_usdt *probe,
  desc = desc + 24;
  }
 
+ // Offset from start of file
+ if (probe->semaphore && probes_shdr)
+ probe->semaphore_offset =
+ probe->semaphore - probes_shdr->sh_addr + probes_shdr->sh_offset;
+ else
+ probe->semaphore_offset = 0;
+
  probe->provider = desc;
  desc += strlen(desc) + 1;
 
@@ -83,7 +91,7 @@ static const char *parse_stapsdt_note(struct bcc_elf_usdt *probe,
  return desc;
 }
 
-static int do_note_segment(Elf_Scn *section, int elf_class,
+static int do_note_segment(Elf_Scn *section, GElf_Shdr *probes_shdr, int elf_class,
  bcc_elf_probecb callback, const char *binpath,
  uint64_t first_inst_offset, void *payload) {
  Elf_Data *data = NULL;
@@ -110,7 +118,7 @@ static int do_note_segment(Elf_Scn *section, int elf_class,
  desc = (const char *)data->d_buf + desc_off;
  desc_end = desc + hdr.n_descsz;
 
- if (parse_stapsdt_note(&probe, desc, elf_class) == desc_end) {
+ if (parse_stapsdt_note(&probe, probes_shdr, desc, elf_class) == desc_end) {
  if (probe.pc < first_inst_offset)
  fprintf(stderr,
  "WARNING: invalid address 0x%lx for probe (%s,%s) in binary %s\n",
@@ -126,9 +134,11 @@ static int do_note_segment(Elf_Scn *section, int elf_class,
 static int listprobes(Elf *e, bcc_elf_probecb callback, const char *binpath,
  void *payload) {
  Elf_Scn *section = NULL;
+ bool found_probes_shdr;
  size_t stridx;
  int elf_class = gelf_getclass(e);
  uint64_t first_inst_offset = 0;
+ GElf_Shdr probes_shdr = {};
 
  if (elf_getshdrstrndx(e, &stridx) != 0)
  return -1;
@@ -148,6 +158,18 @@ static int listprobes(Elf *e, bcc_elf_probecb callback, const char *binpath,
  }
  }
 
+ found_probes_shdr = false;
+ while ((section = elf_nextscn(e, section)) != 0) {
+ if (!gelf_getshdr(section, &probes_shdr))
+ continue;
+
+ char *name = elf_strptr(e, stridx, probes_shdr.sh_name);
+ if (name && !strcmp(name, ".probes")) {
+ found_probes_shdr = true;
+ break;
+ }
+ }
+
  while ((section = elf_nextscn(e, section)) != 0) {
  GElf_Shdr header;
  char *name;
@@ -160,7 +182,8 @@ static int listprobes(Elf *e, bcc_elf_probecb callback, const char *binpath,
 
  name = elf_strptr(e, stridx, header.sh_name);
  if (name && !strcmp(name, ".note.stapsdt")) {
- if (do_note_segment(section, elf_class, callback, binpath,
+ GElf_Shdr *shdr_ptr = found_probes_shdr ? &probes_shdr : NULL;
+ if (do_note_segment(section, shdr_ptr, elf_class, callback, binpath,
  first_inst_offset, payload) < 0)
  return -1;
  }

src/cc/bcc_elf.h

@@ -26,11 +26,15 @@ extern "C" {
 struct bcc_elf_usdt {
  uint64_t pc;
  uint64_t base_addr;
+ // Virtual address semaphore is found at
  uint64_t semaphore;
 
  const char *provider;
  const char *name;
  const char *arg_fmt;
+
+ // Offset from start of file where the semaphore is at
+ uint64_t semaphore_offset;
 };
 
 // Binary module path, bcc_elf_usdt struct, payload

src/cc/bcc_usdt.h

@@ -30,9 +30,12 @@ struct bcc_usdt {
  const char *provider;
  const char *name;
  const char *bin_path;
+ // Virtual address semaphore is found at
  uint64_t semaphore;
  int num_locations;
  int num_arguments;
+ // Offset from start of file where semaphore is at
+ uint64_t semaphore_offset;
 };
 
 struct bcc_usdt_location {

src/cc/libbpf.c

@@ -91,6 +91,8 @@
 
 #define UNUSED(expr) do { (void)(expr); } while (0)
 
+#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
+
 struct bpf_helper {
  char *name;
  char *required_version;
@@ -838,7 +840,8 @@ static int bpf_get_retprobe_bit(const char *event_type)
  * the [k,u]probe. This function tries to create pfd with the perf_kprobe PMU.
  */
 static int bpf_try_perf_event_open_with_probe(const char *name, uint64_t offs,
- int pid, const char *event_type, int is_return)
+ int pid, const char *event_type, int is_return,
+ uint64_t ref_ctr_offset)
 {
  struct perf_event_attr attr = {};
  int type = bpf_find_probe_type(event_type);
@@ -851,6 +854,7 @@ static int bpf_try_perf_event_open_with_probe(const char *name, uint64_t offs,
  attr.wakeup_events = 1;
  if (is_return)
  attr.config |= 1 << is_return_bit;
+ attr.config |= (ref_ctr_offset << PERF_UPROBE_REF_CTR_OFFSET_SHIFT);
 
  /*
  * struct perf_event_attr in latest perf_event.h has the following
@@ -1019,7 +1023,8 @@ static int create_probe_event(char *buf, const char *ev_name,
 // see bpf_try_perf_event_open_with_probe().
 static int bpf_attach_probe(int progfd, enum bpf_probe_attach_type attach_type,
  const char *ev_name, const char *config1, const char* event_type,
- uint64_t offset, pid_t pid, int maxactive)
+ uint64_t offset, pid_t pid, int maxactive,
+ uint32_t ref_ctr_offset)
 {
  int kfd, pfd = -1;
  char buf[PATH_MAX], fname[256];
@@ -1028,7 +1033,8 @@ static int bpf_attach_probe(int progfd, enum bpf_probe_attach_type attach_type,
  if (maxactive <= 0)
  // Try create the [k,u]probe Perf Event with perf_event_open API.
  pfd = bpf_try_perf_event_open_with_probe(config1, offset, pid, event_type,
- attach_type != BPF_PROBE_ENTRY);
+ attach_type != BPF_PROBE_ENTRY,
+ ref_ctr_offset);
 
  // If failed, most likely Kernel doesn't support the perf_kprobe PMU
  // (e12f03d "perf/core: Implement the 'perf_kprobe' PMU") yet.
@@ -1092,17 +1098,17 @@ int bpf_attach_kprobe(int progfd, enum bpf_probe_attach_type attach_type,
 {
  return bpf_attach_probe(progfd, attach_type,
  ev_name, fn_name, "kprobe",
- fn_offset, -1, maxactive);
+ fn_offset, -1, maxactive, 0);
 }
 
 int bpf_attach_uprobe(int progfd, enum bpf_probe_attach_type attach_type,
  const char *ev_name, const char *binary_path,
- uint64_t offset, pid_t pid)
+ uint64_t offset, pid_t pid, uint32_t ref_ctr_offset)
 {
 
  return bpf_attach_probe(progfd, attach_type,
  ev_name, binary_path, "uprobe",
- offset, pid, -1);
+ offset, pid, -1, ref_ctr_offset);
 }
 
 static int bpf_detach_probe(const char *ev_name, const char *event_type)

src/cc/libbpf.h

@@ -86,7 +86,7 @@ int bpf_detach_kprobe(const char *ev_name);
 
 int bpf_attach_uprobe(int progfd, enum bpf_probe_attach_type attach_type,
  const char *ev_name, const char *binary_path,
- uint64_t offset, pid_t pid);
+ uint64_t offset, pid_t pid, uint32_t ref_ctr_offset);
 int bpf_detach_uprobe(const char *ev_name);
 
 int bpf_attach_tracepoint(int progfd, const char *tp_category,

src/cc/usdt.h

@@ -194,6 +194,7 @@ class Probe {
  std::string provider_;
  std::string name_;
  uint64_t semaphore_;
+ uint64_t semaphore_offset_;
 
  std::vector<Location> locations_;
 
@@ -214,11 +215,13 @@ class Probe {
 
 public:
  Probe(const char *bin_path, const char *provider, const char *name,
- uint64_t semaphore, const optional<int> &pid, uint8_t mod_match_inode_only = 1);
+ uint64_t semaphore, uint64_t semaphore_offset,
+ const optional<int> &pid, uint8_t mod_match_inode_only = 1);
 
  size_t num_locations() const { return locations_.size(); }
  size_t num_arguments() const { return locations_.front().arguments_.size(); }
  uint64_t semaphore() const { return semaphore_; }
+ uint64_t semaphore_offset() const { return semaphore_offset_; }
 
  uint64_t address(size_t n = 0) const { return locations_[n].address_; }
  const char *location_bin_path(size_t n = 0) const { return locations_[n].bin_path_.c_str(); }

src/cc/usdt/usdt.cc

@@ -54,12 +54,13 @@ Location::Location(uint64_t addr, const std::string &bin_path, const char *arg_f
 }
 
 Probe::Probe(const char *bin_path, const char *provider, const char *name,
- uint64_t semaphore, const optional<int> &pid,
- uint8_t mod_match_inode_only)
+ uint64_t semaphore, uint64_t semaphore_offset,
+ const optional<int> &pid, uint8_t mod_match_inode_only)
  : bin_path_(bin_path),
  provider_(provider),
  name_(name),
  semaphore_(semaphore),
+ semaphore_offset_(semaphore_offset),
  pid_(pid),
  mod_match_inode_only_(mod_match_inode_only)
  {}
@@ -262,8 +263,8 @@ void Context::add_probe(const char *binpath, const struct bcc_elf_usdt *probe) {
  }
 
  probes_.emplace_back(
- new Probe(binpath, probe->provider, probe->name, probe->semaphore, pid_,
- mod_match_inode_only_)
+ new Probe(binpath, probe->provider, probe->name, probe->semaphore,
+ probe->semaphore_offset, pid_, mod_match_inode_only_)
  );
  probes_.back()->add_location(probe->pc, binpath, probe->arg_fmt);
 }
@@ -347,6 +348,7 @@ void Context::each(each_cb callback) {
  info.bin_path = probe->bin_path().c_str();
  info.name = probe->name().c_str();
  info.semaphore = probe->semaphore();
+ info.semaphore_offset = probe->semaphore_offset();
  info.num_locations = probe->num_locations();
  info.num_arguments = probe->num_arguments();
  callback(&info);

tests/cc/CMakeLists.txt

@@ -38,7 +38,7 @@ set(TEST_LIBBCC_SOURCES
 add_executable(test_libbcc ${TEST_LIBBCC_SOURCES})
 
 file(COPY dummy_proc_map.txt DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-add_library(usdt_test_lib SHARED usdt_test_lib.c)
+add_library(usdt_test_lib SHARED usdt_test_lib.cc)
 
 add_dependencies(test_libbcc bcc-shared)
 target_link_libraries(test_libbcc ${PROJECT_BINARY_DIR}/src/cc/libbcc.so dl usdt_test_lib)