Add build_id support for BPF stackmap

A separate build_id stackmap can be created with the help of new macro BPF_STACK_TRACE_BUILDID. The kernel BPF reports stacktrace in the 
structure bpf_stack_build_id. Changes have been made to BPF modules to
support resolving symbols mentioned in the above format.
An example tool is also available in examples/tracing/stack_buildid_example.py. 
Both python and C++ test cases are added.

examples/tracing/stack_buildid_example.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python
+#
+# An example usage of stack_build_id
+# Most of the code here is borrowed from tools/profile.py
+#
+# Steps for using this code
+# 1) Start ping program in one terminal eg invocation: ping google.com -i0.001
+# 2) Change the path of libc specified in b.add_module() below
+# 3) Invoke the script as 'python stack_buildid_example.py'
+# 4) o/p of the tool is as shown below
+# python example/tracing/stack_buildid_example.py
+# sendto
+# - ping (5232)
+# 2
+#
+# REQUIRES: Linux 4.17+ (BPF_BUILD_ID support)
+# Licensed under the Apache License, Version 2.0 (the "License")
+# 03-Jan-2019 Vijay Nag
+
+from __future__ import print_function
+from bcc import BPF, PerfType, PerfSWConfig
+from sys import stderr
+from time import sleep
+import argparse
+import signal
+import os
+import subprocess
+import errno
+import multiprocessing
+import ctypes as ct
+
+def Get_libc_path():
+ # A small helper function that returns full path
+ # of libc in the system
+ cmd = 'cat /proc/self/maps | grep libc | awk \'{print $6}\' | uniq'
+ output = subprocess.check_output(cmd, shell=True)
+ if not isinstance(output, str):
+ output = output.decode()
+ return output.split('\n')[0]
+
+bpf_text = """
+#include <uapi/linux/ptrace.h>
+#include <uapi/linux/bpf_perf_event.h>
+#include <linux/sched.h>
+
+struct key_t {
+ u32 pid;
+ int user_stack_id;
+ char name[TASK_COMM_LEN];
+};
+BPF_HASH(counts, struct key_t);
+BPF_STACK_TRACE_BUILDID(stack_traces, 128);
+
+int do_perf_event(struct bpf_perf_event_data *ctx) {
+ u32 pid = bpf_get_current_pid_tgid() >> 32;
+
+ // create map key
+ struct key_t key = {.pid = pid};
+ bpf_get_current_comm(&key.name, sizeof(key.name));
+
+ key.user_stack_id = stack_traces.get_stackid(&ctx->regs, BPF_F_USER_STACK);
+
+ if (key.user_stack_id >= 0) {
+ counts.increment(key);
+ }
+ return 0;
+}
+"""
+
+b = BPF(text=bpf_text)
+b.attach_perf_event(ev_type=PerfType.SOFTWARE,
+ ev_config=PerfSWConfig.CPU_CLOCK, fn_name="do_perf_event",
+ sample_period=0, sample_freq=49, cpu=0)
+
+# Add the list of libraries/executables to the build sym cache for sym resolution
+# Change the libc path if it is different on a different machine.
+# libc.so and ping are added here so that any symbols pertaining to
+# libc or ping are resolved. More executables/libraries can be added here.
+b.add_module(Get_libc_path())
+b.add_module("/usr/sbin/sshd")
+b.add_module("/bin/ping")
+counts = b.get_table("counts")
+stack_traces = b.get_table("stack_traces")
+duration = 2
+
+def signal_handler(signal, frame):
+ print()
+
+try:
+ sleep(duration)
+except KeyboardInterrupt:
+ # as cleanup can take some time, trap Ctrl-C:
+ signal.signal(signal.SIGINT, signal_ignore)
+
+user_stack=[]
+for k,v in sorted(counts.items(), key=lambda counts: counts[1].value):
+ user_stack = [] if k.user_stack_id < 0 else \
+ stack_traces.walk(k.user_stack_id)
+
+ user_stack=list(user_stack)
+ for addr in user_stack:
+ print(" %s" % b.sym(addr, k.pid).decode('utf-8', 'replace'))
+ print(" %-16s %s (%d)" % ("-", k.name.decode('utf-8', 'replace'), k.pid))
+ print(" %d\n" % v.value)
+

src/cc/api/BPF.cc

@@ -86,6 +86,8 @@ BPF::~BPF() {
  if (res.code() != 0)
  std::cerr << "Failed to detach all probes on destruction: " << std::endl
  << res.msg() << std::endl;
+ bcc_free_buildsymcache(bsymcache_);
+ bsymcache_ = NULL;
 }
 
 StatusTuple BPF::detach_all() {
@@ -650,6 +652,21 @@ BPFStackTable BPF::get_stack_table(const std::string& name, bool use_debug_file,
  return BPFStackTable({}, use_debug_file, check_debug_file_crc);
 }
 
+BPFStackBuildIdTable BPF::get_stackbuildid_table(const std::string &name, bool use_debug_file,
+ bool check_debug_file_crc) {
+ TableStorage::iterator it;
+
+ if (bpf_module_->table_storage().Find(Path({bpf_module_->id(), name}), it))
+ return BPFStackBuildIdTable(it->second, use_debug_file, check_debug_file_crc, get_bsymcache());
+ return BPFStackBuildIdTable({}, use_debug_file, check_debug_file_crc, get_bsymcache());
+}
+
+bool BPF::add_module(std::string module)
+{
+ return bcc_buildsymcache_add_module(get_bsymcache(), module.c_str()) != 0 ?
+ false : true;
+}
+
 std::string BPF::get_uprobe_event(const std::string& binary_path,
  uint64_t offset, bpf_probe_attach_type type,
  pid_t pid) {

src/cc/api/BPF.h

@@ -49,6 +49,7 @@ class BPF {
  explicit BPF(unsigned int flag = 0, TableStorage* ts = nullptr,
  bool rw_engine_enabled = bpf_module_rw_engine_enabled(), const std::string &maps_ns = "")
  : flag_(flag),
+ bsymcache_(NULL),
  bpf_module_(new BPFModule(flag, ts, rw_engine_enabled, maps_ns)) {}
  StatusTuple init(const std::string& bpf_program,
  const std::vector<std::string>& cflags = {},
@@ -137,6 +138,13 @@ class BPF {
  return BPFPercpuHashTable<KeyType, ValueType>({});
  }
 
+ void* get_bsymcache(void) {
+ if (bsymcache_ == NULL) {
+ bsymcache_ = bcc_buildsymcache_new();
+ }
+ return bsymcache_;
+ }
+
  BPFProgTable get_prog_table(const std::string& name);
 
  BPFCgroupArray get_cgroup_array(const std::string& name);
@@ -147,6 +155,12 @@ class BPF {
  bool use_debug_file = true,
  bool check_debug_file_crc = true);
 
+ BPFStackBuildIdTable get_stackbuildid_table(const std::string &name,
+ bool use_debug_file = true,
+ bool check_debug_file_crc = true);
+
+ bool add_module(std::string module);
+
  StatusTuple open_perf_event(const std::string& name, uint32_t type,
  uint64_t config);
 
@@ -225,6 +239,8 @@ class BPF {
 
  int flag_;
 
+ void *bsymcache_;
+
  std::unique_ptr<std::string> syscall_prefix_;
 
  std::unique_ptr<BPFModule> bpf_module_;

src/cc/api/BPFTable.cc

@@ -316,6 +316,69 @@ std::vector<std::string> BPFStackTable::get_stack_symbol(int stack_id,
  return res;
 }
 
+BPFStackBuildIdTable::BPFStackBuildIdTable(const TableDesc& desc, bool use_debug_file,
+ bool check_debug_file_crc,
+ void *bsymcache)
+ : BPFTableBase<int, stacktrace_buildid_t>(desc),
+ bsymcache_(bsymcache) {
+ if (desc.type != BPF_MAP_TYPE_STACK_TRACE)
+ throw std::invalid_argument("Table '" + desc.name +
+ "' is not a stack table");
+
+ symbol_option_ = {.use_debug_file = use_debug_file,
+ .check_debug_file_crc = check_debug_file_crc,
+ .use_symbol_type = (1 << STT_FUNC) | (1 << STT_GNU_IFUNC)};
+}
+
+void BPFStackBuildIdTable::clear_table_non_atomic() {
+ for (int i = 0; size_t(i) < capacity(); i++) {
+ remove(&i);
+ }
+}
+
+std::vector<bpf_stack_build_id> BPFStackBuildIdTable::get_stack_addr(int stack_id) {
+ std::vector<bpf_stack_build_id> res;
+ struct stacktrace_buildid_t stack;
+ if (stack_id < 0)
+ return res;
+ if (!lookup(&stack_id, &stack))
+ return res;
+ for (int i = 0; (i < BPF_MAX_STACK_DEPTH) && \
+ (stack.trace[i].status == BPF_STACK_BUILD_ID_VALID);
+ i++) {
+ /* End of stack marker is BCC_STACK_BUILD_ID_EMPTY or
+ * BCC_STACK_BUILD_IP(fallback) mechanism.
+ * We do not support fallback mechanism
+ */
+ res.push_back(stack.trace[i]);
+ }
+ return res;
+}
+
+std::vector<std::string> BPFStackBuildIdTable::get_stack_symbol(int stack_id)
+{
+ auto addresses = get_stack_addr(stack_id);
+ std::vector<std::string> res;
+ if (addresses.empty())
+ return res;
+ res.reserve(addresses.size());
+
+ bcc_symbol symbol;
+ struct bpf_stack_build_id trace;
+ for (auto addr : addresses) {
+ memcpy(trace.build_id, addr.build_id, sizeof(trace.build_id));
+ trace.status = addr.status;
+ trace.offset = addr.offset;
+ if (bcc_buildsymcache_resolve(bsymcache_,&trace,&symbol) != 0) {
+ res.emplace_back("[UNKNOWN]");
+ } else {
+ res.push_back(symbol.name);
+ bcc_symbol_free_demangle_name(&symbol);
+ }
+ }
+ return res;
+}
+
 BPFPerfBuffer::BPFPerfBuffer(const TableDesc& desc)
  : BPFTableBase<int, int>(desc), epfd_(-1) {
  if (desc.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)

src/cc/api/BPFTable.h

@@ -315,6 +315,26 @@ class BPFStackTable : public BPFTableBase<int, stacktrace_t> {
  std::map<int, void*> pid_sym_;
 };
 
+// from src/cc/export/helpers.h
+struct stacktrace_buildid_t {
+ struct bpf_stack_build_id trace[BPF_MAX_STACK_DEPTH];
+};
+
+class BPFStackBuildIdTable : public BPFTableBase<int, stacktrace_buildid_t> {
+ public:
+ BPFStackBuildIdTable(const TableDesc& desc, bool use_debug_file,
+ bool check_debug_file_crc, void *bsymcache);
+ ~BPFStackBuildIdTable() = default;
+
+ void clear_table_non_atomic();
+ std::vector<bpf_stack_build_id> get_stack_addr(int stack_id);
+ std::vector<std::string> get_stack_symbol(int stack_id);
+
+ private:
+ void *bsymcache_;
+ bcc_symbol_option symbol_option_;
+};
+
 class BPFPerfBuffer : public BPFTableBase<int, int> {
  public:
  BPFPerfBuffer(const TableDesc& desc);

src/cc/bcc_elf.c

@@ -882,6 +882,20 @@ int bcc_free_memory() {
  return err;
 }
 
+int bcc_elf_get_buildid(const char *path, char *buildid)
+{
+ Elf *e;
+ int fd;
+
+ if (openelf(path, &e, &fd) < 0)
+ return -1;
+
+ if (!find_buildid(e, buildid))
+ return -1;
+
+ return 0;
+}
+
 #if 0
 #include <stdio.h>
 

src/cc/bcc_elf.h

@@ -69,6 +69,7 @@ int bcc_elf_is_shared_obj(const char *path);
 int bcc_elf_is_exe(const char *path);
 int bcc_elf_is_vdso(const char *name);
 int bcc_free_memory();
+int bcc_elf_get_buildid(const char *path, char *buildid);
 
 #ifdef __cplusplus
 }