Improve ProcSyms module path handling

Besides loading modules from /proc/<pid>/root/<path>, open /proc/<pid>/root
in advance with open(..., O_PATH) first, then use openat to read the module file
in case /proc/<pid>/root/<path> cannot be accessed.
This enables the module file to be read even if <pid> is not running.

A class named ModulePath is added, which does the openat call and manages the
returned file descriptor, enabling functions expecting a path to use
/proc/self/fd/... as the path to the module.

This approach has a few limitations. Code using the module path for something
different than open (e.g. debug info in .debug file or perf map detection) does not
work with the /proc/self/fd/... path, and the accessibility check of
/proc/<pid>/root/<path> is prone to a race condition.

src/cc/bcc_syms.cc

@@ -14,25 +14,53 @@
  * limitations under the License.
  */
 
+#include "bcc_syms.h"
+
 #include <cxxabi.h>
-#include <cstring>
 #include <fcntl.h>
+#include <limits.h>
 #include <linux/elf.h>
+#include <signal.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/sysmacros.h>
 #include <sys/types.h>
 #include <unistd.h>
+
 #include <cstdio>
+#include <cstring>
 
 #include "bcc_elf.h"
 #include "bcc_perf_map.h"
 #include "bcc_proc.h"
-#include "bcc_syms.h"
 #include "common.h"
+#include "syms.h"
 #include "vendor/tinyformat.hpp"
 
-#include "syms.h"
+ProcSyms::ModulePath::ModulePath(const std::string &ns_path, int root_fd,
+ int pid, bool enter_ns) {
+ if (!enter_ns) {
+ path_ = ns_path;
+ proc_root_path_ = ns_path;
+ return;
+ }
+ proc_root_path_ = tfm::format("/proc/%d/root%s", pid, ns_path);
+ // filename for openat must not contain any starting slashes, otherwise
+ // it would get treated as an absolute path
+ std::string trimmed_path;
+ size_t non_slash_pos;
+ for (non_slash_pos = 0;
+ non_slash_pos < ns_path.size() && ns_path[non_slash_pos] == '/';
+ non_slash_pos++)
+ ;
+ trimmed_path = ns_path.substr(non_slash_pos);
+ fd_ = openat(root_fd, trimmed_path.c_str(), O_RDONLY);
+ if (fd_ > 0)
+ path_ = tfm::format("/proc/self/fd/%d", fd_);
+ else
+ // openat failed, fall back to /proc/.../root path
+ path_ = proc_root_path_;
+}
 
 bool ProcStat::getinode_(ino_t &inode) {
  struct stat s;
@@ -44,13 +72,45 @@ bool ProcStat::getinode_(ino_t &inode) {
  }
 }
 
+bool ProcStat::refresh_root() {
+ // try to current root and mount namespace for process
+ char current_root[PATH_MAX], current_mount_ns[PATH_MAX];
+ if (readlink(root_symlink_.c_str(), current_root, PATH_MAX) < 0 ||
+ readlink(mount_ns_symlink_.c_str(), current_mount_ns, PATH_MAX) < 0)
+ // readlink failed, process might not exist anymore; keep old fd
+ return false;
+
+ // check if root fd is up to date
+ if (root_fd_ != -1 && current_root == root_ && current_mount_ns == mount_ns_)
+ return false;
+
+ root_ = current_root;
+ mount_ns_ = current_mount_ns;
+
+ // either root fd is invalid or process root and/or mount namespace changed;
+ // re-open root note: when /proc/.../root changes, the open file descriptor
+ // still refers to the old one
+ int original_root_fd = root_fd_;
+ root_fd_ = open(root_symlink_.c_str(), O_PATH);
+ if (root_fd_ == -1)
+ std::cerr << "Opening " << root_symlink_ << " failed: " << strerror(errno)
+ << std::endl;
+ if (original_root_fd > 0)
+ close(original_root_fd);
+ return original_root_fd != root_fd_;
+}
+
 bool ProcStat::is_stale() {
  ino_t cur_inode;
- return getinode_(cur_inode) && (cur_inode != inode_);
+ return getinode_(cur_inode) && (cur_inode != inode_) && refresh_root();
 }
 
-ProcStat::ProcStat(int pid) : procfs_(tfm::format("/proc/%d/exe", pid)) {
+ProcStat::ProcStat(int pid)
+ : procfs_(tfm::format("/proc/%d/exe", pid)),
+ root_symlink_(tfm::format("/proc/%d/root", pid)),
+ mount_ns_symlink_(tfm::format("/proc/%d/ns/mnt", pid)) {
  getinode_(inode_);
+ refresh_root();
 }
 
 void KSyms::_add_symbol(const char *symname, const char *modname, uint64_t addr, void *p) {
@@ -134,8 +194,9 @@ void ProcSyms::refresh() {
 
 int ProcSyms::_add_module(mod_info *mod, int enter_ns, void *payload) {
  ProcSyms *ps = static_cast<ProcSyms *>(payload);
- std::string ns_relative_path = tfm::format("/proc/%d/root%s", ps->pid_, mod->name);
- const char *modpath = enter_ns && ps->pid_ != -1 ? ns_relative_path.c_str() : mod->name;
+ std::shared_ptr<ModulePath> modpath =
+ std::make_shared<ModulePath>(mod->name, ps->procstat_.get_root_fd(),
+ ps->pid_, enter_ns && ps->pid_ != -1);
  auto it = std::find_if(
  ps->modules_.begin(), ps->modules_.end(),
  [=](const ProcSyms::Module &m) { return m.name_ == mod->name; });
@@ -147,14 +208,17 @@ int ProcSyms::_add_module(mod_info *mod, int enter_ns, void *payload) {
  // It only gives the mmap offset. We need the real offset for symbol
  // lookup.
  if (module.type_ == ModuleType::SO) {
- if (bcc_elf_get_text_scn_info(modpath, &module.elf_so_addr_,
+ if (bcc_elf_get_text_scn_info(modpath->path(), &module.elf_so_addr_,
  &module.elf_so_offset_) < 0) {
- fprintf(stderr, "WARNING: Couldn't find .text section in %s\n", modpath);
- fprintf(stderr, "WARNING: BCC can't handle sym look ups for %s", modpath);
+ fprintf(stderr, "WARNING: Couldn't find .text section in %s\n",
+ modpath->alt_path());
+ fprintf(stderr, "WARNING: BCC can't handle sym look ups for %s",
+ modpath->alt_path());
  }
  }
 
- if (!bcc_is_perf_map(modpath) || module.type_ != ModuleType::UNKNOWN)
+ if (!bcc_is_perf_map(modpath->path()) ||
+ module.type_ != ModuleType::UNKNOWN)
  // Always add the module even if we can't read it, so that we could
  // report correct module name. Unless it's a perf map that we only
  // add readable ones.
@@ -225,14 +289,14 @@ bool ProcSyms::resolve_name(const char *module, const char *name,
  return false;
 }
 
-ProcSyms::Module::Module(const char *name, const char *path,
- struct bcc_symbol_option *option)
+ProcSyms::Module::Module(const char *name, std::shared_ptr<ModulePath> path,
+  struct bcc_symbol_option *option)
  : name_(name),
  path_(path),
  loaded_(false),
  symbol_option_(option),
  type_(ModuleType::UNKNOWN) {
- int elf_type = bcc_elf_get_type(path_.c_str());
+ int elf_type = bcc_elf_get_type(path_->path());
  // The Module is an ELF file
  if (elf_type >= 0) {
  if (elf_type == ET_EXEC)
@@ -242,9 +306,9 @@ ProcSyms::Module::Module(const char *name, const char *path,
  return;
  }
  // Other symbol files
- if (bcc_is_valid_perf_map(path_.c_str()) == 1)
+ if (bcc_is_valid_perf_map(path_->path()) == 1)
  type_ = ModuleType::PERF_MAP;
- else if (bcc_elf_is_vdso(name_.c_str()) == 1)
+ else if (bcc_elf_is_vdso(path_->path()) == 1)
  type_ = ModuleType::VDSO;
 
  // Will be stored later
@@ -278,12 +342,13 @@ void ProcSyms::Module::load_sym_table() {
  return;
 
  if (type_ == ModuleType::PERF_MAP)
- bcc_perf_map_foreach_sym(path_.c_str(), _add_symbol, this);
+ bcc_perf_map_foreach_sym(path_->path(), _add_symbol, this);
  if (type_ == ModuleType::EXEC || type_ == ModuleType::SO) {
  if (symbol_option_->lazy_symbolize)
- bcc_elf_foreach_sym_lazy(path_.c_str(), _add_symbol_lazy, symbol_option_, this);
+ bcc_elf_foreach_sym_lazy(path_->path(), _add_symbol_lazy, symbol_option_,
+ this);
  else
- bcc_elf_foreach_sym(path_.c_str(), _add_symbol, symbol_option_, this);
+ bcc_elf_foreach_sym(path_->path(), _add_symbol, symbol_option_, this);
  }
  if (type_ == ModuleType::VDSO)
  bcc_elf_foreach_vdso_sym(_add_symbol, this);
@@ -333,9 +398,13 @@ bool ProcSyms::Module::find_name(const char *symname, uint64_t *addr) {
  };
 
  if (type_ == ModuleType::PERF_MAP)
- bcc_perf_map_foreach_sym(path_.c_str(), cb, &payload);
- if (type_ == ModuleType::EXEC || type_ == ModuleType::SO)
- bcc_elf_foreach_sym(path_.c_str(), cb, symbol_option_, &payload);
+ bcc_perf_map_foreach_sym(path_->path(), cb, &payload);
+ if (type_ == ModuleType::EXEC || type_ == ModuleType::SO) {
+ bcc_elf_foreach_sym(path_->path(), cb, symbol_option_, &payload);
+ if (path_->path() != path_->alt_path())
+ // some features (e.g. some kinds of debug info) don't work with /proc/self/fd/... path
+ bcc_elf_foreach_sym(path_->alt_path(), cb, symbol_option_, &payload);
+ }
  if (type_ == ModuleType::VDSO)
  bcc_elf_foreach_vdso_sym(cb, &payload);
 
@@ -385,9 +454,9 @@ bool ProcSyms::Module::find_addr(uint64_t offset, struct bcc_symbol *sym) {
  // Resolve and cache the symbol name if necessary
  if (!it->is_name_resolved) {
  std::string sym_name(it->data.name_idx.str_len + 1, '\0');
- if (bcc_elf_symbol_str(path_.c_str(), it->data.name_idx.section_idx,
- it->data.name_idx.str_table_idx, &sym_name[0], sym_name.size(),
- it->data.name_idx.debugfile))
+ if (bcc_elf_symbol_str(path_->path(), it->data.name_idx.section_idx,
+  it->data.name_idx.str_table_idx, &sym_name[0],
+  sym_name.size(), it->data.name_idx.debugfile))
  break;
 
  it->data.name = &*(symnames_.emplace(std::move(sym_name)).first);

src/cc/syms.h

@@ -29,11 +29,25 @@
 
 class ProcStat {
  std::string procfs_;
+ std::string root_symlink_;
+ std::string mount_ns_symlink_;
+ // file descriptor of /proc/<pid>/root open with O_PATH used to get into root
+ // of process after it exits; unlike a dereferenced root symlink, *at calls
+ // to this use the process's mount namespace
+ int root_fd_ = -1;
+ // store also root path and mount namespace pair to detect its changes
+ std::string root_, mount_ns_;
  ino_t inode_;
  bool getinode_(ino_t &inode);
 
  public:
  ProcStat(int pid);
+ ~ProcStat() {
+ if (root_fd_ > 0)
+ close(root_fd_);
+ }
+ bool refresh_root();
+ int get_root_fd() { return root_fd_; }
  bool is_stale();
  void reset() { getinode_(inode_); }
 };
@@ -111,6 +125,30 @@ class ProcSyms : SymbolCache {
  VDSO
  };
 
+ class ModulePath {
+ // helper class to get a usable module path independent of the running
+ // process by storing a file descriptor created from openat(2) if possible
+ // if openat fails, falls back to process-dependent path with /proc/.../root
+ private:
+ int fd_;
+ std::string proc_root_path_;
+ std::string path_;
+
+ public:
+ ModulePath(const std::string &ns_path, int root_fd, int pid, bool enter_ns);
+ const char *alt_path() { return proc_root_path_.c_str(); }
+ const char *path() {
+ if (path_ == proc_root_path_ || access(proc_root_path_.c_str(), F_OK) < 0)
+ // cannot stat /proc/.../root/<path>, pid might not exist anymore; use /proc/self/fd/...
+ return path_.c_str();
+ return proc_root_path_.c_str();
+ }
+ ~ModulePath() {
+ if (fd_ > 0)
+ close(fd_);
+ }
+ };
+
  struct Module {
  struct Range {
  uint64_t start;
@@ -120,10 +158,11 @@ class ProcSyms : SymbolCache {
  : start(s), end(e), file_offset(f) {}
  };
 
- Module(const char *name, const char *path, struct bcc_symbol_option *option);
+ Module(const char *name, std::shared_ptr<ModulePath> path,
+ struct bcc_symbol_option *option);
 
  std::string name_;
- std::string path_;
+ std::shared_ptr<ModulePath> path_;
  std::vector<Range> ranges_;
  bool loaded_;
  bcc_symbol_option *symbol_option_;

tests/cc/test_c_api.cc

@@ -317,6 +317,103 @@ TEST_CASE("resolve symbol addresses for a given PID", "[c_api]") {
  bcc_free_symcache(lazy_resolver, getpid());
 }
 
+TEST_CASE("resolve symbol addresses for an exited process", "[c-api]") {
+ struct bcc_symbol sym;
+ struct bcc_symbol lazy_sym;
+ static struct bcc_symbol_option lazy_opt {
+ .use_debug_file = 1, .check_debug_file_crc = 1, .lazy_symbolize = 1,
+#if defined(__powerpc64__) && defined(_CALL_ELF) && _CALL_ELF == 2
+ .use_symbol_type = BCC_SYM_ALL_TYPES | (1 << STT_PPC64_ELFV2_SYM_LEP),
+#else
+ .use_symbol_type = BCC_SYM_ALL_TYPES,
+#endif
+ };
+
+ SECTION("resolve in current namespace") {
+ pid_t child = spawn_child(nullptr, false, false, [](void *) {
+ sleep(5);
+ return 0;
+ });
+ void *resolver = bcc_symcache_new(child, nullptr);
+ void *lazy_resolver = bcc_symcache_new(child, &lazy_opt);
+
+ REQUIRE(resolver);
+ REQUIRE(lazy_resolver);
+
+ kill(child, SIGTERM);
+
+ REQUIRE(bcc_symcache_resolve(resolver, (uint64_t)&_a_test_function, &sym) ==
+ 0);
+
+ char *this_exe = realpath("/proc/self/exe", NULL);
+ REQUIRE(string(this_exe) == sym.module);
+ free(this_exe);
+
+ REQUIRE(string("_a_test_function") == sym.name);
+
+ REQUIRE(bcc_symcache_resolve(lazy_resolver, (uint64_t)&_a_test_function,
+ &lazy_sym) == 0);
+ REQUIRE(string(lazy_sym.name) == sym.name);
+ REQUIRE(string(lazy_sym.module) == sym.module);
+ }
+
+ SECTION("resolve in separate pid namespace") {
+ pid_t child = spawn_child(nullptr, true, false, [](void *) {
+ sleep(5);
+ return 0;
+ });
+ void *resolver = bcc_symcache_new(child, nullptr);
+ void *lazy_resolver = bcc_symcache_new(child, &lazy_opt);
+
+ REQUIRE(resolver);
+ REQUIRE(lazy_resolver);
+
+ kill(child, SIGTERM);
+
+ REQUIRE(bcc_symcache_resolve(resolver, (uint64_t)&_a_test_function, &sym) ==
+ 0);
+
+ char *this_exe = realpath("/proc/self/exe", NULL);
+ REQUIRE(string(this_exe) == sym.module);
+ free(this_exe);
+
+ REQUIRE(string("_a_test_function") == sym.name);
+
+ REQUIRE(bcc_symcache_resolve(lazy_resolver, (uint64_t)&_a_test_function,
+ &lazy_sym) == 0);
+ REQUIRE(string(lazy_sym.name) == sym.name);
+ REQUIRE(string(lazy_sym.module) == sym.module);
+ }
+
+ SECTION("resolve in separate pid and mount namespace") {
+ pid_t child = spawn_child(nullptr, true, true, [](void *) {
+ sleep(5);
+ return 0;
+ });
+ void *resolver = bcc_symcache_new(child, nullptr);
+ void *lazy_resolver = bcc_symcache_new(child, &lazy_opt);
+
+ REQUIRE(resolver);
+ REQUIRE(lazy_resolver);
+
+ kill(child, SIGTERM);
+
+ REQUIRE(bcc_symcache_resolve(resolver, (uint64_t)&_a_test_function, &sym) ==
+ 0);
+
+ char *this_exe = realpath("/proc/self/exe", NULL);
+ REQUIRE(string(this_exe) == sym.module);
+ free(this_exe);
+
+ REQUIRE(string("_a_test_function") == sym.name);
+
+ REQUIRE(bcc_symcache_resolve(lazy_resolver, (uint64_t)&_a_test_function,
+ &lazy_sym) == 0);
+ REQUIRE(string(lazy_sym.name) == sym.name);
+ REQUIRE(string(lazy_sym.module) == sym.module);
+ }
+}
+
 #define STACK_SIZE (1024 * 1024)
 static char child_stack[STACK_SIZE];