bcc/syms: Fix shared lib module offset <-> global addr conversion

`bcc` does various conversions of "global address" to "module offset" and
vice versa. Previous work (#1670) modified the "global address" ->
"module offset" calculation in `ProcSyms::Module::contains` to account
for differences between the file offset a section is loading bytes from
and the requested start address (relative to the base address of the
`.so`). Unfortunately that change didn't also modify "module offset" ->
"global address" calculations, such as the one in
bcc_resolve_global_addr. Update that calculation to account for the
same.

This calculation discrepancy was most apparent for us in production when
trying to attach USDTs to a shared lib with differing requested start
address and file offset. This patch also adds a test w/ comments
describing our specific situation and demonstrating how the patch fixes
the issue.

Signed-off-by: Dave Marchevsky <[email protected]>

src/cc/bcc_syms.cc

@@ -289,11 +289,8 @@ bool ProcSyms::Module::contains(uint64_t addr, uint64_t &offset) const {
  for (const auto &range : ranges_) {
  if (addr >= range.start && addr < range.end) {
  if (type_ == ModuleType::SO || type_ == ModuleType::VDSO) {
- // Offset within the mmap
- offset = addr - range.start + range.file_offset;
-
- // Offset within the ELF for SO symbol lookup
- offset += (elf_so_addr_ - elf_so_offset_);
+ offset = __so_calc_mod_offset(range.start, range.file_offset,
+ elf_so_addr_, elf_so_offset_, addr);
  } else {
  offset = addr;
  }
@@ -619,9 +616,26 @@ int _bcc_syms_find_module(mod_info *info, int enter_ns, void *p) {
  return -1;
 }
 
+uint64_t __so_calc_global_addr(uint64_t mod_start_addr,
+ uint64_t mod_file_offset,
+ uint64_t elf_sec_start_addr,
+ uint64_t elf_sec_file_offset, uint64_t offset) {
+ return offset + (mod_start_addr - mod_file_offset) -
+ (elf_sec_start_addr - elf_sec_file_offset);
+}
+
+uint64_t __so_calc_mod_offset(uint64_t mod_start_addr, uint64_t mod_file_offset,
+ uint64_t elf_sec_start_addr,
+ uint64_t elf_sec_file_offset,
+ uint64_t global_addr) {
+ return global_addr - (mod_start_addr - mod_file_offset) +
+ (elf_sec_start_addr - elf_sec_file_offset);
+}
+
 int bcc_resolve_global_addr(int pid, const char *module, const uint64_t address,
  uint8_t inode_match_only, uint64_t *global) {
  struct stat s;
+ uint64_t elf_so_addr, elf_so_offset;
  if (stat(module, &s))
  return -1;
 
@@ -632,7 +646,11 @@ int bcc_resolve_global_addr(int pid, const char *module, const uint64_t address,
  mod.start == 0x0)
  return -1;
 
- *global = mod.start - mod.file_offset + address;
+ if (bcc_elf_get_text_scn_info(module, &elf_so_addr, &elf_so_offset) < 0)
+ return -1;
+
+ *global = __so_calc_global_addr(mod.start, mod.file_offset, elf_so_addr,
+ elf_so_offset, address);
  return 0;
 }
 

src/cc/bcc_syms.h

@@ -102,6 +102,30 @@ int bcc_resolve_symname(const char *module, const char *symname,
  struct bcc_symbol_option* option,
  struct bcc_symbol *sym);
 
+/* Calculate the global address for 'offset' in a shared object loaded into
+ * a process
+ *
+ * Need to know (start_addr, file_offset) pairs for the /proc/PID/maps module
+ * entry containing the offset and the elf section containing the module's
+ * .text
+ */
+uint64_t __so_calc_global_addr(uint64_t mod_start_addr,
+ uint64_t mod_file_offset,
+ uint64_t elf_sec_start_addr,
+ uint64_t elf_sec_file_offset, uint64_t offset);
+
+/* Given a global address which falls within a shared object's mapping in a
+ * process, calculate the corresponding 'offset' in the .so
+ *
+ * Need to know (start_addr, file_offset) pairs for the /proc/PID/maps module
+ * entry containing the offset and the elf section containing the module's
+ * .text
+ */
+uint64_t __so_calc_mod_offset(uint64_t mod_start_addr, uint64_t mod_file_offset,
+ uint64_t elf_sec_start_addr,
+ uint64_t elf_sec_file_offset,
+ uint64_t global_addr);
+
 #ifdef __cplusplus
 }
 #endif

tests/cc/test_c_api.cc

@@ -600,6 +600,59 @@ TEST_CASE("resolve global addr in libc in this process", "[c_api][!mayfail]") {
  REQUIRE(global_addr == (search.start + local_addr - search.file_offset));
 }
 
+/* Consider the following scenario: we have some process that maps in a shared library [1] with a
+ * USDT probe [2]. The shared library's .text section doesn't have matching address and file off
+ * [3]. Since the location address in [2] is an offset relative to the base address of whatever.so
+ * in whatever process is mapping it, we need to convert the location address 0x77b8c to a global
+ * address in the process' address space in order to attach to the USDT.
+ *
+ * The formula for this (__so_calc_global_addr) is
+ * global_addr = offset + (mod_start_addr - mod_file_offset)
+ * - (elf_sec_start_addr - elf_sec_file_offset)
+ *
+ * Which for our concrete example is
+ * global_addr = 0x77b8c + (0x7f6cda31e000 - 0x72000) - (0x73c90 - 0x72c90)
+ * global_addr = 0x7f6cda322b8c
+ *
+ * [1 - output from `cat /proc/PID/maps`]
+ * 7f6cda2ab000-7f6cda31e000 r--p 00000000 00:2d 5370022276 /whatever.so
+ * 7f6cda31e000-7f6cda434000 r-xp 00072000 00:2d 5370022276 /whatever.so
+ * 7f6cda434000-7f6cda43d000 r--p 00187000 00:2d 5370022276 /whatever.so
+ * 7f6cda43d000-7f6cda43f000 rw-p 0018f000 00:2d 5370022276 /whatever.so
+ *
+ * [2 - output from `readelf -n /whatever.so`]
+ * stapsdt 0x00000038 NT_STAPSDT (SystemTap probe descriptors)
+ * Provider: test
+ * Name: test_probe
+ * Location: 0x0000000000077b8c, Base: 0x0000000000000000, Semaphore: 0x0000000000000000
+ * Arguments: -8@$5
+ *
+ * [3 - output from `readelf -W --sections /whatever.so`]
+ * [Nr] Name Type Address Off Size ES Flg Lk Inf Al
+ * [16] .text PROGBITS 0000000000073c90 072c90 1132dc 00 AX 0 0 16
+ */
+TEST_CASE("conversion of module offset to/from global_addr", "[c_api]") {
+ uint64_t global_addr, offset, calc_offset, mod_start_addr, mod_file_offset;
+ uint64_t elf_sec_start_addr, elf_sec_file_offset;
+
+ /* Initialize per example in comment above */
+ offset = 0x77b8c;
+ mod_start_addr = 0x7f6cda31e000;
+ mod_file_offset = 0x00072000;
+ elf_sec_start_addr = 0x73c90;
+ elf_sec_file_offset = 0x72c90;
+ global_addr = __so_calc_global_addr(mod_start_addr, mod_file_offset,
+ elf_sec_start_addr, elf_sec_file_offset,
+ offset);
+ REQUIRE(global_addr == 0x7f6cda322b8c);
+
+ /* Reverse operation (global_addr -> offset) should yield original offset */
+ calc_offset = __so_calc_mod_offset(mod_start_addr, mod_file_offset,
+ elf_sec_start_addr, elf_sec_file_offset,
+ global_addr);
+ REQUIRE(calc_offset == offset);
+}
+
 TEST_CASE("get online CPUs", "[c_api]") {
  std::vector<int> cpus = ebpf::get_online_cpus();
  int num_cpus = sysconf(_SC_NPROCESSORS_ONLN);