Show the stacktrace of the userspace in opensnoop.

With new option '-c', opensnoop will show the names and line numbers of the two most inner callers in the userspace if there are. It uses BlazeSym (x86-64 only) to do symbolization. Signed-off-by: Kui-Feng Lee <[email protected]>
iovisor · Sep 1, 2022 · d060934 · d060934
1 parent 187baf1
commit d060934
Show file tree

Hide file tree

Showing 9 changed files with 164 additions and 15 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "libbpf-tools/bpftool"]
  path = libbpf-tools/bpftool
  url = https://github.com/libbpf/bpftool
+[submodule "libbpf-tools/blazesym"]
+ path = libbpf-tools/blazesym
+ url = https://github.com/libbpf/blazesym
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -23,14 +23,32 @@ endif()
 
 enable_testing()
 
+execute_process(COMMAND git config --global --add safe.directory ${CMAKE_CURRENT_SOURCE_DIR}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ RESULT_VARIABLE CONFIG_RESULT)
+if(CONFIG_RESULT AND NOT CONFIG_RESULT EQUAL 0)
+ message(WARNING "Failed to add root source directory to safe.directory")
+endif()
+
+# populate submodule blazesym
+if(NOT NO_BLAZESYM)
+ execute_process(COMMAND git config --global --add safe.directory ${CMAKE_CURRENT_SOURCE_DIR}/libbpf-tools/blazesym
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ RESULT_VARIABLE CONFIG_RESULT)
+ if(CONFIG_RESULT AND NOT CONFIG_RESULT EQUAL 0)
+ message(WARNING "Failed to add blazesym source directory to safe.directory")
+ endif()
+
+ execute_process(COMMAND git submodule update --init --recursive -- libbpf-tools/blazesym
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+ RESULT_VARIABLE UPDATE_RESULT)
+ if(UPDATE_RESULT AND NOT UPDATE_RESULT EQUAL 0)
+ message(WARNING "Failed to update submodule blazesym")
+ endif()
+endif()
+
 # populate submodules (libbpf)
 if(NOT CMAKE_USE_LIBBPF_PACKAGE)
- execute_process(COMMAND git config --global --add safe.directory ${CMAKE_CURRENT_SOURCE_DIR}
- WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
- RESULT_VARIABLE CONFIG_RESULT)
- if(CONFIG_RESULT AND NOT CONFIG_RESULT EQUAL 0)
- message(WARNING "Failed to add root source directory to safe.directory")
- endif()
  execute_process(COMMAND git config --global --add safe.directory ${CMAKE_CURRENT_SOURCE_DIR}/src/cc/libbpf
  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
  RESULT_VARIABLE CONFIG_RESULT)

diff --git a/docker/build/Dockerfile.fedora b/docker/build/Dockerfile.fedora
@@ -41,6 +41,10 @@ RUN dnf -y install \
  python3 \
  python3-pip
 
+RUN dnf -y install \
+ rust \
+ cargo
+
 RUN if [[ ! -e /usr/bin/python && -e /usr/bin/python3 ]]; then \
  ln -s $(readlink /usr/bin/python3) /usr/bin/python; \
  fi

diff --git a/libbpf-tools/Makefile b/libbpf-tools/Makefile
@@ -7,6 +7,7 @@ BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
 BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
 LIBBPF_SRC := $(abspath ../src/cc/libbpf/src)
 LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+LIBBLAZESYM_SRC := $(abspath blazesym/target/release/libblazesym.a)
 INCLUDES := -I$(OUTPUT) -I../src/cc/libbpf/include/uapi
 CFLAGS := -g -O2 -Wall
 BPFCFLAGS := -g -O2 -Wall
@@ -16,11 +17,18 @@ ARCH := $(shell uname -m | sed 's/x86_64/x86/' | sed 's/aarch64/arm64/' \
  | sed 's/ppc64le/powerpc/' | sed 's/mips.*/mips/' \
  | sed 's/riscv64/riscv/' | sed 's/loongarch.*/loongarch/')
 BTFHUB_ARCHIVE ?= $(abspath btfhub-archive)
+ifeq ($(ARCH),x86)
+USE_BLAZESYM ?= 1
+endif
 
 ifeq ($(wildcard $(ARCH)/),)
 $(error Architecture $(ARCH) is not supported yet. Please open an issue)
 endif
 
+BZ_APPS = \
+ opensnoop \
+ #
+
 APPS = \
  bashreadline \
  bindsnoop \
@@ -51,7 +59,6 @@ APPS = \
  numamove \
  offcputime \
  oomkill \
- opensnoop \
  readahead \
  runqlat \
  runqlen \
@@ -69,6 +76,7 @@ APPS = \
  tcprtt \
  tcpsynbl \
  vfsstat \
+ $(BZ_APPS) \
  #
 
 # export variables that are used in Makefile.btfgen as well.
@@ -89,6 +97,13 @@ COMMON_OBJ = \
  $(if $(ENABLE_MIN_CORE_BTFS),$(OUTPUT)/min_core_btf_tar.o) \
  #
 
+ifeq ($(USE_BLAZESYM),1)
+COMMON_OBJ += \
+ $(OUTPUT)/libblazesym.a \
+ $(OUTPUT)/blazesym.h \
+ #
+endif
+
 define allow-override
  $(if $(or $(findstring environment,$(origin $(1))),\
  $(findstring command line,$(origin $(1)))),,\
@@ -116,12 +131,30 @@ endif
 ifneq ($(EXTRA_LDFLAGS),)
 LDFLAGS += $(EXTRA_LDFLAGS)
 endif
+ifeq ($(USE_BLAZESYM),1)
+CFLAGS += -DUSE_BLAZESYM=1
+endif
+
+ifeq ($(USE_BLAZESYM),1)
+LDFLAGS += $(OUTPUT)/libblazesym.a -lrt -lpthread -ldl
+endif
 
 .PHONY: clean
 clean:
  $(call msg,CLEAN)
  $(Q)rm -rf $(OUTPUT) $(APPS) $(APP_ALIASES)
 
+$(LIBBLAZESYM_SRC)::
+ $(Q)cd blazesym && cargo build --release --features=cheader
+
+$(OUTPUT)/libblazesym.a: $(LIBBLAZESYM_SRC) | $(OUTPUT)
+ $(call msg,LIB,$@)
+ $(Q)cp $(LIBBLAZESYM_SRC) $@
+
+$(OUTPUT)/blazesym.h: $(LIBBLAZESYM_SRC) | $(OUTPUT)
+ $(call msg,INC,$@)
+ $(Q)cp blazesym/target/release/blazesym.h $@
+
 $(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
  $(call msg,MKDIR,$@)
  $(Q)mkdir -p $@
@@ -134,6 +167,10 @@ $(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT)
  $(call msg,BINARY,$@)
  $(Q)$(CC) $(CFLAGS) $^ $(LDFLAGS) -lelf -lz -o $@
 
+ifeq ($(USE_BLAZESYM),1)
+$(patsubst %,$(OUTPUT)/%.o,$(BZ_APPS)): $(OUTPUT)/blazesym.h
+endif
+
 $(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
 
 $(OUTPUT)/%.o: %.c $(wildcard %.h) $(LIBBPF_OBJ) | $(OUTPUT)

diff --git a/libbpf-tools/blazesym b/libbpf-tools/blazesym
diff --git a/libbpf-tools/opensnoop.bpf.c b/libbpf-tools/opensnoop.bpf.c
@@ -87,6 +87,7 @@ int trace_exit(struct trace_event_raw_sys_exit* ctx)
 {
  struct event event = {};
  struct args_t *ap;
+ uintptr_t stack[3];
  int ret;
  u32 pid = bpf_get_current_pid_tgid();
 
@@ -105,6 +106,12 @@ int trace_exit(struct trace_event_raw_sys_exit* ctx)
  event.flags = ap->flags;
  event.ret = ret;
 
+ bpf_get_stack(ctx, &stack, sizeof(stack),
+ BPF_F_USER_STACK);
+ /* Skip the first address that is usually the syscall it-self */
+ event.callers[0] = stack[1];
+ event.callers[1] = stack[2];
+
  /* emit event */
  bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
  &event, sizeof(event));

diff --git a/libbpf-tools/opensnoop.c b/libbpf-tools/opensnoop.c
@@ -18,6 +18,9 @@
 #include "opensnoop.skel.h"
 #include "btf_helpers.h"
 #include "trace_helpers.h"
+#ifdef USE_BLAZESYM
+#include "blazesym.h"
+#endif
 
 /* Tune the buffer size and wakeup rate. These settings cope with roughly
  * 50k opens/sec.
@@ -32,6 +35,10 @@
 
 static volatile sig_atomic_t exiting = 0;
 
+#ifdef USE_BLAZESYM
+static blazesym *symbolizer;
+#endif
+
 static struct env {
  pid_t pid;
  pid_t tid;
@@ -43,6 +50,9 @@ static struct env {
  bool extended;
  bool failed;
  char *name;
+#ifdef USE_BLAZESYM
+ bool callers;
+#endif
 } env = {
  .uid = INVALID_UID
 };
@@ -54,7 +64,11 @@ const char argp_program_doc[] =
 "Trace open family syscalls\n"
 "\n"
 "USAGE: opensnoop [-h] [-T] [-U] [-x] [-p PID] [-t TID] [-u UID] [-d DURATION]\n"
+#ifdef USE_BLAZESYM
+" [-n NAME] [-e] [-c]\n"
+#else
 " [-n NAME] [-e]\n"
+#endif
 "\n"
 "EXAMPLES:\n"
 " ./opensnoop # trace all open() syscalls\n"
@@ -66,7 +80,11 @@ const char argp_program_doc[] =
 " ./opensnoop -u 1000 # only trace UID 1000\n"
 " ./opensnoop -d 10 # trace for 10 seconds only\n"
 " ./opensnoop -n main # only print process names containing \"main\"\n"
-" ./opensnoop -e # show extended fields\n";
+" ./opensnoop -e # show extended fields\n"
+#ifdef USE_BLAZESYM
+" ./opensnoop -c # show calling functions\n"
+#endif
+"";
 
 static const struct argp_option opts[] = {
  { "duration", 'd', "DURATION", 0, "Duration to trace"},
@@ -80,6 +98,9 @@ static const struct argp_option opts[] = {
  { "print-uid", 'U', NULL, 0, "Print UID"},
  { "verbose", 'v', NULL, 0, "Verbose debug output" },
  { "failed", 'x', NULL, 0, "Failed opens only"},
+#ifdef USE_BLAZESYM
+ { "callers", 'c', NULL, 0, "Show calling functions"},
+#endif
  {},
 };
 
@@ -147,6 +168,11 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
  }
  env.uid = uid;
  break;
+#ifdef USE_BLAZESYM
+ case 'c':
+ env.callers = true;
+ break;
+#endif
  case ARGP_KEY_ARG:
  if (pos_args++) {
  fprintf(stderr,
@@ -177,6 +203,15 @@ void handle_event(void *ctx, int cpu, void *data, __u32 data_sz)
 {
  const struct event *e = data;
  struct tm *tm;
+#ifdef USE_BLAZESYM
+ sym_src_cfg cfgs[] = {
+ { .src_type = SRC_T_PROCESS, .params = { .process = { .pid = e->pid }}},
+ };
+ const blazesym_result *result = NULL;
+ const blazesym_csym *sym;
+ int i, j;
+#endif
+ int sps_cnt;
  char ts[32];
  time_t t;
  int fd, err;
@@ -197,15 +232,45 @@ void handle_event(void *ctx, int cpu, void *data, __u32 data_sz)
  err = - e->ret;
  }
 
+#ifdef USE_BLAZESYM
+ if (env.callers)
+ result = blazesym_symbolize(symbolizer, cfgs, 1, (const uint64_t *)&e->callers, 2);
+#endif
+
  /* print output */
- if (env.timestamp)
+ sps_cnt = 0;
+ if (env.timestamp) {
  printf("%-8s ", ts);
- if (env.print_uid)
- printf("%-6d ", e->uid);
+ sps_cnt += 9;
+ }
+ if (env.print_uid) {
+ printf("%-7d ", e->uid);
+ sps_cnt += 8;
+ }
  printf("%-6d %-16s %3d %3d ", e->pid, e->comm, fd, err);
- if (env.extended)
+ sps_cnt += 7 + 17 + 4 + 4;
+ if (env.extended) {
  printf("%08o ", e->flags);
+ sps_cnt += 9;
+ }
  printf("%s\n", e->fname);
+
+#ifdef USE_BLAZESYM
+ for (i = 0; result && i < result->size; i++) {
+ if (result->entries[i].size == 0)
+ continue;
+ sym = &result->entries[i].syms[0];
+
+ for (j = 0; j < sps_cnt; j++)
+ printf(" ");
+ if (sym->line_no)
+ printf("%s:%ld\n", sym->symbol, sym->line_no);
+ else
+ printf("%s\n", sym->symbol);
+ }
+
+ blazesym_result_free(result);
+#endif
 }
 
 void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt)
@@ -273,15 +338,25 @@ int main(int argc, char **argv)
  goto cleanup;
  }
 
+#ifdef USE_BLAZESYM
+ if (env.callers)
+ symbolizer = blazesym_new();
+#endif
+
  /* print headers */
  if (env.timestamp)
  printf("%-8s ", "TIME");
  if (env.print_uid)
- printf("%-6s ", "UID");
+ printf("%-7s ", "UID");
  printf("%-6s %-16s %3s %3s ", "PID", "COMM", "FD", "ERR");
  if (env.extended)
  printf("%-8s ", "FLAGS");
- printf("%s\n", "PATH");
+ printf("%s", "PATH");
+#ifdef USE_BLAZESYM
+ if (env.callers)
+ printf("/CALLER");
+#endif
+ printf("\n");
 
  /* setup event callbacks */
  pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES,
@@ -319,6 +394,9 @@ int main(int argc, char **argv)
  perf_buffer__free(pb);
  opensnoop_bpf__destroy(obj);
  cleanup_core_btf(&open_opts);
+#ifdef USE_BLAZESYM
+ blazesym_free(symbolizer);
+#endif
 
  return err != 0;
 }
diff --git a/libbpf-tools/opensnoop.h b/libbpf-tools/opensnoop.h
@@ -18,6 +18,7 @@ struct event {
  uid_t uid;
  int ret;
  int flags;
+ __u64 callers[2];
  char comm[TASK_COMM_LEN];
  char fname[NAME_MAX];
 };

diff --git a/src/cc/libbpf b/src/cc/libbpf