Merge pull request iovisor#763 from goldshtn/enhanced-funccount

funccount: Generalized for uprobes, tracepoints, and USDT

man/man8/funccount.8

@@ -1,13 +1,13 @@
 .TH funccount 8 "2015-08-18" "USER COMMANDS"
 .SH NAME
-funccount \- Count kernel function calls matching a pattern. Uses Linux eBPF/bcc.
+funccount \- Count function, tracepoint, and USDT probe calls matching a pattern. Uses Linux eBPF/bcc.
 .SH SYNOPSIS
-.B funccount [\-h] [\-p PID] [\-i INTERVAL] [\-T] [\-r] pattern
+.B funccount [\-h] [\-p PID] [\-i INTERVAL] [\-T] [\-r] [\-d] pattern
 .SH DESCRIPTION
-This tool is a quick way to determine which kernel functions are being called,
+This tool is a quick way to determine which functions are being called,
 and at what rate. It uses in-kernel eBPF maps to count function calls.
 
-WARNING: This uses dynamic tracing of (what can be many) kernel functions, an
+WARNING: This uses dynamic tracing of (what can be many) functions, an
 activity that has had issues on some kernel versions (risk of panics or
 freezes). Test, and know what you are doing, before use.
 
@@ -32,6 +32,9 @@ Include timestamps on output.
 .TP
 \-r
 Use regular expressions for the search pattern.
+.TP
+\-d
+Print the BPF program before starting (for debugging purposes).
 .SH EXAMPLES
 .TP
 Count kernel functions beginning with "vfs_", until Ctrl-C is hit:
@@ -53,19 +56,28 @@ Match kernel functions beginning with "vfs_", using regular expressions:
 Count vfs calls for process ID 181 only:
 #
 .B funccount \-p 181 'vfs_*'
-.SH FIELDS
 .TP
-ADDR
-Address of the instruction pointer that was traced (only useful if the FUNC column is suspicious and you would like to double check the translation).
+Count calls to the sched_fork tracepoint, indicating a fork() performed:
+#
+.B funccount t:sched:sched_fork
+.TP
+Count all GC USDT probes in the Node process:
+#
+.B funccount -p 185 u:node:gc*
+.TP
+Count all malloc() calls in libc:
+#
+.B funccount c:malloc
+.SH FIELDS
 .TP
 FUNC
-Kernel function name
+Function name
 .TP
 COUNT
 Number of calls while tracing
 .SH OVERHEAD
-This traces kernel functions and maintains in-kernel counts, which
-are asynchronously copied to user-space. While the rate of kernel calls
+This traces functions and maintains in-kernel counts, which
+are asynchronously copied to user-space. While the rate of calls
 be very high (>1M/sec), this is a relatively efficient way to trace these
 events, and so the overhead is expected to be small for normal workloads.
 Measure in a test environment before use.
@@ -81,6 +93,8 @@ Linux
 .SH STABILITY
 Unstable - in development.
 .SH AUTHOR
-Brendan Gregg
+Brendan Gregg, Sasha Goldshtein
 .SH SEE ALSO
+stackcount(8)
+funclatency(8)
 vfscount(8)

src/python/bcc/__init__.py

@@ -376,7 +376,8 @@ def attach_raw_socket(fn, dev):
  % (dev, errstr))
  fn.sock = sock
 
- def _get_kprobe_functions(self, event_re):
+ @staticmethod
+ def get_kprobe_functions(event_re):
  with open("%s/../kprobes/blacklist" % TRACEFS) as blacklist_file:
  blacklist = set([line.rstrip().split()[1] for line in
  blacklist_file])
@@ -386,7 +387,6 @@ def _get_kprobe_functions(self, event_re):
  fn = line.rstrip().split()[0]
  if re.match(event_re, fn) and fn not in blacklist:
  fns.append(fn)
- self._check_probe_quota(len(fns))
  return fns
 
  def _check_probe_quota(self, num_new_probes):
@@ -409,7 +409,9 @@ def attach_kprobe(self, event="", fn_name="", event_re="",
 
  # allow the caller to glob multiple functions together
  if event_re:
- for line in self._get_kprobe_functions(event_re):
+ matches = BPF.get_kprobe_functions(event_re)
+ self._check_probe_quota(len(matches))
+ for line in matches:
  try:
  self.attach_kprobe(event=line, fn_name=fn_name, pid=pid,
  cpu=cpu, group_fd=group_fd)
@@ -448,7 +450,7 @@ def attach_kretprobe(self, event="", fn_name="", event_re="",
 
  # allow the caller to glob multiple functions together
  if event_re:
- for line in self._get_kprobe_functions(event_re):
+ for line in BPF.get_kprobe_functions(event_re):
  try:
  self.attach_kretprobe(event=line, fn_name=fn_name, pid=pid,
  cpu=cpu, group_fd=group_fd)
@@ -531,7 +533,8 @@ def find_library(libname):
  res = lib.bcc_procutils_which_so(libname.encode("ascii"))
  return res if res is None else res.decode()
 
- def _get_tracepoints(self, tp_re):
+ @staticmethod
+ def get_tracepoints(tp_re):
  results = []
  events_dir = os.path.join(TRACEFS, "events")
  for category in os.listdir(events_dir):
@@ -570,7 +573,7 @@ def attach_tracepoint(self, tp="", tp_re="", fn_name="", pid=-1,
  """
 
  if tp_re:
- for tp in self._get_tracepoints(tp_re):
+ for tp in BPF.get_tracepoints(tp_re):
  self.attach_tracepoint(tp=tp, fn_name=fn_name, pid=pid,
  cpu=cpu, group_fd=group_fd)
  return
@@ -615,7 +618,13 @@ def _del_uprobe(self, name):
  del self.open_uprobes[name]
  _num_open_probes -= 1
 
- def _get_user_functions(self, name, sym_re):
+ @staticmethod
+ def get_user_functions(name, sym_re):
+ return set([name for (name, _) in
+ BPF.get_user_functions_and_addresses(name, sym_re)])
+
+ @staticmethod
+ def get_user_addresses(name, sym_re):
  """
  We are returning addresses here instead of symbol names because it
  turns out that the same name may appear multiple times with different
@@ -624,10 +633,15 @@ def _get_user_functions(self, name, sym_re):
  it makes sense to return the unique set of addresses that are mapped to
  a symbol that matches the provided regular expression.
  """
+ return set([address for (_, address) in
+ BPF.get_user_functions_and_addresses(name, sym_re)])
+
+ @staticmethod
+ def get_user_functions_and_addresses(name, sym_re):
  addresses = []
  def sym_cb(sym_name, addr):
- if re.match(sym_re, sym_name) and addr not in addresses:
- addresses.append(addr)
+ if re.match(sym_re, sym_name):
+ addresses.append((sym_name, addr))
  return 0
 
  res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb))
@@ -660,7 +674,9 @@ def attach_uprobe(self, name="", sym="", sym_re="", addr=None,
  name = str(name)
 
  if sym_re:
- for sym_addr in self._get_user_functions(name, sym_re):
+ addresses = BPF.get_user_addresses(name, sym_re)
+ self._check_probe_quota(len(addresses))
+ for sym_addr in addresses:
  self.attach_uprobe(name=name, addr=sym_addr,
  fn_name=fn_name, pid=pid, cpu=cpu,
  group_fd=group_fd)
@@ -711,7 +727,7 @@ def attach_uretprobe(self, name="", sym="", sym_re="", addr=None,
  """
 
  if sym_re:
- for sym_addr in self._get_user_functions(name, sym_re):
+ for sym_addr in BPF.get_user_addresses(name, sym_re):
  self.attach_uretprobe(name=name, addr=sym_addr,
  fn_name=fn_name, pid=pid, cpu=cpu,
  group_fd=group_fd)

tests/python/test_probe_count.py

@@ -67,6 +67,10 @@ def test_probe_quota(self):
  with self.assertRaises(Exception):
  self.b.attach_kprobe(event_re=".*", fn_name="count")
 
+ def test_uprobe_quota(self):
+ with self.assertRaises(Exception):
+ self.b.attach_uprobe(name="c", sym_re=".*", fn_name="count")
+
  def tearDown(self):
  self.b.cleanup()