add --kernel-threads-only to tools/offcputime

Summary:
Adds `--kernel-threads-only` arg
The kernel-threads-only arg is exclusive with pid/user-threads-only via `parser.add_mutually_exclusive_group`.
The output message now indicates what we are tracing (pid/user threads/kernel threads/all threads).
Removed the --verbose arg (unused).

Test Plan:
Run with combinations of the args; validate output looks sane:

// test mutually exclusive group
```
devbig680[bcc](abirchall_dev): ~/bcc_run_tool.sh offcputime -k -u 1
[Running] /data/users/abirchall/bcc/tools/offcputime.py -k -u 1
usage: offcputime.py [-h] [-p PID | -k | -u] [-v] [-f] [duration]
offcputime.py: error: argument -u/--user-threads-only: not allowed with argument -k/--kernel-threads-only
```

// kernel threads only
```
devbig680[bcc](abirchall_dev): ~/bcc_run_tool.sh offcputime -f -k 1
[Running] /data/users/abirchall/bcc/tools/offcputime.py -f -k 1
swapper/21;start_secondary;cpu_startup_entry;schedule_preempt_disabled;schedule 11
swapper/16;start_secondary;cpu_startup_entry;schedule_preempt_disabled;schedule 19
swapper/22;start_secondary;cpu_startup_entry;schedule_preempt_disabled;schedule 20
swapper/31;start_secondary;cpu_startup_entry;schedule_preempt_disabled;schedule 20
swapper/23;start_secondary;cpu_startup_entry;schedule_preempt_disabled;schedule 67
swapper/25;start_secondary;cpu_startup_entry;schedule_preempt_disabled;schedule 74
...
```
`~/bcc_run_tool.sh offcputime -f --kernel-threads-only 1`

// user threads only
`~/bcc_run_tool.sh offcputime -f --user-threads-only 1`
`~/bcc_run_tool.sh offcputime -f -u 1`

// specific pid
`~/bcc_run_tool.sh offcputime -f -p $(pidof hphpi) 1`
```
devbig680[bcc](abirchall_dev): ~/bcc_run_tool.sh offcputime --pid $(pidof mcrouter) 10 | head
[Running] /data/users/abirchall/bcc/tools/offcputime.py --pid 95929 10
Tracing off-CPU time (us) of PID 95929 by kernel stack for 10 secs.
```

Note that this last case (specific PID) doesn't appear to be working; I can debug that in a follow up commit.

tools/offcputime.py

@@ -3,11 +3,7 @@
 # offcputime Summarize off-CPU time by kernel stack trace
 # For Linux, uses BCC, eBPF.
 #
-# USAGE: offcputime [-h] [-u] [-p PID] [-v] [-f] [duration]
-#
-# The current implementation uses an unrolled loop for x86_64, and was written
-# as a proof of concept. This implementation should be replaced in the future
-# with an appropriate bpf_ call, when available.
+# USAGE: offcputime [-h] [-p PID | -u | -k] [-f] [duration]
 #
 # Copyright 2016 Netflix, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License")
@@ -43,20 +39,21 @@ def positive_nonzero_int(val):
  ./offcputime # trace off-CPU stack time until Ctrl-C
  ./offcputime 5 # trace for 5 seconds only
  ./offcputime -f 5 # 5 seconds, and output in folded format
- ./offcputime -u # don't include kernel threads (user only)
- ./offcputime -p 185 # trace for PID 185 only
+ ./offcputime -p 185 # only trace threads for PID 185
+ ./offcputime -u # only trace user threads (no kernel)
+ ./offcputime -k # only trace kernel threads (no user)
 """
 parser = argparse.ArgumentParser(
  description="Summarize off-CPU time by kernel stack trace",
  formatter_class=argparse.RawDescriptionHelpFormatter,
  epilog=examples)
 thread_group = parser.add_mutually_exclusive_group()
-thread_group.add_argument("-u", "--useronly", action="store_true",
- help="user threads only (no kernel threads)")
 thread_group.add_argument("-p", "--pid", type=positive_int,
  help="trace this PID only")
-parser.add_argument("-v", "--verbose", action="store_true",
- help="show raw addresses")
+thread_group.add_argument("-k", "--kernel-threads-only", action="store_true",
+ help="kernel threads only (no user threads)")
+thread_group.add_argument("-u", "--user-threads-only", action="store_true",
+ help="user threads only (no kernel threads)")
 parser.add_argument("-f", "--folded", action="store_true",
  help="output folded format")
 parser.add_argument("--stack-storage-size", default=1024,
@@ -94,7 +91,7 @@ def signal_ignore(signal, frame):
  u64 ts, *tsp;
 
  // record previous thread sleep time
- if (FILTER) {
+ if (THREAD_FILTER) {
  pid = prev->pid;
  ts = bpf_ktime_get_ns();
  start.update(&pid, &ts);
@@ -125,13 +122,20 @@ def signal_ignore(signal, frame):
 """
 
 # set thread filter
+thread_context = ""
 if args.pid is not None:
- filter = 'pid == %s' % args.pid
-elif args.useronly:
- filter = '!(prev->flags & PF_KTHREAD)'
+ thread_context = "PID %s" % args.pid
+ thread_filter = 'pid == %s' % args.pid
+elif args.user_threads_only:
+ thread_context = "user threads"
+ thread_filter = '!(prev->flags & PF_KTHREAD)'
+elif args.kernel_threads_only:
+ thread_context = "kernel threads"
+ thread_filter = 'prev->flags & PF_KTHREAD'
 else:
- filter = '1'
-bpf_text = bpf_text.replace('FILTER', filter)
+ thread_context = "all threads"
+ thread_filter = '1'
+bpf_text = bpf_text.replace('THREAD_FILTER', thread_filter)
 
 # set stack storage size
 bpf_text = bpf_text.replace('STACK_STORAGE_SIZE', str(args.stack_storage_size))
@@ -146,7 +150,8 @@ def signal_ignore(signal, frame):
 
 # header
 if not folded:
- print("Tracing off-CPU time (us) by kernel stack", end="")
+ print("Tracing off-CPU time (us) of %s by kernel stack" %
+ thread_context, end="")
  if duration < 99999999:
  print(" for %d secs." % duration)
  else:

tools/offcputime_example.txt

@@ -718,7 +718,7 @@ creating your "off-CPU time flame graphs".
 USAGE message:
 
 # ./offcputime -h
-usage: offcputime.py [-h] [-u | -p PID] [-v] [-f]
+usage: offcputime.py [-h] [-p PID | -k | -u] [-f]
  [--stack-storage-size STACK_STORAGE_SIZE]
  [duration]
 
@@ -729,9 +729,11 @@ positional arguments:
 
 optional arguments:
  -h, --help show this help message and exit
- -u, --useronly user threads only (no kernel threads)
  -p PID, --pid PID trace this PID only
- -v, --verbose show raw addresses
+ -k, --kernel-threads-only
+ kernel threads only (no user threads)
+ -u, --user-threads-only
+ user threads only (no kernel threads)
  -f, --folded output folded format
  --stack-storage-size STACK_STORAGE_SIZE
  the number of unique stack traces that can be stored
@@ -741,5 +743,6 @@ examples:
  ./offcputime # trace off-CPU stack time until Ctrl-C
  ./offcputime 5 # trace for 5 seconds only
  ./offcputime -f 5 # 5 seconds, and output in folded format
- ./offcputime -u # don't include kernel threads (user only)
- ./offcputime -p 185 # trace for PID 185 only
+ ./offcputime -p 185 # only trace threads for PID 185
+ ./offcputime -u # only trace user threads (no kernel)
+ ./offcputime -k # only trace kernel threads (no user)