tools/cpudist: Add extension summary support (iovisor#4131)

Sometimes, I want to known total on-CPU or off-CPU time and count (same as context switch times) at a fixed interval (for example: 1s).

Like iovisor#3384, This patch try to add an option -e to show extension summary (average/total/count).

$ ./cpudist.py -p $(pgrep -nx mysqld) -e 1

     usecs               : count     distribution
         0 -> 1          : 4123     |**************                          |
         2 -> 3          : 11690    |****************************************|
         4 -> 7          : 1668     |*****                                   |
         8 -> 15         : 859      |**                                      |
        16 -> 31         : 618      |**                                      |
        32 -> 63         : 290      |                                        |
        64 -> 127        : 247      |                                        |
       128 -> 255        : 198      |                                        |
       256 -> 511        : 161      |                                        |
       512 -> 1023       : 370      |*                                       |
      1024 -> 2047       : 98       |                                        |
      2048 -> 4095       : 6        |                                        |
      4096 -> 8191       : 16       |                                        |

avg = 33 usecs, total: 682091 usecs, count: 20383

man/man8/cpudist.8

@@ -2,7 +2,7 @@
 .SH NAME
 cpudist \- On- and off-CPU task time as a histogram.
 .SH SYNOPSIS
-.B cpudist [\-h] [-O] [\-T] [\-m] [\-P] [\-L] [\-p PID] [\-I] [interval] [count]
+.B cpudist [\-h] [-O] [\-T] [\-m] [\-P] [\-L] [\-p PID] [\-I] [\-e] [interval] [count]
 .SH DESCRIPTION
 This measures the time a task spends on the CPU before being descheduled, and
 shows the times as a histogram. Tasks that spend a very short time on the CPU
@@ -50,6 +50,9 @@ Only show this PID (filtered in kernel for efficiency).
 \-I
 Include CPU idle time (by default these are excluded).
 .TP
+\-e
+Show extension summary (average/total/count).
+.TP
 interval
 Output interval, in seconds.
 .TP
@@ -63,7 +66,7 @@ Summarize task on-CPU time as a histogram:
 .TP
 Summarize task off-CPU time as a histogram:
 #
-.B cpudist -O
+.B cpudist \-O
 .TP
 Print 1 second summaries, 10 times:
 #
@@ -75,11 +78,15 @@ Print 1 second summaries, using milliseconds as units for the histogram, and inc
 .TP
 Trace PID 185 only, 1 second summaries:
 #
-.B cpudist -p 185 1
+.B cpudist \-p 185 1
 .TP
 Include CPU idle time:
 #
-.B cpudist -I
+.B cpudist \-I
+.TP
+Also show extension summary:
+#
+.B cpudist \-e
 .SH FIELDS
 .TP
 usecs
@@ -111,6 +118,6 @@ Linux
 .SH STABILITY
 Unstable - in development.
 .SH AUTHOR
-Sasha Goldshtein
+Sasha Goldshtein, Rocky Xing
 .SH SEE ALSO
 pidstat(1), runqlat(8)

tools/cpudist.py

@@ -3,7 +3,7 @@
 #
 # cpudist Summarize on- and off-CPU time per task as a histogram.
 #
-# USAGE: cpudist [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [interval] [count]
+# USAGE: cpudist [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [-e] [interval] [count]
 #
 # This measures the time a task spends on or off the CPU, and shows this time
 # as a histogram, optionally per-process.
@@ -14,6 +14,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 #
 # 27-Mar-2022 Rocky Xing Changed to exclude CPU idle time by default.
+# 25-Jul-2022 Rocky Xing Added extension summary support.
 
 from __future__ import print_function
 from bcc import BPF
@@ -28,9 +29,10 @@
  cpudist -P # show each PID separately
  cpudist -p 185 # trace PID 185 only
  cpudist -I # include CPU idle time
+ cpudist -e # show extension summary (average/total/count)
 """
 parser = argparse.ArgumentParser(
- description="Summarize on-CPU time per task as a histogram.",
+ description="Summarize on- and off-CPU time per task as a histogram.",
  formatter_class=argparse.RawDescriptionHelpFormatter,
  epilog=examples)
 parser.add_argument("-O", "--offcpu", action="store_true",
@@ -47,6 +49,8 @@
  help="trace this PID only")
 parser.add_argument("-I", "--include-idle", action="store_true",
  help="include CPU idle time")
+parser.add_argument("-e", "--extension", action="store_true",
+ help="show extension summary (average/total/count)")
 parser.add_argument("interval", nargs="?", default=99999999,
  help="output interval, in seconds")
 parser.add_argument("count", nargs="?", default=99999999,
@@ -57,7 +61,8 @@
 countdown = int(args.count)
 debug = 0
 
-bpf_text = """#include <uapi/linux/ptrace.h>
+bpf_text = """
+#include <uapi/linux/ptrace.h>
 #include <linux/sched.h>
 """
 
@@ -75,6 +80,10 @@
  u64 slot;
 } pid_key_t;
 
+typedef struct ext_val {
+ u64 total;
+ u64 count;
+} ext_val_t;
 
 BPF_HASH(start, entry_key_t, u64, MAX_PID);
 STORAGE
@@ -157,22 +166,40 @@
 else:
  bpf_text = bpf_text.replace('FACTOR', 'delta /= 1000;')
  label = "usecs"
+
+storage_str = ""
+store_str = ""
+
 if args.pids or args.tids:
  section = "pid"
  pid = "tgid"
  if args.tids:
  pid = "pid"
  section = "tid"
- bpf_text = bpf_text.replace('STORAGE',
-  'BPF_HISTOGRAM(dist, pid_key_t, MAX_PID);')
- bpf_text = bpf_text.replace('STORE',
-  'pid_key_t key = {.id = ' + pid + ', .slot = bpf_log2l(delta)}; ' +
-  'dist.increment(key);')
+ storage_str += "BPF_HISTOGRAM(dist, pid_key_t, MAX_PID);"
+ store_str += """
+ pid_key_t key = {.id = ' + pid + ', .slot = bpf_log2l(delta)};
+ dist.increment(key);
+ """
 else:
  section = ""
- bpf_text = bpf_text.replace('STORAGE', 'BPF_HISTOGRAM(dist);')
- bpf_text = bpf_text.replace('STORE',
- 'dist.atomic_increment(bpf_log2l(delta));')
+ storage_str += "BPF_HISTOGRAM(dist);"
+ store_str += "dist.atomic_increment(bpf_log2l(delta));"
+
+if args.extension:
+ storage_str += "BPF_ARRAY(extension, ext_val_t, 1);"
+ store_str += """
+ u32 index = 0;
+ ext_val_t *ext_val = extension.lookup(&index);
+ if (ext_val) {
+ lock_xadd(&ext_val->total, delta);
+ lock_xadd(&ext_val->count, 1);
+ }
+ """
+
+bpf_text = bpf_text.replace("STORAGE", storage_str)
+bpf_text = bpf_text.replace("STORE", store_str)
+
 if debug or args.ebpf:
  print(bpf_text)
  if args.ebpf:
@@ -189,6 +216,8 @@
 
 exiting = 0 if args.interval else 1
 dist = b.get_table("dist")
+if args.extension:
+ extension = b.get_table("extension")
 while (1):
  try:
  sleep(int(args.interval))
@@ -207,6 +236,15 @@ def pid_to_comm(pid):
  return str(pid)
 
  dist.print_log2_hist(label, section, section_print_fn=pid_to_comm)
+
+ if args.extension:
+ total = extension[0].total
+ count = extension[0].count
+ if count > 0:
+ print("\navg = %ld %s, total: %ld %s, count: %ld\n" %
+ (total / count, label, total, label, count))
+ extension.clear()
+
  dist.clear()
 
  countdown -= 1

tools/cpudist_example.txt

@@ -282,9 +282,10 @@ USAGE message:
 
 # ./cpudist.py -h
 
-usage: cpudist.py [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [interval] [count]
+usage: cpudist.py [-h] [-O] [-T] [-m] [-P] [-L] [-p PID] [-I] [-e]
+ [interval] [count]
 
-Summarize on-CPU time per task as a histogram.
+Summarize on- and off-CPU time per task as a histogram.
 
 positional arguments:
  interval output interval, in seconds
@@ -299,6 +300,7 @@ optional arguments:
  -L, --tids print a histogram per thread ID
  -p PID, --pid PID trace this PID only
  -I, --include-idle include CPU idle time
+ -e, --extension show extension summary (average/total/count)
 
 examples:
  cpudist # summarize on-CPU time as a histogram
@@ -308,4 +310,5 @@ examples:
  cpudist -P # show each PID separately
  cpudist -p 185 # trace PID 185 only
  cpudist -I # include CPU idle time
+ cpudist -e # show extension summary (average/total/count)