Python 3 compatibility fixes around string handling (iovisor#986)

This fixes the bcc module and all the affected tools for issues related to string handling in Python 3. Specifically, when passing Python strings to C libraries they are encoded as ASCII, and when constructing Python strings from C strings, they are decoded first.

src/python/bcc/__init__.py

@@ -759,11 +759,12 @@ def get_user_addresses(name, sym_re):
  def get_user_functions_and_addresses(name, sym_re):
  addresses = []
  def sym_cb(sym_name, addr):
- if re.match(sym_re, sym_name):
- addresses.append((sym_name, addr))
+ dname = sym_name.decode()
+ if re.match(sym_re, dname):
+ addresses.append((dname, addr))
  return 0
 
- res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb))
+ res = lib.bcc_foreach_symbol(name.encode('ascii'), _SYM_CB_TYPE(sym_cb))
  if res < 0:
  raise Exception("Error %d enumerating symbols in %s" % (res, name))
  return addresses

src/python/bcc/usdt.py

@@ -138,15 +138,17 @@ def __init__(self, pid=None, path=None):
  "either a pid or a binary path must be specified")
 
  def enable_probe(self, probe, fn_name):
- if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
+ if lib.bcc_usdt_enable_probe(self.context, probe.encode('ascii'),
+ fn_name.encode('ascii')) != 0:
  raise USDTException(
  ("failed to enable probe '%s'; a possible cause " +
  "can be that the probe requires a pid to enable") %
  probe
  )
 
  def enable_probe_or_bail(self, probe, fn_name):
- if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
+ if lib.bcc_usdt_enable_probe(self.context, probe.encode('ascii'),
+ fn_name.encode('ascii')) != 0:
  print(
 """Error attaching USDT probes: the specified pid might not contain the
 given language's runtime, or the runtime was not built with the required
@@ -175,7 +177,7 @@ def _add_probe(probe):
  def attach_uprobes(self, bpf):
  probes = self.enumerate_active_probes()
  for (binpath, fn_name, addr, pid) in probes:
- bpf.attach_uprobe(name=binpath, fn_name=fn_name,
+ bpf.attach_uprobe(name=binpath.decode(), fn_name=fn_name.decode(),
  addr=addr, pid=pid)
 
  def enumerate_active_probes(self):

tools/bashreadline.py

@@ -56,7 +56,7 @@ class Data(ct.Structure):
 
 def print_event(cpu, data, size):
  event = ct.cast(data, ct.POINTER(Data)).contents
- print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str))
+ print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str.decode()))
 
 b["events"].open_perf_buffer(print_event)
 while 1:

tools/biosnoop.py

@@ -175,7 +175,8 @@ def print_event(cpu, data, size):
  delta = float(delta) + (event.ts - prev_ts)
 
  print("%-14.9f %-14.14s %-6s %-7s %-2s %-9s %-7s %7.2f" % (
- delta / 1000000, event.name, event.pid, event.disk_name, rwflg, val,
+ delta / 1000000, event.name.decode(), event.pid,
+ event.disk_name.decode(), rwflg, val,
  event.len, float(event.delta) / 1000000))
 
  prev_ts = event.ts

tools/biotop.py

@@ -212,9 +212,9 @@ def signal_ignore(signal, frame):
 
  # print line
  avg_ms = (float(v.us) / 1000) / v.io
- print("%-6d %-16s %1s %-3d %-3d %-8s %5s %7s %6.2f" % (k.pid, k.name,
- "W" if k.rwflag else "R", k.major, k.minor, diskname, v.io,
- v.bytes / 1024, avg_ms))
+ print("%-6d %-16s %1s %-3d %-3d %-8s %5s %7s %6.2f" % (k.pid,
+ k.name.decode(), "W" if k.rwflag else "R", k.major, k.minor,
+ diskname, v.io, v.bytes / 1024, avg_ms))
 
  line += 1
  if line >= maxrows:

tools/bitesize.py

@@ -72,4 +72,5 @@
 try:
  sleep(99999999)
 except KeyboardInterrupt:
- dist.print_log2_hist("Kbytes", "Process Name")
+ dist.print_log2_hist("Kbytes", "Process Name",
+ section_print_fn=bytes.decode)

tools/btrfsslower.py

@@ -311,12 +311,12 @@ def print_event(cpu, data, size):
 
  if (csv):
  print("%d,%s,%d,%s,%d,%d,%d,%s" % (
- event.ts_us, event.task, event.pid, type, event.size,
- event.offset, event.delta_us, event.file))
+ event.ts_us, event.task.decode(), event.pid, type, event.size,
+ event.offset, event.delta_us, event.file.decode()))
  return
  print("%-8s %-14.14s %-6s %1s %-7s %-8d %7.2f %s" % (strftime("%H:%M:%S"),
- event.task, event.pid, type, event.size, event.offset / 1024,
- float(event.delta_us) / 1000, event.file))
+ event.task.decode(), event.pid, type, event.size, event.offset / 1024,
+ float(event.delta_us) / 1000, event.file.decode()))
 
 # initialize BPF
 b = BPF(text=bpf_text)

tools/cachetop.py

@@ -72,7 +72,7 @@ def get_processes_stats(
  counts = bpf.get_table("counts")
  stats = defaultdict(lambda: defaultdict(int))
  for k, v in counts.items():
- stats["%d-%d-%s" % (k.pid, k.uid, k.comm)][k.ip] = v.value
+ stats["%d-%d-%s" % (k.pid, k.uid, k.comm.decode())][k.ip] = v.value
  stats_list = []
 
  for pid, count in sorted(stats.items(), key=lambda stat: stat[0]):

tools/capable.py

@@ -148,7 +148,8 @@ def print_event(cpu, data, size):
  else:
  name = "?"
  print("%-9s %-6d %-6d %-16s %-4d %-20s %d" % (strftime("%H:%M:%S"),
- event.uid, event.pid, event.comm, event.cap, name, event.audit))
+ event.uid, event.pid, event.comm.decode(), event.cap, name,
+ event.audit))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event)

tools/dcsnoop.py

@@ -147,8 +147,8 @@ class Data(ct.Structure):
 def print_event(cpu, data, size):
  event = ct.cast(data, ct.POINTER(Data)).contents
  print("%-11.6f %-6d %-16s %1s %s" % (
- time.time() - start_ts, event.pid, event.comm, mode_s[event.type],
- event.filename))
+ time.time() - start_ts, event.pid, event.comm.decode(),
+ mode_s[event.type], event.filename.decode()))
 
 # header
 print("%-11s %-6s %-16s %1s %s" % ("TIME(s)", "PID", "COMM", "T", "FILE"))

tools/ext4dist.py

@@ -196,7 +196,7 @@
  if args.interval and (not args.notimestamp):
  print(strftime("%H:%M:%S:"))
 
- dist.print_log2_hist(label, "operation")
+ dist.print_log2_hist(label, "operation", section_print_fn=bytes.decode)
  dist.clear()
 
  countdown -= 1

tools/ext4slower.py

@@ -305,12 +305,12 @@ def print_event(cpu, data, size):
 
  if (csv):
  print("%d,%s,%d,%s,%d,%d,%d,%s" % (
- event.ts_us, event.task, event.pid, type, event.size,
- event.offset, event.delta_us, event.file))
+ event.ts_us, event.task.decode(), event.pid, type, event.size,
+ event.offset, event.delta_us, event.file.decode()))
  return
  print("%-8s %-14.14s %-6s %1s %-7s %-8d %7.2f %s" % (strftime("%H:%M:%S"),
- event.task, event.pid, type, event.size, event.offset / 1024,
- float(event.delta_us) / 1000, event.file))
+ event.task.decode(), event.pid, type, event.size, event.offset / 1024,
+ float(event.delta_us) / 1000, event.file.decode()))
 
 # initialize BPF
 b = BPF(text=bpf_text)

tools/filelife.py

@@ -132,7 +132,7 @@ class Data(ct.Structure):
 def print_event(cpu, data, size):
  event = ct.cast(data, ct.POINTER(Data)).contents
  print("%-8s %-6d %-16s %-7.2f %s" % (strftime("%H:%M:%S"), event.pid,
- event.comm, float(event.delta) / 1000, event.fname))
+ event.comm.decode(), float(event.delta) / 1000, event.fname.decode()))
 
 b["events"].open_perf_buffer(print_event)
 while 1:

tools/fileslower.py

@@ -235,13 +235,13 @@ def print_event(cpu, data, size):
  event = ct.cast(data, ct.POINTER(Data)).contents
 
  ms = float(event.delta_us) / 1000
- name = event.name
+ name = event.name.decode()
  if event.name_len > DNAME_INLINE_LEN:
  name = name[:-3] + "..."
 
  print("%-8.3f %-14.14s %-6s %1s %-7s %7.2f %s" % (
- time.time() - start_ts, event.comm, event.pid, mode_s[event.mode],
- event.sz, ms, name))
+ time.time() - start_ts, event.comm.decode(), event.pid,
+ mode_s[event.mode], event.sz, ms, name))
 
 b["events"].open_perf_buffer(print_event, page_cnt=64)
 while 1:

tools/gethostlatency.py

@@ -123,7 +123,7 @@ class Data(ct.Structure):
 def print_event(cpu, data, size):
  event = ct.cast(data, ct.POINTER(Data)).contents
  print("%-9s %-6d %-16s %10.2f %s" % (strftime("%H:%M:%S"), event.pid,
- event.comm, (event.delta / 1000000), event.host))
+ event.comm.decode(), (event.delta / 1000000), event.host.decode()))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event)

tools/hardirqs.py

@@ -148,7 +148,7 @@
  else:
  print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label))
  for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
- print("%-26s %11d" % (k.name, v.value / factor))
+ print("%-26s %11d" % (k.name.decode(), v.value / factor))
  dist.clear()
 
  countdown -= 1

tools/killsnoop.py

@@ -131,7 +131,7 @@ def print_event(cpu, data, size):
  return
 
  print("%-9s %-6d %-16s %-4d %-6d %d" % (strftime("%H:%M:%S"),
- event.pid, event.comm, event.sig, event.tpid, event.ret))
+ event.pid, event.comm.decode(), event.sig, event.tpid, event.ret))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event)

tools/mdflush.py

@@ -60,8 +60,8 @@ class Data(ct.Structure):
 # process event
 def print_event(cpu, data, size):
  event = ct.cast(data, ct.POINTER(Data)).contents
- print("%-8s %-6d %-16s %s" % (strftime("%H:%M:%S"), event.pid, event.comm,
- event.disk))
+ print("%-8s %-6d %-16s %s" % (strftime("%H:%M:%S"), event.pid,
+ event.comm.decode(), event.disk.decode()))
 
 # read events
 b["events"].open_perf_buffer(print_event)

tools/offcputime.py

@@ -276,7 +276,7 @@ def signal_ignore(signal, frame):
  print(" --")
  for addr in user_stack:
  print(" %s" % b.sym(addr, k.tgid))
- print(" %-16s %s (%d)" % ("-", k.name, k.pid))
+ print(" %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
  print(" %d\n" % v.value)
 
 if missing_stacks > 0:

tools/offwaketime.py

@@ -306,7 +306,7 @@ def signal_ignore(signal, frame):
 
  else:
  # print wakeup name then stack in reverse order
- print(" %-16s %s %s" % ("waker:", k.waker, k.t_pid))
+ print(" %-16s %s %s" % ("waker:", k.waker.decode(), k.t_pid))
  for addr in waker_user_stack:
  print(" %s" % b.sym(addr, k.tgid))
  if args.delimited:
@@ -324,7 +324,7 @@ def signal_ignore(signal, frame):
  print(" -")
  for addr in target_user_stack:
  print(" %s" % b.sym(addr, k.tgid))
- print(" %-16s %s %s" % ("target:", k.target, k.w_pid))
+ print(" %-16s %s %s" % ("target:", k.target.decode(), k.w_pid))
  print(" %d\n" % v.value)
 
 if missing_stacks > 0:

tools/oomkill.py

@@ -68,7 +68,8 @@ def print_event(cpu, data, size):
  avgline = stats.read().rstrip()
  print(("%s Triggered by PID %d (\"%s\"), OOM kill of PID %d (\"%s\")"
  ", %d pages, loadavg: %s") % (strftime("%H:%M:%S"), event.fpid,
- event.fcomm, event.tpid, event.tcomm, event.pages, avgline))
+ event.fcomm.decode(), event.tpid, event.tcomm.decode(), event.pages,
+ avgline))
 
 # initialize BPF
 b = BPF(text=bpf_text)

tools/opensnoop.py

@@ -175,7 +175,7 @@ def print_event(cpu, data, size):
 
  print("%-6d %-16s %4d %3d %s" %
  (event.id & 0xffffffff if args.tid else event.id >> 32,
- event.comm, fd_s, err, event.fname))
+ event.comm.decode(), fd_s, err, event.fname.decode()))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event, page_cnt=64)

tools/profile.py

@@ -292,7 +292,7 @@ def aksym(addr):
  print(" --")
  for addr in user_stack:
  print(" %s" % b.sym(addr, k.pid))
- print(" %-16s %s (%d)" % ("-", k.name, k.pid))
+ print(" %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
  print(" %d\n" % v.value)
 
 # check missing

tools/slabratetop.py

@@ -120,7 +120,7 @@ def signal_ignore(signal, frame):
  line = 0
  for k, v in reversed(sorted(counts.items(),
  key=lambda counts: counts[1].size)):
- print("%-32s %6d %10d" % (k.name, v.count, v.size))
+ print("%-32s %6d %10d" % (k.name.decode(), v.count, v.size))
 
  line += 1
  if line >= maxrows:

tools/sslsniff.py

@@ -200,10 +200,11 @@ def print_event(cpu, data, size, rw):
  " bytes lost) " + "-" * 5
 
  print("%-12s %-18.9f %-16s %-6d %-6d\n%s\n%s\n%s\n\n" % (rw, time_s,
- event.comm,
+ event.comm.decode(),
  event.pid,
  event.len,
- s_mark, event.v0,
+ s_mark,
+ event.v0.decode(),
  e_mark))
 
 b["perf_SSL_write"].open_perf_buffer(print_event_write)

tools/stacksnoop.py

@@ -115,7 +115,7 @@ def print_event(cpu, data, size):
 
  if verbose:
  print("%-18.9f %-12.12s %-6d %-3d %s" %
- (ts, event.comm, event.pid, cpu, function))
+ (ts, event.comm.decode(), event.pid, cpu, function))
  else:
  print("%-18.9f %s" % (ts, function))
 

tools/statsnoop.py

@@ -155,8 +155,8 @@ def print_event(cpu, data, size):
  if args.timestamp:
  print("%-14.9f" % (float(event.ts_ns - start_ts) / 1000000000), end="")
 
- print("%-6d %-16s %4d %3d %s" % (event.pid, event.comm,
- fd_s, err, event.fname))
+ print("%-6d %-16s %4d %3d %s" % (event.pid, event.comm.decode(),
+ fd_s, err, event.fname.decode()))
 
 # loop with callback to print_event
 b["events"].open_perf_buffer(print_event, page_cnt=64)

tools/tcpaccept.py

@@ -159,8 +159,9 @@ def print_ipv4_event(cpu, data, size):
  if start_ts == 0:
  start_ts = event.ts_us
  print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
- print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
- event.ip, inet_ntop(AF_INET, pack("I", event.daddr)),
+ print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
+ event.task.decode(), event.ip,
+ inet_ntop(AF_INET, pack("I", event.daddr)),
  inet_ntop(AF_INET, pack("I", event.saddr)), event.lport))
 
 def print_ipv6_event(cpu, data, size):
@@ -170,8 +171,8 @@ def print_ipv6_event(cpu, data, size):
  if start_ts == 0:
  start_ts = event.ts_us
  print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
- print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
- event.ip, inet_ntop(AF_INET6, event.daddr),
+ print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
+ event.task.decode(), event.ip, inet_ntop(AF_INET6, event.daddr),
  inet_ntop(AF_INET6, event.saddr), event.lport))
 
 # initialize BPF

tools/tcpconnect.py

@@ -201,8 +201,9 @@ def print_ipv4_event(cpu, data, size):
  if start_ts == 0:
  start_ts = event.ts_us
  print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
- print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
- event.ip, inet_ntop(AF_INET, pack("I", event.saddr)),
+ print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
+ event.task.decode(), event.ip,
+ inet_ntop(AF_INET, pack("I", event.saddr)),
  inet_ntop(AF_INET, pack("I", event.daddr)), event.dport))
 
 def print_ipv6_event(cpu, data, size):
@@ -213,7 +214,7 @@ def print_ipv6_event(cpu, data, size):
  start_ts = event.ts_us
  print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
  print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
- event.task, event.ip, inet_ntop(AF_INET6, event.saddr),
+ event.task.decode(), event.ip, inet_ntop(AF_INET6, event.saddr),
  inet_ntop(AF_INET6, event.daddr), event.dport))
 
 # initialize BPF

tools/tcpconnlat.py

@@ -199,8 +199,9 @@ def print_ipv4_event(cpu, data, size):
  if start_ts == 0:
  start_ts = event.ts_us
  print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
- print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid, event.task,
- event.ip, inet_ntop(AF_INET, pack("I", event.saddr)),
+ print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid,
+ event.task.decode(), event.ip,
+ inet_ntop(AF_INET, pack("I", event.saddr)),
  inet_ntop(AF_INET, pack("I", event.daddr)), event.dport,
  float(event.delta_us) / 1000))
 
@@ -211,8 +212,8 @@ def print_ipv6_event(cpu, data, size):
  if start_ts == 0:
  start_ts = event.ts_us
  print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
- print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid, event.task,
- event.ip, inet_ntop(AF_INET6, event.saddr),
+ print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid,
+ event.task.decode(), event.ip, inet_ntop(AF_INET6, event.saddr),
  inet_ntop(AF_INET6, event.daddr), event.dport,
  float(event.delta_us) / 1000))