killsnoop

README.md

@@ -66,6 +66,7 @@ Tools:
 
 - tools/[biosnoop](tools/biosnoop): Trace block device I/O with PID and latency. [Examples](tools/biosnoop_example.txt).
 - tools/[funccount](tools/funccount): Count kernel function calls. [Examples](tools/funccount_example.txt).
+- tools/[killsnoop](tools/killsnoop): Trace signals issued by the kill() syscall. [Examples](tools/killsnoop_example.txt).
 - tools/[opensnoop](tools/opensnoop): Trace open() syscalls. [Examples](tools/opensnoop_example.txt).
 - tools/[pidpersec](tools/pidpersec): Count new processes (via fork). [Examples](tools/pidpersec_example.txt).
 - tools/[syncsnoop](tools/syncsnoop): Trace sync() syscall. [Examples](tools/syncsnoop_example.txt).

man/man8/killsnoop.8

@@ -0,0 +1,86 @@
+.TH killsnoop 8 "2015-08-20" "USER COMMANDS"
+.SH NAME
+killsnoop \- Trace signals issued by the kill() syscall. Uses Linux eBPF/bcc.
+.SH SYNOPSIS
+.B killsnoop [\-h] [\-t] [\-x] [-p PID]
+.SH DESCRIPTION
+killsnoop traces the kill() syscall, to show signals sent via this method. This
+may be useful to troubleshoot failing applications, where an unknown mechanism
+is sending signals.
+
+This works by tracing the kernel sys_kill() function using dynamic tracing, and
+will need updating to match any changes to this function.
+
+Since this uses BPF, only the root user can use this tool.
+.SH REQUIREMENTS
+CONFIG_BPF and bcc.
+.SH OPTIONS
+.TP
+\-h
+Print usage message.
+.TP
+\-t
+Include a timestamp column.
+.TP
+\-x
+Only print failed kill() syscalls.
+.TP
+\-p PID
+Trace this process ID only (filtered in-kernel).
+.SH EXAMPLES
+.TP
+Trace all kill() syscalls:
+#
+.B killsnoop
+.TP
+Trace all kill() syscalls, and include timestamps:
+#
+.B killsnoop \-t
+.TP
+Trace only kill() syscalls that failed:
+#
+.B killsnoop \-x
+.TP
+Trace PID 181 only:
+#
+.B killsnoop \-p 181
+.SH FIELDS
+.TP
+TIME(s)
+Time of the call, in seconds.
+.TP
+PID
+Source process ID
+.TP
+COMM
+Source process name
+.TP
+SIG
+Signal number. See signal(7).
+.TP
+TPID
+Target process ID
+.TP
+RES
+Result. 0 == success, a negative value (of the error code) for failure.
+.SH OVERHEAD
+This traces the kernel kill function and prints output for each event. As the
+rate of this is generally expected to be low (< 100/s), the overhead is also
+expected to be negligible. If you have an application that is calling a very
+high rate of kill()s for some reason, then test and understand overhead before
+use.
+.SH SOURCE
+This is from bcc.
+.IP
+https://github.com/iovisor/bcc
+.PP
+Also look in the bcc distribution for a companion _examples.txt file containing
+example usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg
+.SH SEE ALSO
+opensnoop(8), funccount(8)

tools/killsnoop

@@ -0,0 +1,105 @@
+#!/usr/bin/python
+#
+# killsnoop Trace signals issued by the kill() syscall.
+# For Linux, uses BCC, eBPF. Embedded C.
+#
+# USAGE: killsnoop [-h] [-t] [-x] [-p PID]
+#
+# Copyright (c) 2015 Brendan Gregg.
+# Licensed under the Apache License, Version 2.0 (the "License")
+#
+# 20-Sep-2015 Brendan Gregg Created this.
+
+from __future__ import print_function
+from bcc import BPF
+import argparse
+
+# arguments
+examples = """examples:
+ ./killsnoop # trace all kill() signals
+ ./killsnoop -t # include timestamps
+ ./killsnoop -x # only show failed kills
+ ./killsnoop -p 181 # only trace PID 181
+"""
+parser = argparse.ArgumentParser(
+ description="Trace signals issued by the kill() syscall",
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=examples)
+parser.add_argument("-t", "--timestamp", action="store_true",
+ help="include timestamp on output")
+parser.add_argument("-x", "--failed", action="store_true",
+ help="only show failed opens")
+parser.add_argument("-p", "--pid",
+ help="trace this PID only")
+args = parser.parse_args()
+debug = 0
+
+# define BPF program
+bpf_text = """
+#include <uapi/linux/ptrace.h>
+
+BPF_HASH(args_pid, u32, int);
+BPF_HASH(args_sig, u32, int);
+
+int kprobe__sys_kill(struct pt_regs *ctx, int tpid, int sig)
+{
+ u32 pid = bpf_get_current_pid_tgid();
+
+ FILTER
+ args_pid.update(&pid, &tpid);
+ args_sig.update(&pid, &sig);
+
+ return 0;
+};
+
+int kretprobe__sys_kill(struct pt_regs *ctx)
+{
+ int *tpidp, *sigp, ret = ctx->ax;
+ u32 pid = bpf_get_current_pid_tgid();
+
+ tpidp = args_pid.lookup(&pid);
+ sigp = args_sig.lookup(&pid);
+ if (tpidp == 0 || sigp == 0) {
+ return 0; // missed entry
+ }
+
+ bpf_trace_printk("%d %d %d\\n", *tpidp, *sigp, ret);
+ args_pid.delete(&pid);
+ args_sig.delete(&pid);
+
+ return 0;
+}
+"""
+if args.pid:
+ bpf_text = bpf_text.replace('FILTER',
+ 'if (pid != %s) { return 0; }' % args.pid)
+else:
+ bpf_text = bpf_text.replace('FILTER', '')
+if debug:
+ print(bpf_text)
+
+# initialize BPF
+b = BPF(text=bpf_text)
+
+# header
+if args.timestamp:
+ print("%-14s" % ("TIME(s)"), end="")
+print("%-6s %-16s %-4s %-6s %s" % ("PID", "COMM", "SIG", "TPID", "RESULT"))
+
+start_ts = 0
+
+# format output
+while 1:
+ (task, pid, cpu, flags, ts, msg) = b.trace_fields()
+ (tpid_s, sig_s, ret_s) = msg.split(" ")
+
+ ret = int(ret_s)
+ if (args.failed and (ret >= 0)):
+ continue
+
+ # print columns
+ if args.timestamp:
+ if start_ts == 0:
+ start_ts = ts
+ print("%-14.9f" % (ts - start_ts), end="")
+ print("%-6d %-16s %-4s %-6s %s" % (pid, task, sig_s, tpid_s, ret_s))

tools/killsnoop_example.txt

@@ -0,0 +1,36 @@
+Demonstrations of killsnoop, the Linux eBPF/bcc version.
+
+
+This traces signals sent via the kill() syscall. For example:
+
+# ./killsnoop 
+PID COMM SIG TPID RESULT
+17064 bash 9 27682 0
+17064 bash 9 27682 -3
+17064 bash 0 17064 0
+
+The first line showed a SIGKILL (9) sent from PID 17064 (a bash shell) to
+PID 27682. The result, 0, means success.
+
+The second line showed the same signal sent, this time resulting in a -3
+(ESRCH: no such process).
+
+
+USAGE message:
+
+# ./killsnoop -h
+usage: killsnoop [-h] [-t] [-x] [-p PID]
+
+Trace signals issued by the kill() syscall
+
+optional arguments:
+ -h, --help show this help message and exit
+ -t, --timestamp include timestamp on output
+ -x, --failed only show failed opens
+ -p PID, --pid PID trace this PID only
+
+examples:
+ ./killsnoop # trace all kill() signals
+ ./killsnoop -t # include timestamps
+ ./killsnoop -x # only show failed kills
+ ./killsnoop -p 181 # only trace PID 181