add reset-trace (iovisor#766)

README.md

@@ -109,6 +109,7 @@ Examples:
 - tools/[opensnoop](tools/opensnoop.py): Trace open() syscalls. [Examples](tools/opensnoop_example.txt).
 - tools/[pidpersec](tools/pidpersec.py): Count new processes (via fork). [Examples](tools/pidpersec_example.txt).
 - tools/[profile](tools/profile.py): Profile CPU usage by sampling stack traces at a timed interval. [Examples](tools/profile_example.txt).
+- tools/[reset-trace](tools/reset-trace.sh): Reset the state of tracing. Maintenance tool only. [Examples](tools/reset-trace_example.txt).
 - tools/[runqlat](tools/runqlat.py): Run queue (scheduler) latency as a histogram. [Examples](tools/runqlat_example.txt).
 - tools/[softirqs](tools/softirqs.py): Measure soft IRQ (soft interrupt) event time. [Examples](tools/softirqs_example.txt).
 - tools/[solisten](tools/solisten.py): Trace TCP socket listen. [Examples](tools/solisten_example.txt).

man/man8/reset-trace.8

@@ -0,0 +1,59 @@
+.TH reset-trace 8 "2016-10-18" "USER COMMANDS"
+.SH NAME
+reset-trace \- reset the state of tracing.
+.SH SYNOPSIS
+.B reset-trace [\-F] [\-h] [\-q] [\-v]
+.SH DESCRIPTION
+You will probably never need this tool. If you kill \-9 a bcc tool (plus other
+signals, like SIGTERM), or if a bcc tool crashes, then kernel tracing can be
+left in a semi-enabled state. It's not as bad as it sounds: there may just be
+overhead for writing to ring buffers that are never read. This tool can be
+used to clean up the tracing state, and reset and disable active tracing.
+
+Make sure no other tracing sessions are active. This tool might stop them from
+functioning (perhaps ungracefully).
+
+This specifically clears the state in at least the following files in
+/sys/kernel/debug/tracing: kprobe_events, uprobe_events, trace_pipe.
+Other tracing facilities (ftrace) are checked, and if not in an expected state,
+a note is printed. All tracing files can be reset with \-F for force, but this
+will interfere with any other running tracing sessions (eg, ftrace).
+.SH REQUIREMENTS
+/sys/kernel/debug mounted as debugfs
+.SH OPTIONS
+.TP
+\-F
+Force. Will reset all tracing facilities, including those not used by bcc
+(ftrace). You shouldn't need to use this.
+.TP
+\-h
+USAGE message.
+.TP
+\-q
+Quiet. No output while working.
+.TP
+\-v
+Verbose: print what it is doing.
+.SH EXAMPLES
+.TP
+Reset the state of tracing:
+#
+.B reset-trace
+.TP
+Verbose:
+#
+.B reset-trace \-v
+.TP
+.SH SOURCE
+This is from bcc.
+.IP
+https://github.com/iovisor/bcc
+.PP
+Also look in the bcc distribution for a companion _examples.txt file containing
+example usage, output, and commentary for this tool.
+.SH OS
+Linux
+.SH STABILITY
+Unstable - in development.
+.SH AUTHOR
+Brendan Gregg

tools/reset-trace.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+#
+# reset-trace - reset state of tracing, disabling all tracing.
+# Written for Linux.
+#
+# If a bcc tool crashed and you suspect tracing is partially enabled, you
+# can use this tool to reset the state of tracing, disabling anything still
+# enabled. Only use this tool in the case of error, and, consider filing a
+# bcc ticket so we can fix the error.
+#
+# bcc-used tracing facilities are reset. Other tracing facilities (ftrace) are
+# checked, and if not in an expected state, a note is printed. All tracing
+# files can be reset with -F for force, but this will interfere with any other
+# running tracing sessions (eg, ftrace).
+#
+# USAGE: ./reset-trace [-Fhqv]
+#
+# REQUIREMENTS: debugfs mounted on /sys/kernel/debug
+#
+# COPYRIGHT: Copyright (c) 2016 Brendan Gregg.
+# Licensed under the Apache License, Version 2.0 (the "License")
+#
+# 20-Jul-2014 Brendan Gregg Created this.
+# 18-Oct-2016 " " Updated for bcc use.
+
+tracing=/sys/kernel/debug/tracing
+opt_force=0; opt_verbose=0; opt_quiet=0
+
+function usage {
+ cat <<-END >&2
+ USAGE: reset-trace [-Fhqv]
+ -F # force: reset all tracing files
+ -v # verbose: print details while working
+ -h # this usage message
+ -q # quiet: no output
+ eg,
+ reset-trace # disable semi-enabled tracing
+END
+ exit
+}
+
+function die {
+ echo >&2 "$@"
+ exit 1
+}
+
+function vecho {
+ (( ! opt_verbose )) && return
+ echo "$@"
+}
+
+function writefile {
+ file=$1
+ write=$2
+ if [[ ! -w $file ]]; then
+ echo >&2 "WARNING: file $file not writable/exists. Skipping."
+ return
+ fi
+
+ vecho "Checking $PWD/$file"
+ contents=$(grep -v '^#' $file)
+ if [[ "$contents" != "$expected" ]]; then
+ (( ! opt_quiet )) && echo "Needed to reset $PWD/$file"
+ vecho "$file, before (line enumerated):"
+ (( opt_verbose )) && cat -nv $file
+ cmd="echo $write > $file"
+ if ! eval "$cmd"; then
+ echo >&2 "WARNING: command failed \"$cmd\"." \
+ "bcc still running? Continuing."
+ fi
+ vecho "$file, after (line enumerated):"
+ (( opt_verbose )) && cat -nv $file
+ vecho
+ fi
+}
+
+# only write when force is used
+function checkfile {
+ file=$1
+ write=$2
+ expected=$3
+ if [[ ! -e $file ]]; then
+ echo >&2 "WARNING: file $file doesn't exist. Skipping."
+ return
+ fi
+ if (( opt_force )); then
+ writefile $file $write
+ return
+ fi
+ (( opt_quiet )) && return
+
+ vecho "Checking $PWD/$file"
+ contents=$(grep -v '^#' $file)
+ if [[ "$contents" != "$expected" ]]; then
+ echo "Noticed unrelated tracing file $PWD/$file isn't set as" \
+ "expected. Not reseting (-F to force, -v for verbose)."
+ vecho "Contents of $file is (line enumerated):"
+ (( opt_verbose )) && cat -nv $file
+ vecho "Expected \"$expected\"."
+ fi
+}
+
+### process options
+while getopts Fhqv opt
+do
+ case $opt in
+ F) opt_force=1 ;;
+ q) opt_quiet=1 ;;
+ v) opt_verbose=1 ;;
+ h|?) usage ;;
+ esac
+done
+shift $(( $OPTIND - 1 ))
+
+### reset tracing state
+vecho "Reseting tracing state..."
+vecho
+cd $tracing || die "ERROR: accessing tracing. Root user? /sys/kernel/debug?"
+
+# files bcc uses
+writefile kprobe_events "" ""
+writefile uprobe_events "" ""
+writefile trace "" "" # clears trace_pipe
+
+# non-bcc files
+checkfile current_tracer nop nop
+checkfile set_ftrace_filter "" ""
+checkfile set_graph_function "" ""
+checkfile set_ftrace_pid "" "no pid"
+checkfile events/enable 0 0
+checkfile tracing_thresh 0 0
+checkfile tracing_on 1 1
+
+vecho
+vecho "Done."