regression

#!/bin/bash

#
# libseccomp regression test automation script
#
# Copyright IBM Corp. 2012
# Author: Corey Bryant <coreyb@linux.vnet.ibm.com>
#

#
# This library is free software; you can redistribute it and/or modify it
# under the terms of version 2.1 of the GNU Lesser General Public License as
# published by the Free Software Foundation.
#
# This library is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, see <http:https://www.gnu.org/licenses>.
#

GLBL_ARCH_LE_SUPPORT=" \
	x86 x86_64 x32 \
	arm aarch64 \
	mipsel mipsel64 mipsel64n32 \
	ppc64le \
	riscv64 \
	sh"
GLBL_ARCH_BE_SUPPORT=" \
	mips mips64 mips64n32 \
	parisc parisc64 \
	ppc ppc64 \
	s390 s390x \
	sheb"

GLBL_ARCH_32B_SUPPORT=" \
	x86 x32 \
	arm \
	mips mipsel mips64n32 mipsel64n32 \
	parisc \
	ppc \
	s390 \
	sheb sh"

GLBL_ARCH_64B_SUPPORT=" \
	x86_64 \
	aarch64 \
	mips64 \
	parisc64 \
	ppc64 \
	riscv64 \
	s390x"

GLBL_SYS_ARCH="../tools/scmp_arch_detect"
GLBL_SYS_RESOLVER="../tools/scmp_sys_resolver"
GLBL_SYS_SIM="../tools/scmp_bpf_sim"
GLBL_SYS_API="../tools/scmp_api_level"

####
# functions

#
# Dependency check
#
# Arguments:
#     1    Dependency to check for
#
function check_deps() {
	[[ -z "$1" ]] && return
	which "$1" >& /dev/null
	return $?
}

#
# Dependency verification
#
# Arguments:
#     1    Dependency to check for
#
function verify_deps() {
	[[ -z "$1" ]] && return
	if ! check_deps "$1"; then
		echo "error: install \"$1\" and include it in your \$PATH"
		exit 1
	fi
}

#
# Print out script usage details
#
function usage() {
cat << EOF
usage: regression [-h] [-v] [-j JOBS] [-m MODE] [-a] [-b BATCH_NAME]
                  [-l <LOG>] [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>]

libseccomp regression test automation script
optional arguments:
  -h             show this help message and exit
  -j JOBS        run up to JOBS test jobs in parallel
                  can also be set via LIBSECCOMP_TSTCFG_JOBS env variable
  -m MODE        specified the test mode [c (default), python]
                  can also be set via LIBSECCOMP_TSTCFG_MODE_LIST env variable
  -a             specifies all tests are to be run
  -b BATCH_NAME  specifies batch of tests to be run
                  can also be set via LIBSECCOMP_TSTCFG_BATCHES env variable
  -l [LOG]       specifies log file to write test results to
  -s SINGLE_TEST specifies individual test number to be run
  -t [TEMP_DIR]  specifies directory to create temporary files in
  -T [TEST_TYPE] only run tests matching the specified type
                  can also be set via LIBSECCOMP_TSTCFG_TYPE env variable
  -v             specifies that verbose output be provided
EOF
}

#
# Match on a single word/column in a CSV string
#
# Arguments:
#     1    string containing the CSV
#     2    string containing the word to match
#
# Returns true/0 if a match is found false/1 otherwise.
#
function match_csv_word() {
	[[ -z $1 || -z $2 ]] && return 1

	echo "$1" | sed 's/,/ /g' | grep -w "$2"
}

#
# Generate a string representing the test number
#
# Arguments:
#     1    string containing the batch name
#     2    value of the test number from the input test data file
#     3    value of the subtest number that corresponds to argument 1
#
#  The actual test number from the input test data file is 1 for the first
#  test found in the file, 2 for the second, etc.
#
#  The subtest number is useful for batches that generate multiple tests based
#  on a single line of input from the test data file.  The subtest number
#  should be set to zero if the  corresponding test data is actual test data
#  that was read from the input file, and should be set to a value greater than
#  zero if the corresponding test data is generated.
#
function generate_test_num() {
	local testnumstr=$(printf '%s%%%%%03d-%05d' "$1" $2 $3)
	echo "$testnumstr"
}

#
# Print the test data to the log file
#
# Arguments:
#     1    string containing generated test number
#     2    string containing line of test data
#
function print_data() {
	if [[ -n $verbose ]]; then
		printf "Test %s data:     %s\n" "$1" "$2" >&$logfd
	fi
}

#
# Print the test result to the log file
#
# Arguments:
#     1    string containing generated test number
#     2    string containing the test result (INFO, SUCCESS, ERROR, or FAILURE)
#     3    string containing addition details
#
function print_result() {
	if [[ $2 == "INFO" && -z $verbose ]]; then
		return
	fi
	if [[ $3 == "" ]]; then
		printf "Test %s result:   %s\n" "$1" "$2" >&$logfd
	else
		printf "Test %s result:   %s %s\n" "$1" "$2" "$3" >&$logfd
	fi
}

#
# Print the valgrind header to the log file
#
# Arguments:
#     1    string containing generated test number
#
function print_valgrind() {
	if [[ -n $verbose ]]; then
		printf "Test %s valgrind output\n" "$1" >&$logfd
	fi
}

#
# Get the low or high range value from a range specification
#
# Arguments:
#     1    value specifying range value to retrieve: low (1) or high (2)
#     2    string containing dash-separated range or a single value
#
function get_range() {
	if [[ $2 =~ ^[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$ ]]; then
		# if there's a dash, get the low or high range value
		range_val=$(echo "$2" | cut -d'-' -f "$1")
	else
		# otherwise there should just be a single value
		range_val="$2"
	fi
	echo "$range_val"
}

#
# Get the number sequence for a given range with increments of 1, i.e.
# implement a specialized seq(1).
#
# We use our own implementation based on miniseq in favour to the standard seq
# tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit
# ARM for large numbers (see the mailing thread at
# https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA).
#
# Arguments:
#     1    starting value
#     2    last value
#
function get_seq() {
	# NOTE: this whole thing is a bit hacky, but we need to search around
	#       for miniseq to fix 'make distcheck', someday we should fix this
	if [[ -x ./miniseq ]]; then
		./miniseq "$1" "$2"
	elif [[ -x $basedir/miniseq ]]; then
		$basedir/miniseq "$1" "$2"
	else
		# we're often run from a subshell, so we can't simply exit
		echo "error: unable to find miniseq" >&2
		kill $pid
	fi
}

#
# Run the specified test command (with valgrind if requested)
#
# Arguments:
#     1    string containing generated test number
#     2    string containing command name
#     3    string containing command options
#     4    number for the stdout fd
#     5    number for the stderr fd
#
function run_test_command() {
	local cmd

	if [[ $mode == "python" ]]; then
		cmd="PYTHONPATH=$PYTHONPATH"
		cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)"
		# check and adjust if we are doing a VPATH build
		if [[ -e "./$2.py" ]]; then
			cmd="$cmd /usr/bin/env python $2.py $3"
		else
			cmd="$cmd /usr/bin/env python ${srcdir}/$2.py $3"
		fi
	else
		cmd="$2 $3"
	fi

	# setup the stdout/stderr redirects
	local stdout=$4
	local stderr=$5
	[[ -z $stdout ]] && stdout=$logfd
	[[ -z $stderr ]] && stderr=$logfd

	# run the command
	eval "$cmd" 1>&$stdout 2>&$stderr

	# return the command's return code
	return $?
}

#
# Generate pseudo-random string of alphanumeric characters
#
# The generated string will be no larger than the corresponding
# architecture's register size.
#
function generate_random_data() {
	local rcount
	local rdata
	if [[ $arch == "x86_64" ]]; then
		rcount=$[ ($RANDOM % 16) + 1 ]
	else
		rcount=$[ ($RANDOM % 8) + 1 ]
	fi
	rdata=$(dd if=/dev/urandom bs=64 count=1 status=none | \
		md5sum | awk '{ print $1 }' | head -c"$rcount")
	echo "$rdata"
}

#
# Run the specified "bpf-sim-fuzz" test
#
# Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and
# then run a simulated system call test with pseudo-random fuzz data for the
# syscall and argument values.  Tests that belong to this test type provide the
# following data on a single line in the input batch file:
#
#     Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
#     StressCount - The number of fuzz tests to run against the filter
#
# The following test data is output to the logfile for each generated test:
#
#     Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
#     Syscall - The fuzzed syscall value to be simulated against the filter
#     Arg0-5 - The fuzzed syscall arg values to be simulated against the filter
#
# Arguments:
#     1    string containing the batch name
#     2    value of test number from batch file
#     3    string containing line of test data from batch file
#
function run_test_bpf_sim_fuzz() {
	local rc

	# begin splitting the test data from the line into individual variables
	local line=($3)
	local testname=${line[0]}
	local stress_count=${line[1]}

	# check for stress count configuration via environment variables
	[[ -n $LIBSECCOMP_TSTCFG_STRESSCNT ]] && \
		stress_count=$LIBSECCOMP_TSTCFG_STRESSCNT

	for i in $(get_seq 1 $stress_count); do
		local sys=$(generate_random_data)
		local -a arg=($(generate_random_data) $(generate_random_data) \
			      $(generate_random_data) $(generate_random_data) \
			      $(generate_random_data) $(generate_random_data))

		# get the generated sub-test num string
		local testnumstr=$(generate_test_num "$1" $2 $i)

		# set up log file test data line for this individual test,
		# spacing is added to align the output in the correct columns
		local -a COL_WIDTH=(26 17 17 17 17 17 17)
		local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname)
		testdata+=$(printf "%-${COL_WIDTH[1]}s" $sys)
		testdata+=$(printf "%-${COL_WIDTH[2]}s" ${arg[0]})
		testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[1]})
		testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[2]})
		testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[3]})
		testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[4]})
		testdata+=$(printf "%s" ${arg[5]})

		# print out the generated test data to the log file
		print_data "$testnumstr" "$testdata"

		# set up the syscall argument values to be passed to bpf_sim
		for i in {0..5}; do
			arg[$i]=" -$i ${arg[$i]} "
		done

		# run the test command and put the BPF filter in a temp file
		exec 4>$tmpfile
		run_test_command "$testnumstr" "./$testname" "-b" 4 ""
		rc=$?
		exec 4>&-
		if [[ $rc -ne 0 ]]; then
			print_result $testnumstr "ERROR" "$testname rc=$rc"
			stats_error=$(($stats_error+1))
			return
		fi

		# simulate the fuzzed syscall data against the BPF filter, we
		# don't verify the resulting action since we're just testing for
		# stability
		allow=$($GLBL_SYS_SIM -f $tmpfile -s $sys \
			${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \
			${arg[5]})
		rc=$?
		if [[ $rc -ne 0 ]]; then
			print_result $testnumstr "ERROR" "bpf_sim rc=$rc"
			stats_error=$(($stats_error+1))
		else
			print_result $testnumstr "SUCCESS" ""
			stats_success=$(($stats_success+1))
		fi
		stats_all=$(($stats_all+1))
	done
}

#
# Run the specified "bpf-sim" test
#
# Tests that belong to the "bpf-sim" test type generate a BPF filter and then
# run a simulated system call test to validate the filter.  Tests that belong to
# this test type provide the following data on a single line in the input batch
# file:
#
#     Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
#     Arch - The architecture that the test should be run on (all, x86, x86_64)
#     Syscall - The syscall to simulate against the generated filter
#     Arg0-5 - The syscall arguments to simulate against the generated filter
#     Result - The expected simulation result (ALLOW, KILL, etc.)
#
# If a range of syscall or argument values are specified (e.g. 1-9), a test is
# generated for every combination of range values.  Otherwise, the individual
# test is run.
#
# Arguments:
#     1    string containing the batch name
#     2    value of test number from batch file
#     3    string containing line of test data from batch file
#
function run_test_bpf_sim() {
	local rc
	local LOW=1
	local HIGH=2
	local -a arg_empty=(false false false false false false)

	# begin splitting the test data from the line into individual variables
	local line=($3)
	local testname=${line[0]}
	local testarch=${line[1]}
	local low_syscall  #line[2]
	local high_syscall #line[2]
	local -a low_arg   #line[3-8]
	local -a high_arg  #line[3-8]
	local result=${line[9]}

	# expand the architecture list
	local simarch_tmp
	local simarch_avoid
	simarch_tmp=""
	simarch_avoid=""
	for arch_i in $(echo $testarch | sed -e 's/,/ /g'); do
		case $arch_i in
		all)
			# add the native arch
			simarch_tmp+=" $arch"
			;;
		all_le)
			# add the native arch only if it is little endian
			if echo "$GLBL_ARCH_LE_SUPPORT" | grep -qw "$arch"; then
				simarch_tmp+=" $arch"
			fi
			;;
		+all_le)
			# add all of the little endian architectures
			simarch_tmp+=" $GLBL_ARCH_LE_SUPPORT"
			;;
		all_be)
			# add the native arch only if it is big endian
			if echo "$GLBL_ARCH_BE_SUPPORT" | grep -qw "$arch"; then
				simarch_tmp+=" $arch"
			fi
			;;
		+all_be)
			# add all of the big endian architectures
			simarch_tmp+=" $GLBL_ARCH_BE_SUPPORT"
			;;
		all_32)
			# add the native arch only if it is 32-bit
			if echo "$GLBL_ARCH_32B_SUPPORT" | grep -qw "$arch"; then
				simarch_tmp+=" $arch"
			fi
			;;
		+all_32)
			# add all of the 32-bit architectures
			simarch_tmp+=" $GLBL_ARCH_32B_SUPPORT"
			;;
		all_64)
			# add the native arch only if it is 64-bit
			if echo "$GLBL_ARCH_64B_SUPPORT" | grep -qw "$arch"; then
				simarch_tmp+=" $arch"
			fi
			;;
		+all_64)
			# add all of the 64-bit architectures
			simarch_tmp+=" $GLBL_ARCH_64B_SUPPORT"
			;;
		+*)
			# add the architecture specified
			simarch_tmp+=" ${arch_i:1}"
			;;
		-*)
			# remove the architecture specified
			simarch_avoid+=" ${arch_i:1}"
			;;
		*)
			# add the architecture specified if it is native
			if [[ "$arch_i" == "$arch" ]]; then
				simarch_tmp+=" $arch_i"
			fi
			;;
		esac
	done

	# make sure we remove any undesired architectures
	local simarch_list
	simarch_list=""
	for arch_i in $simarch_tmp; do
		if echo "$simarch_avoid" | grep -q -v -w "$arch_i"; then
			simarch_list+=" $arch_i"
		fi
	done
	simarch_list=$(echo $simarch_list | sed -e 's/  / /g;s/^ //;')

	# do we have any architectures remaining in the list?
	if [[ $simarch_list == "" ]]; then
		print_result $(generate_test_num "$1" $2 1) "SKIPPED" \
			"(architecture difference)"
		stats_skipped=$(($stats_skipped+1))
		return
	fi

	# get low and high range arg values
	line_i=3
	for arg_i in {0..5}; do
		low_arg[$arg_i]=$(get_range $LOW "${line[$line_i]}")
		high_arg[$arg_i]=$(get_range $HIGH "${line[$line_i]}")

		# fix up empty arg values so the nested loops work
		if [[ ${low_arg[$arg_i]} == "N" ]]; then
			arg_empty[$arg_i]=true
			low_arg[$arg_i]=0
			high_arg[$arg_i]=0
		fi

		line_i=$(($line_i+1))
	done

	# loop through the selected architectures
	for simarch in $simarch_list; do
		# print architecture header if necessary
		if [[ $simarch != $simarch_list ]]; then
			echo " test arch:  $simarch" >&$logfd
		fi

		# reset the subtest number
		local subtestnum=1

		# get low and high syscall values and convert them to numbers
		low_syscall=$(get_range $LOW "${line[2]}")
		if [[ ! $low_syscall =~ ^\-?[0-9]+$ ]]; then
			low_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \
				      $low_syscall)
			if [[ $? -ne 0 ]]; then
				print_result $(generate_test_num "$1" $2 1) \
					     "ERROR" "sys_resolver rc=$?"
				stats_error=$(($stats_error+1))
				return
			fi
		fi
		high_syscall=$(get_range $HIGH "${line[2]}")
		if [[ ! $high_syscall =~ ^\-?[0-9]+$ ]]; then
			high_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \
				       $high_syscall)
			if [[ $? -ne 0 ]]; then
				print_result $(generate_test_num "$1" $2 1) \
					     "ERROR" "sys_resolver rc=$?"
				stats_error=$(($stats_error+1))
				return
			fi
		fi

		# if ranges exist, the following will loop through all syscall
		# and arg ranges and generate/run every combination of requested
		# tests; if no ranges were specified, then the single test is
		# run
		for sys in $(get_seq $low_syscall $high_syscall); do
		for arg0 in $(get_seq ${low_arg[0]} ${high_arg[0]}); do
		for arg1 in $(get_seq ${low_arg[1]} ${high_arg[1]}); do
		for arg2 in $(get_seq ${low_arg[2]} ${high_arg[2]}); do
		for arg3 in $(get_seq ${low_arg[3]} ${high_arg[3]}); do
		for arg4 in $(get_seq ${low_arg[4]} ${high_arg[4]}); do
		for arg5 in $(get_seq ${low_arg[5]} ${high_arg[5]}); do
			local -a arg=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5)

			# Get the generated sub-test num string
			local testnumstr=$(generate_test_num "$1" $2 \
					   $subtestnum)

			# format any empty args to print to log file
			for i in {0..5}; do
				if ${arg_empty[$i]}; then
					arg[$i]="N"
				fi
			done

			# set up log file test data line for this
			# individual test, spacing is added to align
			# the output in the correct columns
			local -a COL_WIDTH=(26 08 14 11 17 21 09 06 06)
			local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname)
			testdata+=$(printf "%-${COL_WIDTH[1]}s" $simarch)
			testdata+=$(printf "%-${COL_WIDTH[2]}s" $sys)
			testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[0]})
			testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[1]})
			testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[2]})
			testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[3]})
			testdata+=$(printf "%-${COL_WIDTH[7]}s" ${arg[4]})
			testdata+=$(printf "%-${COL_WIDTH[8]}s" ${arg[5]})
			testdata+=$(printf "%-${COL_WIDTH[9]}s" $result)

			# print out the test data to the log file
			print_data "$testnumstr" "$testdata"

			# set up the syscall arguments to be passed to bpf_sim
			for i in {0..5}; do
				if ${arg_empty[$i]}; then
					arg[$i]=""
				else
					arg[$i]=" -$i ${arg[$i]} "
				fi
			done

			# run the test command and put the BPF in a temp file
			exec 4>$tmpfile
			run_test_command "$testnumstr" "./$testname" "-b" 4 ""
			rc=$?
			exec 4>&-
			if [[ $rc -ne 0 ]]; then
				print_result $testnumstr \
					     "ERROR" "$testname rc=$rc"
				stats_error=$(($stats_error+1))
				return
			fi

			# simulate the specified syscall against the BPF filter
			# and verify the results
			action=$($GLBL_SYS_SIM -a $simarch -f $tmpfile \
				 -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \
				 ${arg[3]} ${arg[4]} ${arg[5]})
			rc=$?
			if [[ $rc -ne 0 ]]; then
				print_result $testnumstr \
					     "ERROR" "bpf_sim rc=$rc"
				stats_error=$(($stats_error+1))
			elif [[ "$action" != "$result" ]]; then
				print_result $testnumstr "FAILURE" \
					     "bpf_sim resulted in $action"
				stats_failure=$(($stats_failure+1))
			else
				print_result $testnumstr "SUCCESS" ""
				stats_success=$(($stats_success+1))
			fi
			stats_all=$(($stats_all+1))

			subtestnum=$(($subtestnum+1))
		done # syscall
		done # arg0
		done # arg1
		done # arg2
		done # arg3
		done # arg4
		done # arg5
	done # architecture
}

#
# Run the specified "basic" test
#
# Tests that belong to the "basic" test type will simply have the command
# specified in the input batch file.  The command must return zero for success
# and non-zero for failure.
#
# Arguments:
#     1    value of test number from batch file
#     2    string containing line of test data from batch file
#
function run_test_basic() {
	local rc
	local cmd

	# if the test is a script, only run it in native/c mode
	if [[ $mode != "c" && "$2" == *.sh ]]; then
		print_result "$1" "SKIPPED" "(only valid in native/c mode)"
		stats_skipped=$(($stats_skipped+1))
		return
	fi

	# print out the input test data to the log file
	print_data "$1" "$2"

	# check and adjust if we are doing a VPATH build
	if [[ -x "./$2" ]]; then
		cmd="./$2"
	else
		cmd="${srcdir}/$2"
	fi

	# run the command
	run_test_command "$1" "$cmd" "" "" ""
	rc=$?
	if [[ $rc -ne 0 ]]; then
		print_result $1 "FAILURE" "$2 rc=$rc"
		stats_failure=$(($stats_failure+1))
	else
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	fi
	stats_all=$(($stats_all+1))
}

#
# Run the specified "bpf-valgrind" test
#
# Tests that belong to the "bpf-valgrind" test type generate a BPF filter
# while running under valgrind to detect any memory errors.
#
# Arguments:
#     1    value of test number from batch file
#     2    string containing line of test data from batch file
#
function run_test_bpf_valgrind() {
	local rc

	# we only support the native/c test mode here
	if [[ $mode != "c" ]]; then
		print_result "$1" "SKIPPED" "(only valid in native/c mode)"
		stats_skipped=$(($stats_skipped+1))
		return
	fi

	# print out the input test data to the log file
	print_data "$1" "$2"

	# build the command
	testvalgrind="valgrind \
		       --tool=memcheck \
		       --error-exitcode=1 \
		       --leak-check=full \
		       --read-var-info=yes \
		       --track-origins=yes \
		       --suppressions=$basedir/valgrind_test.supp"
	if [[ -n $logfile ]]; then
		testvalgrind+=" --log-fd=$logfd"
	fi
	if [[ -z $verbose ]]; then
		testvalgrind+=" --quiet --log-fd=4"
	fi

	# run the command
	exec 4>/dev/null
	print_valgrind "$1"
	run_test_command "$1" "$testvalgrind --" "./$2 -b" 4 2
	rc=$?
	exec 4>&-
	if [[ $rc -ne 0 ]]; then
		print_result $1 "FAILURE" "$2 rc=$rc"
		stats_failure=$(($stats_failure+1))
	else
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	fi
	stats_all=$(($stats_all+1))
}

#
# Run the specified "live" test
#
# Tests that belong to the "live" test type will attempt to run a live test
# of the libseccomp library on the host system; for obvious reasons the host
# system must support seccomp mode 2 for this to work correctly.
#
# Arguments:
#     1    value of test number from batch file
#     2    string containing line of test data from batch file
#
function run_test_live() {
	local rc
	local api
	local line=($2)

	# parse the test line
	line_cmd=${line[0]}
	line_api=${line[1]}
	line_act=${line[2]}
	line_test="$line_cmd $line_api $line_act"

	# check the api level
	api=$($GLBL_SYS_API)
	if [[ $api -lt $line_api ]]; then
		# runtime api level is too low
		print_result "$1" "SKIPPED" "(api level)"
		stats_skipped=$(($stats_skipped+1))
		return
	fi

	# print out the input test data to the log file
	print_data "$1" "$2"

	# run the command
	exec 4>/dev/null
	run_test_command "$1" "./$line_cmd" "$line_act" "" 4
	rc=$?
	exec 4>&-
	stats_all=$(($stats_all+1))

	# setup the arch specific return values
	case "$arch" in
	x86|x86_64|x32|arm|aarch64|parisc|parisc64|ppc|ppc64|ppc64le|ppc|s390|s390x|riscv64|sh|sheb)
		rc_kill_process=159
		rc_kill=159
		rc_allow=160
		rc_trap=161
		rc_trace=162
		rc_errno=163
		rc_log=164
		;;
	mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32)
		rc_kill_process=140
		rc_kill=140
		rc_allow=160
		rc_trap=161
		rc_trace=162
		rc_errno=163
		rc_log=164
		;;
	*)
		print_result $testnumstr "ERROR" "arch $arch not supported"
		stats_error=$(($stats_error+1))
		return
		;;
	esac

	# verify the results
	if [[ $line_act == "KILL_PROCESS" && $rc -eq $rc_kill_process ]]; then
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	elif [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	elif [[ $line_act == "TRACE" ]]; then
		print_result $1 "ERROR" "unsupported action \"$line_act\""
		stats_error=$(($stats_error+1))
	elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	elif [[ $line_act == "LOG" && $rc -eq $rc_log ]]; then
		print_result $1 "SUCCESS" ""
		stats_success=$(($stats_success+1))
	else
		print_result $1 "FAILURE" "$line_test rc=$rc"
		stats_failure=$(($stats_failure+1))
	fi
}

#
# Run a single test from the specified batch
#
# Arguments:
#     1    string containing the batch name
#     2    value of test number from batch file
#     3    string containing line of test data from batch file
#     4    string containing test type that this test belongs to
#
function run_test() {
	# generate the test number string for the line of batch test data
	local testnumstr=$(generate_test_num "$1" $2 1)

	# ensure we only run tests which match the specified type
	match_csv_word "$type" "$4"
	local type_match=$?
	[[ -n $type && $type_match -eq 1 ]] && return

	# execute the function corresponding to the test type
	if [[ "$4" == "basic" ]]; then
		run_test_basic "$testnumstr" "$3"
	elif [[ "$4" == "bpf-sim" ]]; then
		run_test_bpf_sim "$1" $2 "$3"
	elif [[ "$4" == "bpf-sim-fuzz" ]]; then
		run_test_bpf_sim_fuzz "$1" $2 "$3"
	elif [[ "$4" == "bpf-valgrind" ]]; then
		# only run this test if valgrind is installed
		if check_deps valgrind; then
			run_test_bpf_valgrind "$testnumstr" "$3"
		else
			print_result $testnumstr "SKIPPED" \
				"(valgrind not installed)"
			stats_skipped=$(($stats_skipped+1))
		fi
	elif [[ "$4" == "live" ]]; then
		# only run this test if explicitly requested
		if [[ -n $type ]]; then
			run_test_live "$testnumstr" "$3"
		else
			print_result $testnumstr "SKIPPED" \
				"(must specify live tests)"
			stats_skipped=$(($stats_skipped+1))
		fi
	else
		print_result $testnumstr "ERROR" "test type $4 not supported"
		stats_error=$(($stats_error+1))
	fi
}

#
# Run the requested test batch
#
# Arguments:
#     1    Batch name
#
function run_test_batch() {
	local testnum=1
	local batch_name=$1

	# open temporary file
	if [[ -n $tmpdir ]]; then
		tmpfile=$(mktemp -t regression_XXXXXX --tmpdir=$tmpdir)
	else
		tmpfile=$(mktemp -t regression_XXXXXX)
	fi

	# reset the stats
	stats_all=0
	stats_skipped=0
	stats_success=0
	stats_failure=0
	stats_error=0

	# print a test batch header
	echo " batch name: $batch_name" >&$logfd

	# loop through each line and run the requested tests
	while read line; do
		# strip whitespace, comments, and blank lines
		line=$(echo "$line" | \
			sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \
			sed -e '/^[#].*$/d;/^$/d')
		if [[ -z $line ]]; then
			continue
		fi

		if [[ $line =~ ^"test type": ]]; then
			test_type=$(echo "$line" | \
					sed -e 's/^test type: //;')
			# print a test mode and type header
			echo " test mode:  $mode" >&$logfd
			echo " test type:  $test_type" >&$logfd
			continue
		fi

		if [[ ${single_list[@]} ]]; then
			for i in ${single_list[@]}; do
				if [ $i -eq $testnum ]; then
					# we're running a single test
					run_test "$batch_name" \
							$testnum "$line" \
							"$test_type"
				fi
			done
		else
			# we're running a test from a batch
			run_test "$batch_name" \
					$testnum "$line" "$test_type"
		fi
		testnum=$(($testnum+1))
	done < "$file"


	# dump our stats
	local stats=$batch_name.$mode.stats
	> $stats
	echo -n "$stats_all $stats_skipped $stats_success " >> $stats
	echo -n "$stats_failure $stats_error " >> $stats
	echo "" >> $stats

	# cleanup the temporary file we created
	rm -f $tmpfile
}

#
# Run the requested test batch
#
# Arguments:
#     1    Log file
#     2    PID to watch
#
function tail_log() {
	local log=$1
	local pid=$2

	# dump the output
	tail -n +0 --pid=$pid -f $log

	# accumulate the stats
	local stats=$(echo $log | sed 's/\.log$/.stats/')
	stats_all=$(( $stats_all + $(awk '{ print $1 }' $stats) ))
	stats_skipped=$(( $stats_skipped + $(awk '{ print $2 }' $stats) ))
	stats_success=$(( $stats_success + $(awk '{ print $3 }' $stats) ))
	stats_failure=$(( $stats_failure + $(awk '{ print $4 }' $stats) ))
	stats_error=$(( $stats_error + $(awk '{ print $5 }' $stats) ))
}

#
# Run the requested tests
#
function run_tests() {
	local job_cnt=0
	local tail_cnt=0
	local -a job_pids
	local -a job_logs

	# loop through all test files
	for file in $basedir/*.tests; do
		local batch_requested=false
		local batch_name=""

		# extract the batch name from the file name
		batch_name=$(basename $file .tests)

		# check if this batch was requested
		if [[ ${batch_list[@]} ]]; then
			for b in ${batch_list[@]}; do
				if [[ $b == $batch_name ]]; then
					batch_requested=true
					break
				fi
			done
			if ! $batch_requested; then
				continue
			fi
		fi

		# run the test batch
		run_test_batch $batch_name >& $batch_name.$mode.log &
		job_pids[job_cnt]=$!
		job_logs[job_cnt]=$batch_name.$mode.log
		job_cnt=$(( $job_cnt + 1 ))

		# output the next log if the job queue is full
		if [[ $(jobs | wc -l) -ge $jobs ]]; then
			tail_log ${job_logs[$tail_cnt]} ${job_pids[$tail_cnt]}
			tail_cnt=$(( $tail_cnt + 1 ))
		fi
	done

	# output any leftovers
	for i in $(seq $tail_cnt $(( $job_cnt - 1 ))); do
		tail_log ${job_logs[$i]} ${job_pids[$i]}
	done
}

####
# main

# verify general script dependencies
verify_deps head
verify_deps sed
verify_deps awk
verify_deps tr

# global variables
declare -a batch_list
declare -a single_list
arch=
batch_count=0
logfile=
logfd=
mode_list=""
runall=
singlecount=0
tmpfile=""
tmpdir=""
type=
verbose=
jobs=1
stats_all=0
stats_skipped=0
stats_success=0
stats_failure=0
stats_error=0

# set the test root directory
basedir=$(dirname $0)

# set the test harness pid
pid=$$

# parse the command line
while getopts "ab:gj:l:m:s:t:T:vh" opt; do
	case $opt in
	a)
		runall=1
		;;
	b)
		batch_list[batch_count]="$OPTARG"
		batch_count=$(($batch_count+1))
		;;
	j)
		jobs=$OPTARG
		;;
	l)
		logfile="$OPTARG"
		;;
	m)
		case $OPTARG in
		c)
			mode_list="$mode_list c"
			;;
		python)
			verify_deps python
			mode_list="$mode_list python"
			;;
		*)
			usage
			exit 1
		esac
		;;
	s)
		single_list[single_count]=$OPTARG
		single_count=$(($single_count+1))
		;;
	t)
		tmpdir="$OPTARG"
		;;
	T)
		type="$OPTARG"
		;;
	v)
		verbose=1
		;;
	h|*)
		usage
		exit 1
		;;
	esac
done

# use mode list from environment if provided
[[ -z $mode_list && -n $LIBSECCOMP_TSTCFG_MODE_LIST ]] && mode_list=$LIBSECCOMP_TSTCFG_MODE_LIST

# use job count from environment if provided and do some sanity checking
[[ -n $LIBSECCOMP_TSTCFG_JOBS ]] && jobs=$LIBSECCOMP_TSTCFG_JOBS
if [[ $jobs -lt 1 ]]; then
	jobs=$(cat /proc/cpuinfo | grep "^processor" | wc -l)
fi

# determine the mode test automatically
if [[ -z $mode_list ]]; then
	# always perform the native c tests
	mode_list="c"

	# query the build configuration
	if [[ -r "../configure.h" ]]; then
		# python tests
		[[ "$(grep "ENABLE_PYTHON" ../configure.h | \
		   awk '{ print $3 }')" = "1" ]] && \
			mode_list="$mode_list python"
	fi
fi

# check if we specified a list of tests via the environment variable
if [[ -n $LIBSECCOMP_TSTCFG_BATCHES ]]; then
	for i in $(echo "$LIBSECCOMP_TSTCFG_BATCHES" | sed 's/,/ /g'); do
		batch_list[batch_count]="$i"
		batch_count=$(($batch_count+1))
	done
fi

# default to all tests if batch or single tests not requested
if [[ -z $batch_list ]] && [[ -z $single_list ]]; then
	runall=1
fi

# drop any requested batch and single tests if all tests were requested
if [[ -n $runall ]]; then
	batch_list=()
	single_list=()
fi

# check for configuration via environment variables
[[ -z $type && -n $LIBSECCOMP_TSTCFG_TYPE ]] && type=$LIBSECCOMP_TSTCFG_TYPE

# open log file for append (default to stdout)
if [[ -n $logfile ]]; then
	# force single threaded to preserve the output
	jobs=1

	logfd=3
	exec 3>>"$logfile"
else
	logfd=1
fi

# determine the current system's architecture
arch=$($GLBL_SYS_ARCH)

# display the test output and run the requested tests
echo "=============== $(date) ===============" >&$logfd
echo "Regression Test Report (\"regression $*\")" >&$logfd
for mode in $mode_list; do
	run_tests
done
echo "Regression Test Summary" >&$logfd
echo " tests run: $stats_all" >&$logfd
echo " tests skipped: $stats_skipped" >&$logfd
echo " tests passed: $stats_success" >&$logfd
echo " tests failed: $stats_failure" >&$logfd
echo " tests errored: $stats_error" >&$logfd
echo "============================================================" >&$logfd

# cleanup and exit
rc=0
[[ $stats_failure -gt 0 ]] && rc=$(($rc + 2))
[[ $stats_error -gt 0 ]] && rc=$(($rc + 4))

exit $rc