refactor test targets and let build fail on test errors

Makefile.in

@@ -74,6 +74,7 @@ clean:
  for dir in $$(dirname $(ALL_ITEMS)) $(MYDIRS); do \
  $(MAKE) -C $$dir clean; \
  done
+ $(MAKE) -C test clean
  rm -f $(MANPAGES) $(MANPAGES:%=%.gz) firejail*.rpm
  rm -f $(SECCOMP_FILTERS)
  rm -f test/utils/index.html*
@@ -227,45 +228,11 @@ scan-build: clean
 # make test
 #
 
+TESTS=profiles private-lib apps apps-x11 apps-x11-xorg sysutils utils environment filters arguments fs fcopy fnetfilter
+TEST_TARGETS=$(patsubst %,test-%,$(TESTS))
 
-test-profiles:
- cd test/profiles; ./profiles.sh | grep TESTING
-
-test-private-lib:
- cd test/private-lib; ./private-lib.sh | grep TESTING
-
-test-apps:
- cd test/apps; ./apps.sh | grep TESTING
-
-test-apps-x11:
- cd test/apps-x11; ./apps-x11.sh | grep TESTING
-
-test-apps-x11-xorg:
- cd test/apps-x11-xorg; ./apps-x11-xorg.sh | grep TESTING
-
-test-sysutils:
- cd test/sysutils; ./sysutils.sh | grep TESTING
-
-test-utils:
- cd test/utils; ./utils.sh | grep TESTING
-
-test-environment:
- cd test/environment; ./environment.sh | grep TESTING
-
-test-filters:
- cd test/filters; ./filters.sh | grep TESTING
-
-test-arguments:
- cd test/arguments; ./arguments.sh | grep TESTING
-
-test-fs:
- cd test/fs; ./fs.sh | grep TESTING
-
-test-fcopy:
- cd test/fcopy; ./fcopy.sh | grep TESTING
-
-test-fnetfilter:
- cd test/fnetfilter; ./fnetfilter.sh | grep TESTING
+$(TEST_TARGETS):
+ $(MAKE) -C test $(subst test-,,$@)
 
 test: test-profiles test-private-lib test-fcopy test-fnetfilter test-fs test-utils test-sysutils test-environment test-apps test-apps-x11 test-apps-x11-xorg test-filters test-arguments
  echo "TEST COMPLETE"
@@ -284,32 +251,32 @@ test-travis: test-profiles test-fcopy test-fnetfilter test-fs test-utils test-sy
 
 # a firejail-test account is required, public/private key setup
 test-ssh:
- cd test/ssh; ./ssh.sh | grep TESTING
+ $(MAKE) -C test $(subst test-,,$@)
 
 # requires root access
 test-chroot:
- cd test/chroot; ./chroot.sh | grep testing
+ $(MAKE) -C test $(subst test-,,$@)
 
 # Huge appimage files, not included in "make dist" archive
 test-appimage:
- cd test/appimage; ./appimage.sh | grep TESTING
+ $(MAKE) -C test $(subst test-,,$@)
 
 # Root access, network devices are created before the test
 # restart your computer to get rid of these devices
 test-network:
- cd test/network; ./network.sh | grep TESTING
+ $(MAKE) -C test $(subst test-,,$@)
 
 # requires the same setup as test-network
 test-stress:
- cd test/stress; ./stress.sh | grep TESTING
+ $(MAKE) -C test $(subst test-,,$@)
 
 # Tests running a root user
 test-root:
- cd test/root; su -c ./root.sh | grep TESTING
+ $(MAKE) -C test $(subst test-,,$@)
 
 # OverlayFS is not available on all platforms
 test-overlay:
- cd test/overlay; ./overlay.sh | grep TESTING
+ $(MAKE) -C test $(subst test-,,$@)
 
 # For testing hidepid system, the command to set it up is "mount -o remount,rw,hidepid=2 /proc"
 

configure

@@ -4257,7 +4257,7 @@ fi
 
 ac_config_files="$ac_config_files mkdeb.sh"
 
-ac_config_files="$ac_config_files Makefile src/common.mk src/lib/Makefile src/fcopy/Makefile src/fnet/Makefile src/firejail/Makefile src/fnetfilter/Makefile src/firemon/Makefile src/libtrace/Makefile src/libtracelog/Makefile src/firecfg/Makefile src/fbuilder/Makefile src/fsec-print/Makefile src/ftee/Makefile src/faudit/Makefile src/fseccomp/Makefile src/fldd/Makefile src/libpostexecseccomp/Makefile src/fsec-optimize/Makefile src/profstats/Makefile src/man/Makefile"
+ac_config_files="$ac_config_files Makefile src/common.mk src/lib/Makefile src/fcopy/Makefile src/fnet/Makefile src/firejail/Makefile src/fnetfilter/Makefile src/firemon/Makefile src/libtrace/Makefile src/libtracelog/Makefile src/firecfg/Makefile src/fbuilder/Makefile src/fsec-print/Makefile src/ftee/Makefile src/faudit/Makefile src/fseccomp/Makefile src/fldd/Makefile src/libpostexecseccomp/Makefile src/fsec-optimize/Makefile src/profstats/Makefile src/man/Makefile test/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -4988,6 +4988,7 @@ do
  "src/fsec-optimize/Makefile") CONFIG_FILES="$CONFIG_FILES src/fsec-optimize/Makefile" ;;
  "src/profstats/Makefile") CONFIG_FILES="$CONFIG_FILES src/profstats/Makefile" ;;
  "src/man/Makefile") CONFIG_FILES="$CONFIG_FILES src/man/Makefile" ;;
+ "test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;;
 
  *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
  esac

configure.ac

@@ -223,7 +223,7 @@ AC_CONFIG_FILES([mkdeb.sh], [chmod +x mkdeb.sh])
 AC_OUTPUT(Makefile src/common.mk src/lib/Makefile src/fcopy/Makefile src/fnet/Makefile src/firejail/Makefile src/fnetfilter/Makefile \
 src/firemon/Makefile src/libtrace/Makefile src/libtracelog/Makefile src/firecfg/Makefile src/fbuilder/Makefile src/fsec-print/Makefile \
 src/ftee/Makefile src/faudit/Makefile src/fseccomp/Makefile src/fldd/Makefile src/libpostexecseccomp/Makefile src/fsec-optimize/Makefile \
-src/profstats/Makefile src/man/Makefile)
+src/profstats/Makefile src/man/Makefile test/Makefile)
 
 echo
 echo "Configuration options:"

test/Makefile.in

@@ -0,0 +1,11 @@
+TESTS=$(patsubst %/,%,$(wildcard */))
+
+.PHONY: $(TESTS)
+
+$(TESTS):
+ echo $(TESTS)
+ cd $@ && ./$@.sh 2>&1 | tee $@.log
+ cd $@ && grep -a TESTING $@.log && grep -a -L "TESTING ERROR" $@.log
+
+clean:
+ for test in $(TESTS); do rm -f "$$test/$$test.log"; done