Introduce notion of a test group, use for CA tests

Grouping our tests should make it easier to understand the intent than
one long poorly-arranged list. It also is convenient for running just
the tests for a specific component when working on that component.

We need at least one test group so this isn't dead code; I decided to
collect the tests for the `ca-derivations` and `dynamic-derivations`
experimental features in groups. Do
```bash
make ca.test-group -jN
```
and
```bash
make dyn-drv.test-group -jN
```
to try running just them.

I originally did this as part of #8397 for being able to just the local
overlay store alone. I am PRing it separately now so we can separate
general infra from new features.

Co-authored-by: Valentin Gagarin <[email protected]>

Makefile

@@ -27,6 +27,8 @@ makefiles += \
  src/libstore/tests/local.mk \
  src/libexpr/tests/local.mk \
  tests/local.mk \
+ tests/ca/local.mk \
+ tests/dyn-drv/local.mk \
  tests/test-libstoreconsumer/local.mk \
  tests/plugins/local.mk
 else

doc/manual/src/contributing/testing.md

@@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
 The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
 Each test is a bash script.
 
+### Running the whole test suite
+
 The whole test suite can be run with:
 
 ```shell-session
@@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS]
 ...
 ```
 
+### Grouping tests
+
+Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
+Each test group is in a subdirectory of `tests`.
+For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
+
+That test group can be run like this:
+
+```shell-session
+$ make ca.test-group -j50
+ran test tests/ca/nix-run.sh... [PASS]
+ran test tests/ca/import-derivation.sh... [PASS]
+...
+```
+
+The test group is defined in Make like this:
+```makefile
+$(test-group-name)-tests := \
+ $(d)/test0.sh \
+ $(d)/test1.sh \
+ ...
+
+install-tests-groups += $(test-group-name)
+```
+
+### Running individual tests
+
 Individual tests can be run with `make`:
 
 ```shell-session

mk/lib.mk

@@ -10,6 +10,7 @@ bin-scripts :=
 noinst-scripts :=
 man-pages :=
 install-tests :=
+install-tests-groups :=
 
 ifdef HOST_OS
  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
@@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
 $(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
 $(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
 $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
-$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
+$(foreach test, $(install-tests), \
+ $(eval $(call run-install-test,$(test))) \
+ $(eval installcheck: $(test).test))
+$(foreach test-group, $(install-tests-groups), \
+ $(eval $(call run-install-test-group,$(test-group))) \
+ $(eval installcheck: $(test-group).test-group) \
+ $(foreach test, $($(test-group)-tests), \
+ $(eval $(call run-install-test,$(test))) \
+ $(eval $(test-group).test-group: $(test).test)))
+
 $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
 
 
@@ -151,6 +161,14 @@ ifdef libs-list
  @echo "The following libraries can be built:"
  @echo ""
  @for i in $(libs-list); do echo " $$i"; done
+endif
+ifdef install-tests-groups
+ @echo ""
+ @echo "The following groups of functional tests can be run:"
+ @echo ""
+ @for i in $(install-tests-groups); do echo " $$i.test-group"; done
+ @echo ""
+ @echo "(installcheck includes tests in test groups too.)"
 endif
  @echo ""
  @echo "The following variables control the build:"

mk/tests.mk

@@ -4,8 +4,6 @@ test-deps =
 
 define run-install-test
 
- installcheck: $1.test
-
  .PHONY: $1.test
  $1.test: $1 $(test-deps)
  @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
@@ -16,6 +14,12 @@ define run-install-test
 
 endef
 
+define run-install-test-group
+
+ .PHONY: $1.test-group
+
+endef
+
 .PHONY: check installcheck
 
 print-top-help += \

tests/ca/local.mk

@@ -0,0 +1,27 @@
+ca-tests := \
+ $(d)/build-with-garbage-path.sh \
+ $(d)/build.sh \
+ $(d)/concurrent-builds.sh \
+ $(d)/derivation-json.sh \
+ $(d)/duplicate-realisation-in-closure.sh \
+ $(d)/gc.sh \
+ $(d)/import-derivation.sh \
+ $(d)/new-build-cmd.sh \
+ $(d)/nix-copy.sh \
+ $(d)/nix-run.sh \
+ $(d)/nix-shell.sh \
+ $(d)/post-hook.sh \
+ $(d)/recursive.sh \
+ $(d)/repl.sh \
+ $(d)/selfref-gc.sh \
+ $(d)/signatures.sh \
+ $(d)/substitute.sh \
+ $(d)/why-depends.sh
+
+install-tests-groups += ca
+
+clean-files += \
+ $(d)/config.nix
+
+test-deps += \
+ tests/ca/config.nix

tests/dyn-drv/local.mk

@@ -0,0 +1,11 @@
+dyn-drv-tests := \
+ $(d)/text-hashed-output.sh \
+ $(d)/recursive-mod-json.sh
+
+install-tests-groups += dyn-drv
+
+clean-files += \
+ $(d)/config.nix
+
+test-deps += \
+ tests/dyn-drv/config.nix

tests/local.mk

@@ -14,7 +14,6 @@ nix_tests = \
  flakes/absolute-paths.sh \
  flakes/build-paths.sh \
  flakes/flake-in-submodule.sh \
- ca/gc.sh \
  gc.sh \
  nix-collect-garbage-d.sh \
  remote-store.sh \
@@ -28,8 +27,6 @@ nix_tests = \
  user-envs-migration.sh \
  binary-cache.sh \
  multiple-outputs.sh \
- ca/build.sh \
- ca/new-build-cmd.sh \
  nix-build.sh \
  gc-concurrent.sh \
  repair.sh \
@@ -47,24 +44,17 @@ nix_tests = \
  referrers.sh \
  optimise-store.sh \
  substitute-with-invalid-ca.sh \
- ca/concurrent-builds.sh \
  signing.sh \
- ca/build-with-garbage-path.sh \
  hash.sh \
  gc-non-blocking.sh \
  check.sh \
- ca/substitute.sh \
  nix-shell.sh \
- ca/signatures.sh \
- ca/nix-shell.sh \
- ca/nix-copy.sh \
  check-refs.sh \
  build-remote-input-addressed.sh \
  secure-drv-outputs.sh \
  restricted.sh \
  fetchGitSubmodules.sh \
  flakes/search-root.sh \
- ca/duplicate-realisation-in-closure.sh \
  readfile-context.sh \
  nix-channel.sh \
  recursive.sh \
@@ -80,10 +70,7 @@ nix_tests = \
  nar-access.sh \
  pure-eval.sh \
  eval.sh \
- ca/post-hook.sh \
  repl.sh \
- ca/repl.sh \
- ca/recursive.sh \
  binary-cache-build-remote.sh \
  search.sh \
  logging.sh \
@@ -109,13 +96,8 @@ nix_tests = \
  fmt.sh \
  eval-store.sh \
  why-depends.sh \
- ca/why-depends.sh \
  derivation-json.sh \
- ca/derivation-json.sh \
  import-derivation.sh \
- ca/import-derivation.sh \
- dyn-drv/text-hashed-output.sh \
- dyn-drv/recursive-mod-json.sh \
  nix_path.sh \
  case-hack.sh \
  placeholders.sh \
@@ -124,8 +106,7 @@ nix_tests = \
  build.sh \
  build-delete.sh \
  output-normalization.sh \
- ca/nix-run.sh \
- selfref-gc.sh ca/selfref-gc.sh \
+ selfref-gc.sh \
  db-migration.sh \
  bash-profile.sh \
  pass-as-file.sh \
@@ -150,16 +131,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
 
 clean-files += \
  $(d)/common/vars-and-functions.sh \
- $(d)/config.nix \
- $(d)/ca/config.nix \
- $(d)/dyn-drv/config.nix
+ $(d)/config.nix
 
 test-deps += \
  tests/common/vars-and-functions.sh \
  tests/config.nix \
- tests/ca/config.nix \
- tests/test-libstoreconsumer/test-libstoreconsumer \
- tests/dyn-drv/config.nix
+ tests/test-libstoreconsumer/test-libstoreconsumer
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
  test-deps += tests/plugins/libplugintest.$(SO_EXT)