Code coverage (#501)

* Delete unused yml file

* Reorganize and tidy `ci.yml`

* Change compilation flags and add switch for code coverage flags

* Fix and consolidate .gitignore files 

* Remove logfile printout

* Separate testing setup from main `Makefile`

* Update `make clean`

* Tidy makefiles

.github/workflows/ci.yml

@@ -9,93 +9,97 @@
 #
 # -----------------------------------------------------------------------------
 
-name: AtChem2-CI
+name: AtChem2 CI
 
 # ------------------------------ EVENTS ------------------------------ #
-# Controls when the action will run
+# Controls when the workflow is activated
 on:
- # Triggers the workflow on push or pull request events but only for the master branch
+ # Triggers when a pull request is created or updated (only on the
+ # master branch)
  pull_request:
  branches: [ master ]
 
- # Allows you to run this workflow manually from the Actions tab
- workflow_dispatch:
-
- # Triggers when pushed to the master (either by merging a Pull Request, or direct commit to master)
+ # Triggers when a push is made to the master branch (either by
+ # merging a pull request, or by direct commit)
  push:
  branches:
  - master
 
+ # Run manually from the Actions tab
+ workflow_dispatch:
+
 # ------------------------------ JOBS ------------------------------ #
-# A workflow run consists of one or more jobs that can run sequentially or in parallel
+# This workflow contains a single job called `testing`, which compiles
+# the AtChem2 model, runs the testsuite, and checks the testsuite code
+# coverage using Codecov
 jobs:
- # This workflow contains a single job called `build`
- build:
- # Type of runner that the job will run on - we use a matrix to run on two OSs
- # Runner images are described here: https://github.com/actions/runner-images
+ testing:
+  # The job runs on both linux (ubuntu) and macos runner images,
+ # with three versions of gnu fortran -- the runner images are
+ # described here: https://github.com/actions/runner-images
  runs-on: ${{ matrix.os }}
  strategy:
  matrix:
- os: [ubuntu-22.04, macos-11, macos-12]
+ os: [ubuntu-22.04, macos-11] #, macos-12]
  fortran: [9, 10, 11]
- exclude: #TODO: fortran9 does not work on macos12
- - os: macos-12
- fortran: 9
+ # exclude: #TODO: fortran9 does not work on macos12
+ # - os: macos-12
+ #  fortran: 9
  fail-fast: false
 
  # -------------------------------------------------------------
- # Steps represent a sequence of tasks that will be executed as part of the job
+ # Sequence of tasks to be executed as part of the `testing` job:
+ # 1. checkout the repository
+ # 2. install gfortran, AtChem2 dependencies
+ # 3. compile AtChem2
+ # 4. run all tests (indent, style, unit, model)
+ # 5. recompile AtChem2, run unit and model tests for Codecov
  steps:
- # Checkout the repository under $GITHUB_WORKSPACE, so the job can access it
+
+ # -------------------------------------------------------------
+ # (1) Checkout the repository under $GITHUB_WORKSPACE
  - uses: actions/checkout@v3
  with:
  fetch-depth: 2
 
  # -------------------------------------------------------------
- # Install gfortran - if not supplied on the OS image
- # macOS-11 only
- - name: Install gfortran-9 on macOS-11
+ # (2) Install gfortran, if not supplied with the runner image,
+ # and the AtChem2 dependencies (CVODE, openlibm, numdiff, FRUIT)
+
+ # gfortran on macos-11
+ - name: Install gfortran-9 (macos-11)
  if: matrix.os == 'macos-11' && matrix.fortran == 9
  run: brew install gcc@${{ matrix.fortran }}
 
- # macOS-12 only
- - name: Install gfortran-9 or gfortran-10 on macOS-12
+ # gfortran on macos-12
+ - name: Install gfortran-9 and gfortran-10 (macos-12)
  if: matrix.os == 'macos-12' && (matrix.fortran == 9 || matrix.fortran == 10)
  run: brew install gcc@${{ matrix.fortran }}
 
- # -------------------------------------------------------------
- # Install cvode
- # ubuntu only
- - name: Install cvode (ubuntu)
- if: startsWith(matrix.os, 'ubuntu')
+ # AtChem2 dependencies
+ - name: Install CVODE
  run: ./tools/install/install_cvode.sh $PWD $(which gfortran-${{ matrix.fortran }})
 
- # macOS only
- - name: Install cvode (macOS)
- if: startsWith(matrix.os, 'macos')
- run: ./tools/install/install_cvode.sh $PWD $(which gfortran-${{ matrix.fortran }})
-
- # -------------------------------------------------------------
- # Install all other dependencies
- # both OSs
  - name: Install openlibm
  run: ./tools/install/install_openlibm.sh $PWD
 
  - name: Install numdiff
  run: ./tools/install/install_numdiff.sh $PWD
 
- - name: Install fruit
+ - name: Install FRUIT
  run: sudo ./tools/install/install_fruit.sh $PWD
 
  # -------------------------------------------------------------
- # Build AtChem2 and run all tests
+ # (3) Compile AtChem2 using the standard compilation flags
  - name: Build AtChem2
- # Set FORT_VERSION for use inside the Makefile (called from build_atchem2.sh)
+ # Set $FORT_VERSION for use with the `Makefile`, called from
+ # the `build_atchem2.sh` script
  env:
  FORT_VERSION: ${{ matrix.fortran }}
  run: |
  cp tools/install/Makefile.skel Makefile
  ./build/build_atchem2.sh ./model/mechanism.fac
+ # macos only
  if [ $RUNNER_OS != "Linux" ] ; then
  install_name_tool -change libopenlibm.4.dylib $PWD/openlibm-0.8.1/libopenlibm.4.dylib atchem2 ;
  install_name_tool -change @rpath/libsundials_cvode.2.dylib $PWD/cvode/lib/libsundials_cvode.dylib atchem2 ;
@@ -104,31 +108,42 @@ jobs:
  fi
 
  # -------------------------------------------------------------
- - name: Indent and style tests
+ # (4) Run all tests: indent, style, unit, model
+
+ - name: Indent and Style tests
  run: |
  make indenttest
  make styletest
 
- # -------------------------------------------------------------
- - name: Run AtChem2
+ - name: Unit and Model tests
+ # Set $FORT_VERSION for use with the `Makefile`
+ env:
+ FORT_VERSION: ${{ matrix.fortran }}
  run: |
- # Run Atchem2
- ./atchem2
- # Upload `build` coverage to codecov
- if [[ $RUNNER_OS == "Linux" ]]; then sudo ln -f -s /usr/bin/gcov-${{ matrix.fortran }} /usr/bin/gcov ; bash <(curl -s https://codecov.io/bash) -F build ; fi
+ export PATH=$PATH:$PWD/numdiff/bin
+ make unittests
+ make oldtests # NB: oldtests will eventually be merged into modeltests
+ #make modeltests # NB: modeltests are temporarily deactivated (pass on linux, fail on macos)
 
  # -------------------------------------------------------------
- - name: Run tests
- # Set FORT_VERSION for use inside the Makefile
+ # (5) Recompile AtChem2 using the code coverage flags, then
+ # upload the gcov files to Codecov
+
+ - name: Unit and Model tests (x Codecov)
+ if: matrix.os == 'ubuntu-22.04' && matrix.fortran == 11
+ # Set $FORT_VERSION for use with the `Makefile`
  env:
  FORT_VERSION: ${{ matrix.fortran }}
  run: |
- # Run unit tests
  export PATH=$PATH:$PWD/numdiff/bin
- make unittests
- # Upload `unittests` coverage to codecov
- if [[ $RUNNER_OS == "Linux" ]]; then sudo ln -f -s /usr/bin/gcov-${{ matrix.fortran }} /usr/bin/gcov ; bash <(curl -s https://codecov.io/bash) -F unittests ; fi
  make clean
- # Run full build tests - this will upload `tests` coverage to codecov for each test
- make oldtests
- #make modeltests #TODO: modeltests are temporarily deactivated (pass on linux, fail on macos)
+ make unittests CCOV=true
+ mv tests/unit_tests/*.gc* ./ # Needed to deal with gcc11 changing the way it handles gcda and gcna files
+ make modeltests CCOV=true # Run only the new model testsuite
+
+ - name: Upload coverage reports to Codecov
+ uses: codecov/codecov-action@v3
+ with:
+ #token: ${{ secrets.CODECOV_TOKEN }}
+ #directory: ./tests/unit_tests/
+ gcov: true

.gitignore

@@ -1,6 +1,7 @@
-# Ignore the executable, the Makefile, and the compiler-generated files in this directory
+# Ignore the executable, the Makefile, the Python bytecode, and the compiler-generated files
 atchem2
 Makefile
+*.pyc
 *.gcda
 *.gcno
 *.xml

tests/.gitignore

@@ -1,15 +1,16 @@
 # Ignore output/log files and mechanism files of the test runs
 *.log
 *.out
+*.output
 mechanism.f90
 mechanism.o
 mechanism.prod
 mechanism.reac
 mechanism.ro2
 mechanism.species
 mechanism.so
-tests/*/output/*
-model_tests/*/output/*
+tests/*/output/reactionRates/*
+model_tests/*/output/reactionRates/*
 
 # Except configuration and reference files
 !*.cmp

tests/makefile.tests

@@ -0,0 +1,84 @@
+# -----------------------------------------------------------------------------
+#
+# Copyright (c) 2009 - 2012 Chris Martin, Kasia Boronska, Jenny Young,
+# Peter Jimack, Mike Pilling
+#
+# Copyright (c) 2017 Sam Cox, Roberto Sommariva
+#
+# This file is part of the AtChem2 software package.
+#
+# This file is covered by the MIT license which can be found in the file
+# LICENSE.md at the top level of the AtChem2 distribution.
+#
+# -----------------------------------------------------------------------------
+
+# This is a secondary makefile, included in the main Makefile. It contains
+# the information needed to set up and run the AtChem2 Testsuite.
+#
+# The following variables are defined in the main Makefile:
+# - $FRUITDIR is the path to FRUIT, the FORTRAN Unit Test Framework
+# - $UNITTEST_SRCS are the Fortran source files needed for the unit tests
+# - $FORT_COMP is the Fortran compiler
+
+
+# ==================== Unit tests ==================== #
+
+# setup FRUIT
+UNITTESTDIR = tests/unit_tests
+fruit_code = $(FRUITDIR)/src/fruit.f90
+unittest_code = $(UNITTEST_SRCS) $(shell ls tests/unit_tests/*_test.f90 )
+unittest_code_gen = $(UNITTESTDIR)/fruit_basket_gen.f90 $(UNITTESTDIR)/fruit_driver_gen.f90
+all_unittest_code = $(fruit_code) $(unittest_code) $(unittest_code_gen)
+fruit_driver = $(UNITTESTDIR)/fruit_driver.exe
+
+# copy `fruit_generator.rb` to the unit tests directory and replace the
+# path to FRUIT with $FRUITDIR
+$(UNITTESTDIR)/fruit_basket_gen.f90 : $(unittest_code)
+ @echo ""
+ @cp tests/fruit_generator.rb $(UNITTESTDIR)
+ @cd $(UNITTESTDIR); \
+ sed -i "18s,.*,load \"$(FRUITDIR)/rake_base.rb\"," fruit_generator.rb; \
+ ruby fruit_generator.rb
+
+# build `fruit_driver.exe` from the individual unit tests
+$(fruit_driver) : $(all_unittest_code)
+ $(FORT_COMP) -o $(fruit_driver) -J$(OBJ) -I$(OBJ) $(all_unittest_code) $(FFLAGS) $(LDFLAGS)
+
+# ==================== Model tests ==================== #
+
+# search `tests/tests/` for all subdirectories, which should reflect the full list of tests
+OLDTESTS := $(shell ls -d tests/tests/*/ | sed 's,tests/tests/,,g' | sed 's,/,,g')
+
+# search `tests/model_tests/` for all subdirectories, which should reflect the full list of model tests
+MODELTESTDIR = tests/model_tests
+MODELTESTS := $(shell ls -d tests/model_tests/*/ | sed 's,tests/model_tests/,,g' | sed 's,/,,g')
+
+# ==================== Makefile rules ==================== #
+
+.PHONY: indenttest styletest unittests oldtests modeltests alltests
+
+indenttest:
+ @echo ""
+ @echo "Make: Running the indent test."
+ @./tests/run_indent_test.sh
+
+styletest:
+ @echo ""
+ @echo "Make: Running the style test."
+ @./tests/run_style_test.sh
+
+unittests: $(fruit_driver)
+ @export DYLD_LIBRARY_PATH=$(FORT_LIB):$(CVODELIBDIR):$(OPENLIBMDIR); $(fruit_driver)
+ @echo ""
+
+oldtests:
+ @echo ""
+ @echo "Make: Running the old tests:" $(OLDTESTS)
+ @./tests/run_tests.sh "$(OLDTESTS)" "$(FORT_LIB):$(CVODELIBDIR):$(OPENLIBMDIR)"
+
+modeltests:
+ @echo ""
+ @echo "Make: Running the model tests:" $(MODELTESTS)
+ @./tests/run_model_tests.sh "$(MODELTESTS)" "$(FORT_LIB):$(CVODELIBDIR):$(OPENLIBMDIR)"
+
+alltests: indenttest styletest oldtests modeltests unittests

tests/run_model_tests.sh

@@ -55,7 +55,7 @@ function find_string {
 # A mismatch generates a test failure. Numdiff is used to cope with small numerical
 # differences due to differing hardware, OS, and package versions.
 #
-# $2 is used to pass CVODELIB in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
+# $2 is used to pass $CVODELIBDIR in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
 #
 # $this_test_failures is used to keep a track of whether each test is passing: empty indicates
 # a pass, while non-empty indicates a failure.
@@ -215,8 +215,6 @@ $this_file_failures"
  echo $this_test_failures >> $LOG_FILE
 done
 
-if [[ "$RUNNER_OS" == "Linux" ]]; then bash <(curl -s https://codecov.io/bash) -F tests ; fi
-
 # After all tests are run, exit with a FAIL if $fail_counter>0, otherwise PASS.
 if [[ "$fail_counter" -gt 0 ]]; then
  echo "==> Model tests FAILED [" $fail_counter/$test_counter "]"
@@ -229,5 +227,5 @@ echo "" >> $LOG_FILE
 echo "Execution of model tests script finished." >> $LOG_FILE
 
 echo "==> Model tests logfile:" $LOG_FILE
-cat $LOG_FILE
+
 exit $model_tests_passed

tests/run_tests.sh

@@ -55,7 +55,7 @@ function find_string {
 # A mismatch generates a test failure. Numdiff is used to cope with small numerical
 # differences due to differing hardware, OS, and package versions.
 #
-# $2 is used to pass CVODELIB in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
+# $2 is used to pass $CVODELIBDIR in from Makefile, in order to be able to set DYLD_LIBRARY_PATH on macOS.
 #
 # $this_test_failures is used to keep a track of whether each test is passing: empty indicates
 # a pass, while non-empty indicates a failure.
@@ -215,8 +215,6 @@ $this_file_failures"
  echo $this_test_failures >> $LOG_FILE
 done
 
-if [[ "$RUNNER_OS" == "Linux" ]]; then bash <(curl -s https://codecov.io/bash) -F tests ; fi
-
 # After all tests are run, exit with a FAIL if $fail_counter>0, otherwise PASS.
 if [[ "$fail_counter" -gt 0 ]]; then
  echo "==> Tests FAILED [" $fail_counter/$test_counter "]"
@@ -225,5 +223,5 @@ else
  echo "==> Tests PASSED [" $test_counter/$test_counter "]"
  model_tests_passed=0
 fi
-cat $LOG_FILE
+
 exit $model_tests_passed