Base/CI: Boot serenity in CI in a mode that runs tests on target

Build a new version of Serenity in CI that doesn't have all the debug symbols on, or we'd be waiting a very long time to boot. Insert a TestRunner entry into SystemServer.ini that will run a shell script that runs tests in /bin and /usr/Tests and shuts down the system in the new self-test boot mode. Also make sure enough basic services are started in self-test such that the tests will actually run properly.
lanmonster · Feb 28, 2021 · 611bbc4 · 611bbc4
1 parent 5046213
commit 611bbc4
Show file tree

Hide file tree

Showing 4 changed files with 151 additions and 59 deletions.
diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
@@ -9,20 +9,13 @@ env:
  SERENITY_ROOT: ${{ github.workspace }}
 
 jobs:
- build_and_test:
+ build_and_test_serenity:
  runs-on: ${{ matrix.os }}
  strategy:
  fail-fast: false
  matrix:
- include:
- - os: ubuntu-20.04
- allow-test-failure: false
- # There sure is a lot of logic here, is there a better way?
- # TODO: Make IRC notifications its own job/workflow?
- should-notify-irc: ${{ github.repository == 'SerenityOS/serenity' && (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master')) }}
- - os: macos-10.15
- allow-test-failure: true
- should-notify-irc: false
+ debug-macros: ['ALL_DEBUG', 'NORMAL_DEBUG']
+ os: [ubuntu-20.04]
 
  steps:
  - uses: actions/checkout@v2
@@ -37,7 +30,6 @@ jobs:
  - name: Purge interfering packages
  # Remove GCC 9 and clang-format 10 (installed by default)
  run: sudo apt-get purge -y gcc-9 g++-9 libstdc++-9-dev clang-format-10
- if: ${{ runner.os == 'Linux' }}
  - name: "Install Ubuntu dependencies"
  # These packages are already part of the ubuntu-20.04 image:
  # cmake gcc-10 g++-10 shellcheck libgmp-dev
@@ -46,17 +38,9 @@ jobs:
  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
  sudo add-apt-repository "deb http:https://apt.llvm.org/focal/ llvm-toolchain-focal-11 main"
  sudo apt-get update
- sudo apt-get install clang-format-11 libstdc++-10-dev libmpfr-dev libmpc-dev ninja-build npm
- # If we ever do any qemu-emulation on Github Actions, we should re-enable this:
- # e2fsprogs qemu-system-i386 qemu-utils
- if: ${{ runner.os == 'Linux' }}
+ sudo apt-get install clang-format-11 libstdc++-10-dev libmpfr-dev libmpc-dev ninja-build npm e2fsprogs qemu-utils qemu-system-i386
  - name: Use GCC 10 instead
  run: sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 60 --slave /usr/bin/g++ g++ /usr/bin/g++-10
- if: ${{ runner.os == 'Linux' }}
-
- - name: Install macOS dependencies
- run: brew install coreutils ninja
- if: ${{ runner.os == 'macOS' }}
 
  - name: Install JS dependencies
  run: sudo npm install -g prettier
@@ -72,7 +56,6 @@ jobs:
 
  - name: Lint (Phase 1/2)
  run: ${{ github.workspace }}/Meta/lint-ci.sh
- if: ${{ runner.os == 'Linux' }}
  - name: Toolchain cache
  uses: actions/cache@v2
  with:
@@ -86,62 +69,79 @@ jobs:
  # TODO: ccache
  # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/
  # https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml
+ - name: Create build environment with debug macros
+ working-directory: ${{ github.workspace }}
+ # Build the entire project with debug macros turned on, to prevent code rot.
+ # However, it is unweildy and slow to run tests with them enabled, so we will build twice.
+ run: |
+ mkdir -p Build
+ cd Build
+ cmake .. -GNinja -DBUILD_LAGOM=ON -DENABLE_ALL_THE_DEBUG_MACROS=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10
+ if: ${{ matrix.debug-macros == 'ALL_DEBUG' }}
  - name: Create build environment
  working-directory: ${{ github.workspace }}
  # Note that this needs to run *even if* the Toolchain was built,
  # in order to set options like BUILD_LAGOM.
  run: |
  mkdir -p Build
  cd Build
- cmake .. -GNinja -DBUILD_LAGOM=ON -DENABLE_ALL_THE_DEBUG_MACROS=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10
+ cmake .. -GNinja -DBUILD_LAGOM=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10
+ if: ${{ matrix.debug-macros == 'NORMAL_DEBUG' }}
 
- # === ACTUALLY BUILD AND TEST ===
+ # === ACTUALLY BUILD ===
 
  - name: Build Serenity and Tests
  working-directory: ${{ github.workspace }}/Build
  run: cmake --build .
  - name: Lint (Phase 2/2)
  working-directory: ${{ github.workspace }}/Meta
  run: ./check-symbols.sh
- - name: Run CMake tests
- working-directory: ${{ github.workspace }}/Build
- run: CTEST_OUTPUT_ON_FAILURE=1 ninja test || ${{ matrix.allow-test-failure }}
- timeout-minutes: 2
- - name: Run JS tests
- working-directory: ${{ github.workspace }}/Build/Meta/Lagom
- run: DISABLE_DBG_OUTPUT=1 ./test-js || ${{ matrix.allow-test-failure }}
- - name: Run LibCompress tests
- working-directory: ${{ github.workspace }}/Build/Meta/Lagom
- run: ./test-compress
 
- # === NOTIFICATIONS ===
+ - name: Create Serenity Rootfs
+ if: ${{ matrix.debug-macros == 'NORMAL_DEBUG'}}
+ working-directory: ${{ github.workspace }}/Build
+ run: ninja install && ninja image
 
- - name: Dump event info
- if: always()
- # Usually unnecessary, but insanely useful if IRC notifications fail.
- run: |
- cat <<"EOF"
- ${{ toJSON(github.event) }}
- EOF
- - name: Generate IRC message
- if: matrix.should-notify-irc == true && !cancelled()
- run: |
- ${{ github.workspace }}/Meta/notify_irc.py <<"EOF"
- ["${{ github.actor }}", ${{ github.run_id }}, "${{ job.status }}",
- ${{ toJSON(github.event) }}
- ]
- EOF
+ - name: Run On-Target Tests
+ if: ${{ matrix.debug-macros == 'NORMAL_DEBUG'}}
+ working-directory: ${{ github.workspace }}/Build
+ env:
+ SERENITY_KERNEL_CMDLINE: "boot_mode=self-test"
+ SERENITY_RUN: "ci"
+ run: ninja run
+ timeout-minutes: 10
+ - name: Print target logs
+ if: ${{ !cancelled() && matrix.debug-macros == 'NORMAL_DEBUG'}}
+ working-directory: ${{ github.workspace }}/Build
+ run: cat ./debug.log
 
- build_lagom_with_fuzzers:
- runs-on: ubuntu-20.04
+ build_and_test_lagom:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - with-fuzzers: FUZZ
+ os: ubuntu-20.04
+ allow-test-failure: false
+ - with-fuzzers: NO_FUZZ
+ os: ubuntu-20.04
+ allow-test-failure: false
+ - with-fuzzers: NO_FUZZ
+ os: macos-10.15
+ allow-test-failure: true
 
  steps:
  - uses: actions/checkout@v2
 
  # === OS SETUP ===
  #
- - name: Install dependencies
+ - name: Install Ubuntu dependencies
  run: sudo apt-get install ninja-build
+ if: ${{ runner.os == 'Linux' }}
+ - name: Install macOS dependencies
+ run: brew install ninja
+ if: ${{ runner.os == 'macOS' }}
  - name: Check versions
  run: set +e; clang --version; clang++ --version; ninja --version
 
@@ -150,15 +150,68 @@ jobs:
  # TODO: ccache
  # https://cristianadam.eu/20200113/speeding-up-c-plus-plus-github-actions-using-ccache/
  # https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml
- - name: Create build environment
+ - name: Create build environment (fuzz)
  working-directory: ${{ github.workspace }}/Meta/Lagom
  run: |
  mkdir -p Build
  cd Build
  cmake -GNinja -DBUILD_LAGOM=ON -DENABLE_FUZZER_SANITIZER=ON -DENABLE_ADDRESS_SANITIZER=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ ..
+ if: ${{ matrix.with-fuzzers == 'FUZZ' }}
 
- # === ACTUALLY BUILD ===
+ - name: Create build environment (no fuzz)
+ working-directory: ${{ github.workspace }}/Meta/Lagom
+ run: |
+ mkdir -p Build
+ cd Build
+ cmake -GNinja -DBUILD_LAGOM=ON -DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10 ..
+ if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }}
+
+ # === ACTUALLY BUILD AND TEST ===
 
- - name: Build Lagom with Fuzzers
+ - name: Build Lagom
  working-directory: ${{ github.workspace }}/Meta/Lagom/Build
  run: cmake --build .
+
+ - name: Run CMake tests
+ working-directory: ${{ github.workspace }}/Meta/Lagom/Build
+ run: CTEST_OUTPUT_ON_FAILURE=1 ninja test || ${{ matrix.allow-test-failure }}
+ timeout-minutes: 2
+ if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }}
+ - name: Run JS tests
+ working-directory: ${{ github.workspace }}/Meta/Lagom/Build
+ if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }}
+ run: DISABLE_DBG_OUTPUT=1 ./test-js || ${{ matrix.allow-test-failure }}
+ - name: Run LibCompress tests
+ working-directory: ${{ github.workspace }}/Meta/Lagom/Build
+ run: ./test-compress
+ if: ${{ matrix.with-fuzzers == 'NO_FUZZ' }}
+
+ notify_irc:
+ needs: [build_and_test_serenity, build_and_test_lagom]
+ runs-on: ubuntu-20.04
+ if: always() && github.repository == 'SerenityOS/serenity' && (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/master'))
+
+ steps:
+ - uses: actions/checkout@v2
+ # Sets environment variable env.WORKFLOW_CONCLUSION to one of [neutral, success, skipped, cancelled, timed_out, action_required, failure]
+ # depending on result of all needs jobs
+ - uses: technote-space/workflow-conclusion-action@v2
+
+ # === NOTIFICATIONS ===
+
+ - name: Dump event info
+ if: always()
+ # Usually unnecessary, but insanely useful if IRC notifications fail.
+ run: |
+ cat <<"EOF"
+ ${{ toJSON(github.event) }}
+ ${{ toJSON(needs) }}
+ EOF
+ - name: Generate IRC message
+ if: always()
+ run: |
+ ${{ github.workspace }}/Meta/notify_irc.py <<"EOF"
+ ["${{ github.actor }}", ${{ github.run_id }}, "${{ env.WORKFLOW_CONCLUSION }}",
+ ${{ toJSON(github.event) }}
+ ]
+ EOF
diff --git a/Base/etc/SystemServer.ini b/Base/etc/SystemServer.ini
@@ -4,7 +4,7 @@ SocketPermissions=660
 Lazy=1
 Priority=low
 User=protocol
-BootModes=text,graphical
+BootModes=text,graphical,self-test
 MultiInstance=1
 AcceptSocketConnections=1
 
@@ -39,13 +39,13 @@ Lazy=1
 Priority=low
 KeepAlive=1
 User=lookup
-BootModes=text,graphical
+BootModes=text,graphical,self-test
 
 [DHCPClient]
 Priority=low
 KeepAlive=1
 User=root
-BootModes=text,graphical
+BootModes=text,graphical,self-test
 
 [NotificationServer]
 Socket=/tmp/portal/notify
@@ -175,3 +175,11 @@ AcceptSocketConnections=1
 Executable=/bin/CrashDaemon
 KeepAlive=1
 User=anon
+
+[TestRunner@ttyS0]
+Executable=/home/anon/tests/run-tests-and-shutdown.sh
+StdIO=/dev/ttyS0
+Environment=DO_SHUTDOWN_AFTER_TESTS=1 TERM=xterm PATH=/bin:/usr/bin:/usr/local/bin
+User=anon
+WorkingDirectory=/home/anon
+BootModes=self-test
diff --git a/Base/home/anon/tests/run-tests-and-shutdown.sh b/Base/home/anon/tests/run-tests-and-shutdown.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+echo "==== Running Tests on SerenityOS ===="
+
+run() {
+ test_cmd=($*)
+ echo "Running test -- $test_cmd"
+ if $test_cmd {
+ echo "::debug file=$test_cmd:: $test_cmd passed!"
+ } else {
+ echo "::error file=$test_cmd:: $test_cmd returned non-zero exit code, check logs!"
+ }
+}
+
+# TODO: It'd be nice to have a list+list op (as opposed to nest-on-in-another)
+# TODO: It'd be nice to have a list.length or enumerate(list) operation to allow terminal progress bar
+# TODO: test-web requires the window server
+system_tests=(test-js test-pthread test-compress (test-crypto bigint -t))
+# FIXME: Running too much at once is likely to run into #5541. Remove commented out find below when stable
+all_tests=($system_tests) #$(find /usr/Tests -type f | grep -v Kernel | grep -v .inc | shuf))
+
+for list in $all_tests {
+ for $list { run $it }
+}
+
+echo "==== Done running tests ===="
+
+if test $DO_SHUTDOWN_AFTER_TESTS {
+ shutdown -n
+}
diff --git a/Meta/lint-shell-scripts.sh b/Meta/lint-shell-scripts.sh
@@ -11,7 +11,8 @@ if [ "$#" -eq "0" ]; then
  '*.sh' \
  ':!:Toolchain' \
  ':!:Ports' \
- ':!:Userland/Shell/Tests'
+ ':!:Userland/Shell/Tests' \
+ ':!:Base/home/anon/tests'
  )
 else
  files=()