tests: update snapd-testing-tools and fix nested workflow (canonical#…

…13287)

* Squashed 'tests/lib/external/snapd-testing-tools/' changes from 1c8efb77e1..510d95f429

510d95f429 add extra check for error in auto-refresh detection function
3289d4031b Try open the log with latin-1 encoding when utf-8 is not working
9db785499f improved how the tools are waiting for system reboot
2a5c4414a3 fix shellcheck errors
5e7b63883d Fixes for osquery and tests pkgs (canonical#43)
4c9145e2ac support reboot waiting for auto-refresh
45768f5188 show changes in unknown status after refresh
8013c30c2a Remove support for ubuntu 22.10
b32b80bf54 Fix remote.rait-for test in bionic
5675c625e9 Enable fedora 38
55f4471957 Support for new oss
f2e88b357c New tool used to query spread json reports
cacd35ede0 utils/spread-shellcheck: explain disabled warnings (canonical#42)
c82afb2dee Support --no-install-recommends parameter when installing dependencies with tests.pkgs
b84eea92e2 spread-shellcheck: fix quotes in environment variables (canonical#41)
ab1e51c29f New comparison in os-query for core systems (canonical#40)
e5ae22a5d4 systemd units can be overwritten
63540b845a Fix error messages in remote pull and push
75e8a426a5 make sure the unit is removed in tests.systemd test
9089ff5c02 Update tests to use the new tests.systemd stop-unit
44ecd5e56a Move tests.systemd stop-units to stop-unit
01a2a83b4b Update tests.systemd to have stop units as systemd.sh
162e93bd35 update tests.systemd CLI options to be the same than retry command
14aa43a405 new feature to re-run failed spread tests (canonical#39)
604cb782db Fix shellcheck in systemd tool
bfc71082c8 Update the tests.systemd to allow parameters waiting for service status
8a2d0a99df Adding quiet tool and removing set +-x from tests.pkgs
d90935d2a4 A comment explaining about the default values for wait-for
3232c5dba7 Add support for ubuntu 23.04
a7164fba07 remove fedora 35 support, add fedora 37 support
89b9eb5301 Update systems supported
92bb6a0664 Include snap-sufix in the snaps.name tool

git-subtree-dir: tests/lib/external/snapd-testing-tools
git-subtree-split: 510d95f4292c9edbb292bcb4560c4fee98a3d01c

.github/workflows/test.yaml

@@ -514,6 +514,9 @@ jobs:
 
  echo "Running spread log analyzer"
  ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "$RUN_TESTS" spread-results.json > "$FAILED_TESTS_FILE"
+
+ echo "List of failed tests saved"
+ cat "$FAILED_TESTS_FILE"
  else
  echo "No spread log found, saving empty list of failed tests"
  touch "$FAILED_TESTS_FILE"
@@ -555,13 +558,17 @@ jobs:
  - name: Checkout code
  uses: actions/checkout@v3
 
+ - name: Get previous attempt
+ id: get-previous-attempt
+ run: |
+ echo "previous_attempt=$(( ${{ github.run_attempt }} - 1 ))" >> $GITHUB_OUTPUT
+ shell: bash
+
  - name: Get previous cache
- uses: actions/cache/restore@v3
+ uses: actions/cache@v3
  with:
  path: "${{ github.workspace }}/.test-results"
- key: "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.system }}-${{ github.run_attempt }}"
- restore-keys: |
- "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.system }}-"
+ key: "${{ github.job }}-results-${{ github.run_id }}-${{ matrix.system }}-${{ steps.get-previous-attempt.outputs.previous_attempt }}"
 
  - name: Prepare test results env and vars
  id: prepare-test-results-env
@@ -661,6 +668,9 @@ jobs:
 
  echo "Running spread log analyzer"
  ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "$RUN_TESTS" spread-results.json > "$FAILED_TESTS_FILE"
+
+ echo "List of failed tests saved"
+ cat "$FAILED_TESTS_FILE"
  else
  echo "No spread log found, saving empty list of failed tests"
  touch "$FAILED_TESTS_FILE"

tests/lib/external/snapd-testing-tools/remote/remote.wait-for

@@ -153,32 +153,65 @@ wait_for_device_initialized() {
 }
 
 wait_for_refresh_reboot() {
+ echo "remote.wait-for: waiting for refresh reboot"
+
+ local change_id=${1:-}
+ local boot_id=${2:-}
+ if [ -z "$change_id" ] || [ -z "$boot_id" ]; then
+ echo "remote.wait-for: either change_id or boot_id not provided"
+ return 1
+ fi
+
  wait_for_ssh "$DEFAULT_WAIT_FOR_SSH_ATTEMPTS" "$DEFAULT_WAIT_FOR_SSH_WAIT"
- for _ in $(seq 3); do
- if remote.exec "sudo journalctl -u snapd -n 30" | grep -q "Waiting for system reboot"; then
- echo "remote.wait-for: waiting for system reboot"
- remote.exec --timeout 3 "sudo reboot" || true
- wait_for_no_ssh "$DEFAULT_WAIT_FOR_NO_SSH_ATTEMPTS" "$DEFAULT_WAIT_FOR_NO_SSH_WAIT"
- break
+ for _ in $(seq 5); do
+ # The refresh is being executed
+ if remote.exec "snap changes" | grep -Eq "$change_id.*Doing.*(Auto-refresh|Refresh)"; then
+ # The systems is waiting for reboot
+ if remote.exec "sudo journalctl -u snapd -n 30" | grep -q "Waiting for system reboot"; then
+ echo "remote.wait-for: waiting for system reboot"
+ remote.exec "sudo reboot" || true
+ wait_for_no_ssh "$DEFAULT_WAIT_FOR_NO_SSH_ATTEMPTS" "$DEFAULT_WAIT_FOR_NO_SSH_WAIT"
+ break
+ fi
+ echo "remote.wait-for: either auto-refresh or refresh in progress"
+ fi
+
+ # when the refresh has finished and no reboot needed, the function returns
+ if remote.exec "snap changes" | grep -Eq "$change_id.*Done.*(Auto-refresh|Refresh)"; then
+ echo "remote.wait-for: refresh completed, reboot not required"
+ return
+ fi
+
+ if remote.exec "snap changes" | grep -Eq "$change_id.*Error.*(Auto-refresh|Refresh)"; then
+ echo "remote.wait-for: refresh finished with error"
+ return 1
  fi
+
+ echo "remote.wait-for: system reboot not detected"
  sleep 1
  done
  wait_for_ssh "$DEFAULT_WAIT_FOR_SSH_ATTEMPTS" "$DEFAULT_WAIT_FOR_SSH_WAIT"
+ if [ "$(remote.exec "cat /proc/sys/kernel/random/boot_id")" == "$boot_id" ]; then
+ echo "remote.wait-for: boot id did not change"
+ return 1
+ else
+ echo "remote.wait-for: boot id changed, refresh completed with reboot"
+ fi
 }
 
 wait_for_refresh(){
  echo "remote.wait-for: waiting for either auto-refresh or refresh"
 
  change_line="$(remote.exec 'snap changes' | grep -E 'Doing.*(Auto-refresh|Refresh)' || true)"
  if [ -n "$change_line" ]; then
- echo "remote.wait-for: Refresh in progress"
+ echo "remote.wait-for: refresh in progress"
  change_id="$(echo "$change_line" | awk '{ print $1 }')"
+ boot_id="$(remote.exec "cat /proc/sys/kernel/random/boot_id")"
 
-
- while remote.exec "snap changes" | grep -Eq "$change_id.*Doing.*(Auto-refresh|Refresh)"; do
- echo -n '.'
- # As the system could reboot anytime, this command could fail
- wait_for_refresh_reboot || true
+ for _ in $(seq 20); do
+ if wait_for_refresh_reboot "$change_id" "$boot_id"; then
+ break
+ fi
  done
  echo ""
 

tests/lib/external/snapd-testing-tools/utils/log-parser

@@ -278,8 +278,12 @@ class LogReader:
  return os.path.exists(self.filepath)
 
  def read_spread_log(self):
- with open(self.filepath) as filepath:
- self.lines = filepath.readlines()
+ try:
+ with open(self.filepath, 'r', encoding='utf-8') as filepath:
+ self.lines = filepath.readlines()
+ except UnicodeDecodeError:
+ with open(self.filepath, 'r', encoding='latin-1') as filepath:
+ self.lines = filepath.readlines()
 
  # Find the start of the log, the log file could include
  # initial lines which are not part of the spread log itself