From 8ee09f05d15276b295ea199f08fb3bc2e2d16121 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Sat, 2 May 2026 15:06:44 +0300 Subject: ci: fail workflows on kernel issues found in dmesg Recently i wasted several hours searching bug in workflows/userspace. Turned out in dmesg we had highly visible WARNING that we dont watch. After each test run, dump dmesg and grep for canonical kernel-issue markers (WARNING, BUG, Oops, kernel panic, GPF, KASAN, kernel UBSAN, soft/hard lockup, hung tasks, bad page state, invalid opcode). Fail the job if any are present, with a GitHub Actions error annotation. Until now, kernel WARNs from out-of-tree drivers (e.g. vlan_mon tripping the new ETH_P_ALL ptype_head WARN_ON in 6.6+) were silently swallowed by the existing 'Display processes and dmesg after tests' steps -- visible only if a human inspected the log. The check runs with if: always() so it triggers both on test failure and on test completion. Applied to all four workflows that load kernel modules: run-tests-asan-ubsan.yml, run-tests.yml (Test-in-Qemu, Test-in-Alpine, Test-in-GH, Test-in-GH-Coverage), run-tests-32bit.yml, and run-tests-bigendian.yml. P.S. Some whitespace churn included. Signed-off-by: Denys Fedoryshchenko --- .github/workflows/run-tests-32bit.yml | 11 +++++++- .github/workflows/run-tests-asan-ubsan.yml | 23 +++++++++++----- .github/workflows/run-tests-bigendian.yml | 11 +++++++- .github/workflows/run-tests.yml | 44 +++++++++++++++++++++++++++--- 4 files changed, 76 insertions(+), 13 deletions(-) (limited to '.github') diff --git a/.github/workflows/run-tests-32bit.yml b/.github/workflows/run-tests-32bit.yml index 94d158ab..637a6951 100644 --- a/.github/workflows/run-tests-32bit.yml +++ b/.github/workflows/run-tests-32bit.yml @@ -155,8 +155,17 @@ jobs: - name: Run tests (all) timeout-minutes: 5 run: > - ssh -i ssh-key -p2222 root@localhost "cd accel-ppp/tests && + ssh -i ssh-key -p2222 root@localhost "cd accel-ppp/tests && python3 -m pytest -Wall --order-dependencies -v -m \"not chap_secrets\"" - name: Display processes and dmesg after tests if: ${{ always() }} run: ssh -i ssh-key -p2222 -o ConnectTimeout=5 root@localhost "ps aux | grep accel- && dmesg" + + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + ssh -i ssh-key -p2222 -o ConnectTimeout=5 root@localhost "dmesg" > /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi diff --git a/.github/workflows/run-tests-asan-ubsan.yml b/.github/workflows/run-tests-asan-ubsan.yml index 81fa2fb0..df990657 100644 --- a/.github/workflows/run-tests-asan-ubsan.yml +++ b/.github/workflows/run-tests-asan-ubsan.yml @@ -26,16 +26,16 @@ jobs: steps: - name: Install build tools (using apt) run: > - sudo apt update && + sudo apt update && NEEDRESTART_SUSPEND=1 DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true sudo -E apt -y install git build-essential cmake gcc linux-headers-`uname -r` - libpcre2-dev libssl-dev liblua5.1-0-dev kmod python3-pip + libpcre2-dev libssl-dev liblua5.1-0-dev kmod python3-pip iproute2 ppp pppoe isc-dhcp-client - name: Install testing tools (using pip) run: > - sudo apt -y install python3-pytest python3-pytest-dependency python3-pytest-order || - sudo pip3 install pytest pytest-dependency pytest-order || + sudo apt -y install python3-pytest python3-pytest-dependency python3-pytest-order || + sudo pip3 install pytest pytest-dependency pytest-order || sudo pip3 install --break-system-packages pytest pytest-dependency pytest-order - name: Check out repository code @@ -51,8 +51,8 @@ jobs: run: > CFLAGS="-fsanitize=${{ matrix.sanitizer }} -fno-sanitize-recover=all -fno-omit-frame-pointer -O2 -g" LDFLAGS="-fsanitize=${{ matrix.sanitizer }}" - cmake -DBUILD_IPOE_DRIVER=TRUE -DBUILD_VLAN_MON_DRIVER=TRUE -DCMAKE_INSTALL_PREFIX=/usr - -DKDIR=/usr/src/linux-headers-`uname -r` + cmake -DBUILD_IPOE_DRIVER=TRUE -DBUILD_VLAN_MON_DRIVER=TRUE -DCMAKE_INSTALL_PREFIX=/usr + -DKDIR=/usr/src/linux-headers-`uname -r` -DLUA=TRUE -DSHAPER=TRUE -DRADIUS=TRUE .. - name: make && make install @@ -72,4 +72,13 @@ jobs: working-directory: ./tests env: ${{ matrix.env_name }}: ${{ matrix.env_value }} - run: sudo -E python3 -m pytest -Wall --order-dependencies -v \ No newline at end of file + run: sudo -E python3 -m pytest -Wall --order-dependencies -v + + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + sudo dmesg | tee /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi diff --git a/.github/workflows/run-tests-bigendian.yml b/.github/workflows/run-tests-bigendian.yml index d5dce83b..64b49a11 100644 --- a/.github/workflows/run-tests-bigendian.yml +++ b/.github/workflows/run-tests-bigendian.yml @@ -162,8 +162,17 @@ jobs: - name: Run tests (all) timeout-minutes: 5 run: > - ssh -i ssh-key -p2222 root@localhost "cd accel-ppp/tests && + ssh -i ssh-key -p2222 root@localhost "cd accel-ppp/tests && python3 -m pytest -Wall --order-dependencies -v -m \"not chap_secrets\"" - name: Display processes and dmesg after tests if: ${{ always() }} run: ssh -i ssh-key -p2222 -o ConnectTimeout=5 root@localhost "ps aux | grep accel- && dmesg" + + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + ssh -i ssh-key -p2222 -o ConnectTimeout=5 root@localhost "dmesg" > /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index af7b5727..6db3574c 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -170,14 +170,23 @@ jobs: - name: Run tests (all) timeout-minutes: 5 run: > - ssh -i ssh-key -p2222 user@localhost "cd accel-ppp/tests && + ssh -i ssh-key -p2222 user@localhost "cd accel-ppp/tests && sudo python3 -m pytest -Wall --order-dependencies -v" - name: Display processes and dmesg after tests if: ${{ always() }} run: > - ssh -i ssh-key -p2222 user@localhost "ps aux | grep accel- && + ssh -i ssh-key -p2222 user@localhost "ps aux | grep accel- && sudo dmesg" + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + ssh -i ssh-key -p2222 user@localhost "sudo dmesg" > /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi + Test-in-Alpine: #if: ${{ false }} # disable for now runs-on: ubuntu-24.04 @@ -299,14 +308,23 @@ jobs: - name: Run tests (all) timeout-minutes: 5 run: > - ssh -i ssh-key -p2222 alpine@localhost "cd accel-ppp/tests && + ssh -i ssh-key -p2222 alpine@localhost "cd accel-ppp/tests && doas python3 -m pytest -Wall --order-dependencies -v -m \"not chap_secrets\"" - name: Display processes and dmesg after tests if: ${{ always() }} run: > - ssh -i ssh-key -p2222 alpine@localhost "ps aux | grep accel- && + ssh -i ssh-key -p2222 alpine@localhost "ps aux | grep accel- && doas dmesg" + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + ssh -i ssh-key -p2222 alpine@localhost "doas dmesg" > /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi + Test-in-GH: #if: ${{ false }} # disable for now @@ -363,6 +381,15 @@ jobs: working-directory: ./tests run: sudo python3 -m pytest -Wall --order-dependencies -v + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + sudo dmesg | tee /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi + Test-in-GH-Coverage: #if: ${{ false }} # disable for now strategy: @@ -419,6 +446,15 @@ jobs: working-directory: ./tests run: sudo python3 -m pytest -Wall --order-dependencies -v || exit 0 + - name: Check dmesg for kernel issues + if: ${{ always() }} + run: | + sudo dmesg | tee /tmp/dmesg.log + if grep -E 'WARNING:|BUG:|Oops:|kernel panic|general protection fault|KASAN|UBSAN:|soft lockup|hard LOCKUP|INFO: task .+ blocked|Bad page state|invalid opcode' /tmp/dmesg.log; then + echo "::error::Kernel issues detected in dmesg" + exit 1 + fi + - name: Generate coverage reports (default(txt), csv, html) run: | mkdir -p tests/report -- cgit v1.2.3