Merge pull request #4843 from Firefishy/T7101

T7101: Add hardware watchdog support via systemd

6 files changed, 560 insertions, 8 deletions

diff --git a/data/templates/system/watchdog.conf.j2 b/data/templates/system/watchdog.conf.j2
new file mode 100644
index 000000000..d5f7e5ec4
--- /dev/null
+++ b/data/templates/system/watchdog.conf.j2
@@ -0,0 +1,5 @@
+### Autogenerated by system_watchdog.py ###
+[Manager]
+RuntimeWatchdogSec={{ timeout }}
+ShutdownWatchdogSec={{ shutdown_timeout }}
+RebootWatchdogSec={{ reboot_timeout }}
diff --git a/interface-definitions/system_watchdog.xml.in b/interface-definitions/system_watchdog.xml.in
new file mode 100644
index 000000000..c651bc652
--- /dev/null
+++ b/interface-definitions/system_watchdog.xml.in
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<interfaceDefinition>
+  <node name="system">
+    <children>
+      <node name="watchdog" owner="${vyos_conf_scripts_dir}/system_watchdog.py">
+        <properties>
+          <help>Hardware watchdog configuration</help>
+          <priority>9999</priority>
+        </properties>
+        <children>
+          <leafNode name="module">
+            <properties>
+              <help>Kernel module to load for watchdog device (optional)</help>
+              <valueHelp>
+                <format>txt</format>
+                <description>Module name (e.g. 'softdog', 'iTCO_wdt', 'sp5100_tco')</description>
+              </valueHelp>
+              <constraint>
+                <validator name="watchdog-module"/>
+              </constraint>
+              <constraintErrorMessage>Module must be an available watchdog kernel driver module</constraintErrorMessage>
+            </properties>
+          </leafNode>
+          <leafNode name="timeout">
+            <properties>
+              <help>Watchdog timeout for runtime in seconds (1-65535)</help>
+              <valueHelp>
+                <format>u32:1-65535</format>
+                <description>Seconds</description>
+              </valueHelp>
+              <constraint>
+                <validator name="numeric" argument="--range 1-65535"/>
+              </constraint>
+              <constraintErrorMessage>Timeout must be between 1 and 65535 seconds</constraintErrorMessage>
+            </properties>
+            <defaultValue>10</defaultValue>
+          </leafNode>
+          <leafNode name="shutdown-timeout">
+            <properties>
+              <help>Watchdog timeout during shutdown in seconds (60-65535)</help>
+              <valueHelp>
+                <format>u32:60-65535</format>
+                <description>Seconds</description>
+              </valueHelp>
+              <constraint>
+                <validator name="numeric" argument="--range 60-65535"/>
+              </constraint>
+              <constraintErrorMessage>Shutdown timeout must be between 60 and 65535 seconds</constraintErrorMessage>
+            </properties>
+            <defaultValue>120</defaultValue>
+          </leafNode>
+          <leafNode name="reboot-timeout">
+            <properties>
+              <help>Watchdog timeout during reboot in seconds (60-65535)</help>
+              <valueHelp>
+                <format>u32:60-65535</format>
+                <description>Seconds</description>
+              </valueHelp>
+              <constraint>
+                <validator name="numeric" argument="--range 60-65535"/>
+              </constraint>
+              <constraintErrorMessage>Reboot timeout must be between 60 and 65535 seconds</constraintErrorMessage>
+            </properties>
+            <defaultValue>120</defaultValue>
+          </leafNode>
+        </children>
+      </node>
+    </children>
+  </node>
+</interfaceDefinition>
diff --git a/python/vyos/utils/kernel.py b/python/vyos/utils/kernel.py
index 4255d88d5..48a7e88ea 100644
--- a/python/vyos/utils/kernel.py
+++ b/python/vyos/utils/kernel.py
@@ -19,16 +19,48 @@ import os
 # https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/net/wireguard/messages.h?h=linux-6.6.y#n45
 WIREGUARD_REKEY_AFTER_TIME = 120
 
+
+def load_module(name: str, quiet: bool = True, dry_run: bool = False) -> int:
+    """Load a kernel module via modprobe.
+
+    Returns the modprobe return code.
+    """
+
+    from vyos.utils.process import run
+
+    if is_module_loaded(name):
+        return 0
+
+    cmd = ['modprobe']
+    if dry_run:
+        cmd.append('-n')
+    if quiet:
+        cmd.append('-q')
+    cmd.append(name)
+    return run(cmd)
+
+
+def unload_module(name: str) -> int:
+    """Unload a kernel module via rmmod.
+
+    Returns the rmmod return code.
+    """
+
+    from vyos.utils.process import run
+
+    if not is_module_loaded(name):
+        return 0
+
+    return run(['rmmod', name])
+
 def check_kmod(k_mod):
     """ Common utility function to load required kernel modules on demand """
     from vyos import ConfigError
-    from vyos.utils.process import call
     if isinstance(k_mod, str):
         k_mod = k_mod.split()
     for module in k_mod:
-        if not os.path.exists(f'/sys/module/{module}'):
-            if call(f'modprobe {module}') != 0:
-                raise ConfigError(f'Loading Kernel module {module} failed')
+        if load_module(module) != 0:
+            raise ConfigError(f'Loading Kernel module {module} failed')
 
 
 def is_module_loaded(module):
@@ -38,13 +70,11 @@ def is_module_loaded(module):
 def unload_kmod(k_mod):
     """ Common utility function to unload required kernel modules on demand """
     from vyos import ConfigError
-    from vyos.utils.process import call
     if isinstance(k_mod, str):
         k_mod = k_mod.split()
     for module in k_mod:
-        if is_module_loaded(module):
-            if call(f'rmmod {module}') != 0:
-                raise ConfigError(f'Unloading Kernel module {module} failed')
+        if unload_module(module) != 0:
+            raise ConfigError(f'Unloading Kernel module {module} failed')
 
 def list_loaded_modules():
     """ Returns the list of currently loaded kernel modules """
diff --git a/smoketest/scripts/cli/test_system_watchdog.py b/smoketest/scripts/cli/test_system_watchdog.py
new file mode 100644
index 000000000..a263540f7
--- /dev/null
+++ b/smoketest/scripts/cli/test_system_watchdog.py
@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import unittest
+
+from base_vyostest_shim import VyOSUnitTestSHIM
+from vyos.utils.process import cmd
+
+base_path = ['system', 'watchdog']
+
+
+class TestSystemWatchdog(VyOSUnitTestSHIM.TestCase):
+    def tearDown(self):
+        self.cli_delete(base_path)
+        self.cli_commit()
+        super().tearDown()
+
+    def test_enable_watchdog_softdog(self):
+        """Configure watchdog (presence enables) with softdog and check state"""
+        # Presence of 'system watchdog' enables watchdog; set module to softdog
+        self.cli_set(base_path)
+        self.cli_set(base_path + ['module', 'softdog'])
+        self.cli_commit()
+        # Check if softdog module is loaded
+        lsmod = cmd('lsmod')
+        self.assertIn('softdog', lsmod)
+        # Check /dev/watchdog0 exists
+        self.assertTrue(
+            os.path.exists('/dev/watchdog0'), '/dev/watchdog0 does not exist'
+        )
+        # Check systemd config file exists
+        config_path = '/run/systemd/system.conf.d/watchdog.conf'
+        self.assertTrue(
+            os.path.exists(config_path), f"Systemd config file not found: {config_path}"
+        )
+
+    def test_invalid_module_rejected(self):
+        """Verify that a non-existent watchdog module causes commit failure"""
+        # Choose a module name unlikely to exist; include a prefix to avoid collision with real names
+        bogus_module = 'zzzx_watchdog_unit_test_fake'
+        self.cli_set(base_path)
+
+        # Module validation is preferred at set-time. Depending on the test harness,
+        # this may raise on cli_set() or on cli_commit(). Accept either.
+        try:
+            self.cli_set(base_path + ['module', bogus_module])
+        except Exception as e:
+            self.assertRegex(
+                str(e), r"Module must be an available watchdog kernel driver module"
+            )
+            return
+
+        # If set-time validation did not trigger, commit-time validation must.
+        with self.assertRaisesRegex(
+            Exception,
+            r"Watchdog( driver)? module '.*' was not found or cannot be loaded",
+        ):
+            self.cli_commit()
+
+    def test_timeout_upper_limit(self):
+        """Verify watchdog timeout upper bound (65535) is enforced"""
+        self.cli_set(base_path)
+        self.cli_set(base_path + ['module', 'softdog'])
+
+        # 65535 must be accepted
+        self.cli_set(base_path + ['timeout', '65535'])
+        self.cli_commit()
+
+        # 65536 must be rejected (ideally at set-time by XML validator)
+        try:
+            self.cli_set(base_path + ['timeout', '65536'])
+        except Exception as e:
+            # Error message depends on validator/harness formatting
+            self.assertRegex(str(e), r"65535|Timeout must be between")
+            return
+
+        with self.assertRaisesRegex(Exception, r"65535|Timeout must be between"):
+            self.cli_commit()
+
+    def test_shutdown_and_reboot_timeout_written(self):
+        """Verify shutdown-timeout and reboot-timeout are applied to systemd config"""
+        self.cli_set(base_path)
+        self.cli_set(base_path + ['module', 'softdog'])
+
+        # Lowest valid values
+        self.cli_set(base_path + ['shutdown-timeout', '60'])
+        self.cli_set(base_path + ['reboot-timeout', '60'])
+        self.cli_commit()
+
+        config_path = '/run/systemd/system.conf.d/watchdog.conf'
+        with open(config_path, 'r') as f:
+            conf = f.read()
+        self.assertIn('ShutdownWatchdogSec=60', conf)
+        self.assertIn('RebootWatchdogSec=60', conf)
+
+        # Highest valid values
+        self.cli_set(base_path + ['shutdown-timeout', '65535'])
+        self.cli_set(base_path + ['reboot-timeout', '65535'])
+        self.cli_commit()
+
+        with open(config_path, 'r') as f:
+            conf = f.read()
+        self.assertIn('ShutdownWatchdogSec=65535', conf)
+        self.assertIn('RebootWatchdogSec=65535', conf)
+
+    def test_shutdown_and_reboot_timeout_lower_bound(self):
+        """Verify shutdown-timeout/reboot-timeout enforce lower bound (60)"""
+        self.cli_set(base_path)
+        self.cli_set(base_path + ['module', 'softdog'])
+        self.cli_commit()
+
+        for key in ['shutdown-timeout', 'reboot-timeout']:
+            try:
+                self.cli_set(base_path + [key, '59'])
+            except Exception as e:
+                self.assertRegex(
+                    str(e), r"60|timeout must be between|Timeout must be between"
+                )
+                continue
+
+            with self.assertRaisesRegex(
+                Exception, r"60|timeout must be between|Timeout must be between"
+            ):
+                self.cli_commit()
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on())
diff --git a/src/conf_mode/system_watchdog.py b/src/conf_mode/system_watchdog.py
new file mode 100755
index 000000000..059e74dd9
--- /dev/null
+++ b/src/conf_mode/system_watchdog.py
@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+#
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from sys import exit
+from pathlib import Path
+from typing import Optional
+
+from vyos.config import Config
+from vyos.base import Warning
+from vyos.template import render
+from vyos.utils.kernel import load_module
+from vyos.utils.process import call, cmd
+from vyos import ConfigError
+from vyos import airbag
+
+airbag.enable()
+
+watchdog_config_dir = Path('/run/systemd/system.conf.d')
+watchdog_config_file = watchdog_config_dir / 'watchdog.conf'
+modules_load_directory = Path('/run/modules-load.d')
+modules_load_file = modules_load_directory / 'watchdog.conf'
+WATCHDOG_DEV = Path('/dev/watchdog0')
+WATCHDOG_SYSFS = Path('/sys/class/watchdog/watchdog0')
+
+
+def _get_watchdog_driver_module_name() -> Optional[str]:
+    """Return the kernel module name backing watchdog0, if discoverable."""
+
+    module_link = WATCHDOG_SYSFS / 'device/driver/module'
+    if not module_link.exists():
+        return None
+
+    try:
+        resolved = module_link.resolve()
+    except OSError:
+        return None
+
+    # Expected to resolve to /sys/module/<module_name>
+    module_name = resolved.name.strip()
+    return module_name or None
+
+
+def _read_sysfs_int(path: Path) -> Optional[int]:
+    try:
+        return int(path.read_text().strip())
+    except (OSError, ValueError):
+        return None
+
+
+def _get_watchdog_timeout_limits() -> tuple[int, int]:
+    """Return (min_timeout, max_timeout) from sysfs if available.
+
+    If sysfs is unavailable (device not present/loaded yet), fall back to a
+    conservative common kernel max of 65535 seconds.
+    """
+
+    if not WATCHDOG_SYSFS.exists():
+        return 1, 65535
+
+    min_timeout = _read_sysfs_int(WATCHDOG_SYSFS / 'min_timeout')
+    max_timeout = _read_sysfs_int(WATCHDOG_SYSFS / 'max_timeout')
+
+    # Some drivers may not expose min/max. Fall back to sane defaults.
+    if min_timeout is None:
+        min_timeout = 1
+    if max_timeout is None:
+        max_timeout = 65535
+
+    return min_timeout, max_timeout
+
+
+def _verify_watchdog_module(module: str) -> None:
+    # Dry-run modprobe (-n) in quiet mode (-q) verifies availability without loading
+    if load_module(module, quiet=True, dry_run=True) != 0:
+        raise ConfigError(
+            f"Watchdog driver module '{module}' was not found or cannot be loaded"
+        )
+
+    # Ensure the module looks like a watchdog driver and not an arbitrary module.
+    # Use modinfo filename location as the heuristic.
+    filename = cmd(['modinfo', '-F', 'filename', module], raising=ConfigError)
+    filename_l = filename.strip().lower()
+
+    # Accept modules located under drivers/watchdog, plus explicit exception for
+    # ipmi_watchdog which lives in drivers/char/ipmi.
+    is_watchdog_driver = '/watchdog/' in filename_l or filename_l.endswith(
+        '/ipmi_watchdog.ko'
+    )
+
+    if not is_watchdog_driver:
+        raise ConfigError(
+            f"Kernel module '{module}' does not look like a watchdog driver module (modinfo filename: {filename.strip()})"
+        )
+
+
+def get_config(config=None):
+    if config:
+        conf = config
+    else:
+        conf = Config()
+    base = ['system', 'watchdog']
+
+    if not conf.exists(base):
+        return None
+
+    watchdog = conf.get_config_dict(
+        base, key_mangling=('-', '_'), get_first_key=True, with_recursive_defaults=True
+    )
+
+    return watchdog
+
+
+def verify(watchdog):
+    if watchdog is None:
+        return None
+
+    module = watchdog.get('module')
+    device_exists = WATCHDOG_DEV.exists()
+
+    # Require a usable watchdog: either device already present or a module provided
+    if not module and not device_exists:
+        raise ConfigError(
+            "No watchdog device found at /dev/watchdog0 and no module configured. "
+            "Use 'system watchdog module <name>' to load the required watchdog driver for your system."
+        )
+
+    # If a module is provided, ensure it exists and is a watchdog module
+    if module:
+        _verify_watchdog_module(module)
+
+    # Validate runtime watchdog timeout against kernel driver limits if available.
+    # Shutdown/Reboot watchdog settings are systemd-level timers and are not
+    # constrained by the watchdog device driver's min/max.
+    if 'timeout' in watchdog:
+        try:
+            value = int(watchdog['timeout'])
+        except (TypeError, ValueError):
+            raise ConfigError("Invalid value for 'timeout'")
+
+        min_timeout, max_timeout = _get_watchdog_timeout_limits()
+        if value < min_timeout:
+            raise ConfigError(
+                f"'timeout' must be >= {min_timeout} seconds (driver minimum)"
+            )
+        if value > max_timeout:
+            raise ConfigError(
+                f"'timeout' must be <= {max_timeout} seconds (driver maximum)"
+            )
+
+    return None
+
+
+def generate(watchdog):
+    # If watchdog node removed entirely, clean up everything
+    if watchdog is None:
+        watchdog_config_file.unlink(missing_ok=True)
+        modules_load_file.unlink(missing_ok=True)
+        return None
+
+    # Persist kernel module autoload on boot if specified (even if not enabled)
+    module = watchdog.get('module')
+    if module:
+        try:
+            modules_load_directory.mkdir(parents=True, exist_ok=True)
+            modules_load_file.write_text(f"{module}\n")
+        except OSError as e:
+            Warning(f"Failed writing modules-load configuration: {e}")
+    else:
+        # If module option removed, drop persisted autoload file
+        modules_load_file.unlink(missing_ok=True)
+
+    # Try to load kernel module if specified and /dev/watchdog0 is missing
+    if not WATCHDOG_DEV.exists():
+        if module:
+            # Try to load the module using vyos call wrapper for logging/airbag integration
+            try:
+                rc = load_module(module, quiet=True, dry_run=False)
+            except OSError as e:
+                Warning(
+                    f"Could not execute modprobe for watchdog module '{module}': {e}"
+                )
+            else:
+                if rc != 0:
+                    Warning(
+                        f"Could not load watchdog module '{module}' (modprobe exit code {rc})"
+                    )
+    # Re-check for device
+    if not WATCHDOG_DEV.exists():
+        Warning("/dev/watchdog0 not found. Systemd watchdog will not be enabled.")
+        watchdog_config_file.unlink(missing_ok=True)
+        return None
+
+    # If a module was configured explicitly, warn if the actual driver module
+    # bound to watchdog0 differs from what the user configured.
+    if module and WATCHDOG_SYSFS.exists():
+        actual_module = _get_watchdog_driver_module_name()
+        if actual_module and actual_module != module:
+            Warning(
+                f"Configured watchdog driver module '{module}' does not match watchdog0 driver module '{actual_module}'"
+            )
+
+    # Ensure the directory exists
+    watchdog_config_dir.mkdir(parents=True, exist_ok=True)
+
+    # Pass through configured time values directly as seconds
+    render(str(watchdog_config_file), 'system/watchdog.conf.j2', watchdog)
+
+    return None
+
+
+def apply(watchdog):
+    # Reload systemd daemon to apply/unload the watchdog configuration
+    # The watchdog settings take immediate effect after systemd is reloaded
+    call('systemctl daemon-reload')
+
+    return None
+
+
+if __name__ == '__main__':
+    try:
+        c = get_config()
+        verify(c)
+        generate(c)
+        apply(c)
+    except ConfigError as e:
+        print(e)
+        exit(1)
diff --git a/src/validators/watchdog-module b/src/validators/watchdog-module
new file mode 100644
index 000000000..0ae68b46d
--- /dev/null
+++ b/src/validators/watchdog-module
@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+#
+# Copyright VyOS maintainers and contributors <maintainers@vyos.io>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+import sys
+
+from vyos.utils.kernel import load_module
+from vyos.utils.process import rc_cmd
+
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        # No value to validate
+        return 1
+
+    module = sys.argv[1].strip()
+    if not module:
+        return 1
+
+    # Keep the module name format strict.
+    if not re.fullmatch(r"[a-zA-Z0-9_\-]+", module):
+        return 1
+
+    # Ensure the module exists and is loadable (dry-run).
+    # This does not load the module.
+    try:
+        rc = load_module(module, quiet=True, dry_run=True)
+    except OSError:
+        return 1
+
+    if rc != 0:
+        return 1
+
+    # Validate that the module looks like a watchdog driver.
+    # Use modinfo filename location as the heuristic.
+    rc, out = rc_cmd(["modinfo", "-F", "filename", module])
+    if rc != 0:
+        return 1
+    filename = (out or "").strip().lower()
+
+    # Accept modules located under drivers/watchdog, plus explicit exception for
+    # ipmi_watchdog which lives in drivers/char/ipmi.
+    is_watchdog_driver = (
+        ("/watchdog/" in filename)
+        or filename.endswith("/ipmi_watchdog.ko")
+    )
+
+    return 0 if is_watchdog_driver else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())