From 2054c3c53f8bc83733c8ef81c2cc57f4e6b27f5a Mon Sep 17 00:00:00 2001 From: Oleksandr Kuchmystyi Date: Thu, 9 Jul 2026 11:16:09 +0300 Subject: kdump: T8868: Implement kernel crash dump support Add new CLI subtree 'system option kdump' with memory reservation and local dump path configuration. Also add 'show system kdump' and 'show system kdump dumps' operation mode commands to get status of the configuration and list of dumps. Extend 'show tech-support report' and 'generate tech-support archive' with kdump service status, active config files and the recently kernel dump file. --- data/templates/system/kdump_tools.conf.j2 | 7 + debian/control | 4 + debian/vyos-1x.postinst | 1 + interface-definitions/system_option.xml.in | 38 +++ op-mode-definitions/show-system.xml.in | 14 ++ python/vyos/defaults.py | 8 + python/vyos/utils/boot.py | 1 + python/vyos/utils/kernel.py | 70 ++++++ python/vyos/utils/memory.py | 28 +++ smoketest/scripts/cli/test_system_option.py | 153 +++++++++++- smoketest/scripts/system/test_kernel_options.py | 12 + src/conf_mode/system_option.py | 99 +++++++- .../system/kdump-tools.service.d/override.conf | 19 ++ src/op_mode/generate_tech-support_archive.py | 28 ++- src/op_mode/image_installer.py | 18 +- src/op_mode/kdump.py | 256 +++++++++++++++++++++ src/op_mode/show_techsupport_report.py | 5 + 17 files changed, 754 insertions(+), 7 deletions(-) create mode 100644 data/templates/system/kdump_tools.conf.j2 create mode 100644 python/vyos/utils/memory.py create mode 100644 src/etc/systemd/system/kdump-tools.service.d/override.conf create mode 100644 src/op_mode/kdump.py diff --git a/data/templates/system/kdump_tools.conf.j2 b/data/templates/system/kdump_tools.conf.j2 new file mode 100644 index 000000000..5dce145fd --- /dev/null +++ b/data/templates/system/kdump_tools.conf.j2 @@ -0,0 +1,7 @@ +### Autogenerated by system_option.py ### + +# Enable/disable kdump-tools service - default off +USE_KDUMP={{ '0' if disabled else '1' }} + +# Local directory where crash dumps are stored +KDUMP_COREDIR="{{ dump_path }}" diff --git a/debian/control b/debian/control index 1e43fe63e..d6c62f215 100644 --- a/debian/control +++ b/debian/control @@ -109,7 +109,11 @@ Depends: iproute2 (>= 6.0.0), linux-cpupower, hwloc, +# For "system option kdump" kexec-tools, + kdump-tools, + makedumpfile, +# End "system option kdump" # ipaddrcheck is widely used in IP value validators ipaddrcheck, ethtool (>= 6.10), diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst index a263b6b37..9e9d7e99d 100644 --- a/debian/vyos-1x.postinst +++ b/debian/vyos-1x.postinst @@ -134,6 +134,7 @@ setcap cap_sys_time=pe /bin/date # create needed directories mkdir -p /var/log/user mkdir -p /var/core +mkdir -p /var/crash mkdir -p /opt/vyatta/etc/config/auth mkdir -p /opt/vyatta/etc/config/scripts mkdir -p /opt/vyatta/etc/config/user-data diff --git a/interface-definitions/system_option.xml.in b/interface-definitions/system_option.xml.in index a87bee1a9..beeca74c6 100644 --- a/interface-definitions/system_option.xml.in +++ b/interface-definitions/system_option.xml.in @@ -413,6 +413,44 @@ #include + + + Kernel crash dump (kdump) settings + + + + + Amount of memory to reserve for the crash kernel + + auto + Automatically determine memory reservation + + + u32:128-1048576 + Memory size in megabytes + + + (auto) + + + + auto + + + + Local directory to store kernel crash dumps + + txt + Absolute path to crash dump directory + + + + + + /var/crash + + + plays sound via system speaker when you can login diff --git a/op-mode-definitions/show-system.xml.in b/op-mode-definitions/show-system.xml.in index c7b57893f..d11890363 100644 --- a/op-mode-definitions/show-system.xml.in +++ b/op-mode-definitions/show-system.xml.in @@ -97,6 +97,20 @@ dmesg + + + Show kernel crash dump (kdump) status and configuration + + ${vyos_op_scripts_dir}/kdump.py show_status + + + + Show recorded kernel crash dumps + + ${vyos_op_scripts_dir}/kdump.py show_dumps + + + Show user accounts diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 5829b859f..854f95ff0 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -49,6 +49,7 @@ systemd_services = { 'openconnect': 'ocserv.service', 'syslog' : 'syslog.service', 'snmpd' : 'snmpd.service', + 'kdump': 'kdump-tools.service', } internal_ports = { @@ -115,3 +116,10 @@ config_sync_exclusion_list = os.path.join( # Sits between the l3mdev rule (1000) and the l3mdev unreachable rule (2000), # ensuring fwmark-tagged tunnel packets are routed into the correct VRF table. wireguard_fwmark_pref = '1998' + +# Tiered auto-sizing based on total system RAM, determined by +# empirical testing across representative VyOS deployments: +# 1G - 8G RAM -> reserve 512M +# 8G - 64G RAM -> reserve 768M +# 64G+ RAM -> reserve 1G +KDUMP_DEFAULT_MEMORY_AUTO = '1G-8G:512M,8G-64G:768M,64G-:1G' diff --git a/python/vyos/utils/boot.py b/python/vyos/utils/boot.py index f804cd94e..dc6fcd408 100644 --- a/python/vyos/utils/boot.py +++ b/python/vyos/utils/boot.py @@ -15,6 +15,7 @@ import os + def boot_configuration_complete() -> bool: """ Check if the boot config loader has completed """ diff --git a/python/vyos/utils/kernel.py b/python/vyos/utils/kernel.py index 726025605..69574cefc 100644 --- a/python/vyos/utils/kernel.py +++ b/python/vyos/utils/kernel.py @@ -17,10 +17,18 @@ import os from typing import Tuple from typing import Optional +from vyos.utils.file import read_file + # A list of used Kernel constants # https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/drivers/net/wireguard/messages.h?h=linux-6.6.y#n45 WIREGUARD_REKEY_AFTER_TIME = 120 +CMDLINE_PATH = '/proc/cmdline' + +# Kernel interface files exposing crash kernel state at runtime +KEXEC_CRASH_LOADED = '/sys/kernel/kexec_crash_loaded' +KEXEC_CRASH_SIZE = '/sys/kernel/kexec_crash_size' + def load_module(name: str, quiet: bool = True, dry_run: bool = False) -> int: """Load a kernel module via modprobe. @@ -172,3 +180,65 @@ def get_kernel_serial_console() -> Tuple[Optional[str], Optional[str], Optional[ ) return (None, None, None) + + +def get_kernel_boot_arg(argument=None) -> str | None: + """Read and parse kernel boot arguments from the kernel command line. + + Args: + argument: The name of a specific boot argument to look up (e.g. + 'crashkernel'). If omitted or None, the full raw command + line string is returned. + + Returns: + If argument is None: the full kernel command line string. + If argument is given: the value of that argument (the part after '='), + or None if the argument is not present on the command line. + + Examples: + >>> get_kernel_boot_arg() + 'ro quiet crashkernel=256M console=tty0' + + >>> get_kernel_boot_arg('crashkernel') + '256M' + + >>> get_kernel_boot_arg('quiet') + '' + + >>> get_kernel_boot_arg('nonexistent') + None + """ + cmdline = read_file(CMDLINE_PATH) + if not argument: + return cmdline + + # Kernel arguments with values are formatted as 'key=value' tokens + # separated by spaces. Build the prefix to match against: + key = f'{argument}=' + + # Kernel parses the command line from left to right. + # A later argument will overwrite an earlier one. + for part in reversed(cmdline.split()): + if part.startswith(key): + # Strip the 'key=' prefix and return only the value portion + return part[len(key) :] + elif part == argument: + # Statement without value portion should be return empty string + return '' + + return None + + +def is_crash_kernel_loaded() -> bool: + """Return True when a capture kernel is currently loaded via kexec""" + return read_file(KEXEC_CRASH_LOADED, defaultonfailure='0') == '1' + + +def get_crash_kernel_size() -> int: + """Return the number of bytes reserved for the capture kernel""" + + if not is_crash_kernel_loaded(): + return 0 + + raw = read_file(KEXEC_CRASH_SIZE, defaultonfailure='0') + return int(raw) if raw.isdigit() else 0 diff --git a/python/vyos/utils/memory.py b/python/vyos/utils/memory.py new file mode 100644 index 000000000..c279168b5 --- /dev/null +++ b/python/vyos/utils/memory.py @@ -0,0 +1,28 @@ +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +from vyos.utils.file import read_file + + +def get_memory_info() -> dict: + """Returns system memory information parsed from /proc/meminfo""" + data = read_file('/proc/meminfo') + + result = {} + regex = r'^(?P\S+):\s+(?P[0-9]+)\s+kB\s*$' + for match in re.finditer(regex, data, flags=re.MULTILINE): + result[match['key']] = int(match['value']) + + return result diff --git a/smoketest/scripts/cli/test_system_option.py b/smoketest/scripts/cli/test_system_option.py index f0b3bfb53..2ab27693b 100755 --- a/smoketest/scripts/cli/test_system_option.py +++ b/smoketest/scripts/cli/test_system_option.py @@ -17,22 +17,40 @@ import os import subprocess import unittest +import tempfile from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError +from vyos.defaults import systemd_services +from vyos.defaults import KDUMP_DEFAULT_MEMORY_AUTO from vyos.utils.cpu import get_cpus from vyos.utils.file import read_file +from vyos.utils.file import write_file from vyos.utils.process import is_systemd_service_active from vyos.utils.system import sysctl_read from vyos.system import image base_path = ['system', 'option'] +kdump_path = base_path + ['kdump'] + + +def _get_grub_config(): + """Read GRUB config file for current running image""" + return read_file(f'{image.grub.GRUB_DIR_VYOS_VERS}/{image.get_running_image()}.cfg') class TestSystemOption(VyOSUnitTestSHIM.TestCase): + def setUp(self): + self.tmp_path = tempfile.TemporaryDirectory(prefix='system-option-test-') + + # always forward to base class + super().setUp() + def tearDown(self): self.cli_delete(base_path) self.cli_commit() + self.tmp_path.cleanup() + # always forward to base class super().tearDown() @@ -129,7 +147,7 @@ class TestSystemOption(VyOSUnitTestSHIM.TestCase): self.cli_commit() # Read GRUB config file for current running image - tmp = read_file(f'{image.grub.GRUB_DIR_VYOS_VERS}/{image.get_running_image()}.cfg') + tmp = _get_grub_config() self.assertIn(' mitigations=off', tmp) self.assertIn(' intel_idle.max_cstate=0 processor.max_cstate=1', tmp) self.assertIn(' quiet', tmp) @@ -174,6 +192,139 @@ class TestSystemOption(VyOSUnitTestSHIM.TestCase): out = subprocess.check_output(['openssl', 'list', '-providers'], text=True) self.assertNotIn('OpenSSL FIPS Provider', out) + def test_kdump_base(self): + # Test basic kdump functionality + + dump_path = self.tmp_path.name + service = systemd_services['kdump'] + config_file = '/etc/default/kdump-tools' + initramfs_hook = '/etc/initramfs-tools/conf.d/zzzz-kdump-vyos-overlay' + + self.cli_set(kdump_path + ['dump-path', dump_path]) + self.cli_commit() + + self.assertTrue( + os.path.exists(config_file), + 'kdump-tools config file was not created after enabling kdump', + ) + + self.assertTrue( + os.path.exists(initramfs_hook), + 'initramfs conf.d hook was not created after enabling kdump', + ) + + hook_content = read_file(initramfs_hook) + self.assertIn( + 'MODULES=most', + hook_content, + 'initramfs hook must contain MODULES=most to work on OverlayFS root', + ) + + self.assertTrue( + is_systemd_service_active(service), + f'{service} must be active after enabling kdump', + ) + + # Disabling kdump must remove the initramfs conf.d drop-in + self.cli_delete(kdump_path) + self.cli_commit() + + self.assertFalse( + os.path.exists(initramfs_hook), + 'initramfs hook must be removed after disabling kdump', + ) + + self.assertTrue( + os.path.exists(config_file), + 'kdump-tools config file must still exist after disabling kdump', + ) + config_content = read_file(config_file) + self.assertIn( + 'USE_KDUMP=0', + config_content, + 'Disabled kdump-tools config must contain USE_KDUMP=0', + ) + + def test_kdump_memory(self): + # Test memory reservation logic + + memory = '128' + self.cli_set(kdump_path + ['memory', memory]) + self.cli_commit() + + grub_cfg = _get_grub_config() + self.assertIn( + f'crashkernel={memory}', + grub_cfg, + '`crashkernel` value not found in GRUB config after setting explicit memory', + ) + + # Tiered crashkernel= value produced when memory is set to 'auto' + auto_memory = KDUMP_DEFAULT_MEMORY_AUTO + + self.cli_set(kdump_path + ['memory', 'auto']) + self.cli_commit() + + grub_cfg = _get_grub_config() + self.assertIn( + f'crashkernel={auto_memory}', + grub_cfg, + "'memory auto' did not produce the expected tiered `crashkernel` value in GRUB config", + ) + + def test_kdump_memory_error(self): + # Test special cases where invalid memory range values should raise an error + + special_cases = ( + '1', # Invalid low value + '1048570', # ~1TB which often more then available RAM + ) + for case in special_cases: + with self.subTest(case=case): + with self.assertRaises(ConfigSessionError): + self.cli_set(kdump_path + ['memory', case]) + self.cli_commit() + + def test_op_show_kdump_status(self): + # Test operational mode command 'show system kdump' + + result = self.op_mode(['show', 'system', 'kdump']) + self.assertIn('not configured', result) + + memory = '256' + self.cli_set(kdump_path + ['memory', memory]) + self.cli_commit() + + result = self.op_mode(['show', 'system', 'kdump']) + self.assertIn('Crash kernel', result) + self.assertIn(memory, result) + + def test_op_show_kdump_dumps(self): + # Test operational mode command 'show system kdump dumps' + dump_path = self.tmp_path.name + + self.cli_set(kdump_path + ['dump-path', dump_path]) + self.cli_commit() + + result = self.op_mode(['show', 'system', 'kdump', 'dumps']) + self.assertIn('No kernel crash dumps recorded', result) + + timestamp1 = '202607070802' + timestamp2 = '202607070939' + + for ts in (timestamp1, timestamp2): + sub = os.path.join(dump_path, ts) + os.makedirs(sub) + + # Write a small synthetic vmcore so the size check is non-trivial + write_file(os.path.join(sub, f'dump.{ts}'), '0' * 1024) + write_file(os.path.join(sub, f'dmesg.{ts}'), f'synthetic dmesg {ts}') + + result = self.op_mode(['show', 'system', 'kdump', 'dumps']) + + self.assertIn(f'{dump_path}/{timestamp1}', result) + self.assertIn(f'{dump_path}/{timestamp2}', result) + if __name__ == '__main__': unittest.main(verbosity=2, failfast=VyOSUnitTestSHIM.TestCase.debug_on()) diff --git a/smoketest/scripts/system/test_kernel_options.py b/smoketest/scripts/system/test_kernel_options.py index c39d5a26b..9cecf561f 100755 --- a/smoketest/scripts/system/test_kernel_options.py +++ b/smoketest/scripts/system/test_kernel_options.py @@ -168,6 +168,18 @@ class TestKernelModules(unittest.TestCase): tmp = re.findall(f'{option}=y', self._config_data) self.assertTrue(tmp) + def test_kdump(self): + options = [ + 'CONFIG_KEXEC', + 'CONFIG_CRASH_DUMP', + 'CONFIG_DEBUG_INFO', + 'CONFIG_PROC_VMCORE', + ] + for option in options: + with self.subTest(option=option): + tmp = re.findall(f'{option}=y', self._config_data) + self.assertTrue(tmp, msg='Required kdump option must be enabled') + def test_openvpn_dco(self): options_to_check = ['CONFIG_OVPN'] for option in options_to_check: diff --git a/src/conf_mode/system_option.py b/src/conf_mode/system_option.py index 8951adcff..190db232d 100755 --- a/src/conf_mode/system_option.py +++ b/src/conf_mode/system_option.py @@ -26,22 +26,27 @@ from vyos.base import Warning from vyos.config import Config from vyos.configverify import verify_source_interface from vyos.configverify import verify_interface_exists +from vyos.defaults import KDUMP_DEFAULT_MEMORY_AUTO +from vyos.system import image from vyos.system import grub_util from vyos.template import render from vyos.utils.boot import boot_configuration_complete from vyos.utils.convert import range_str_to_list from vyos.utils.convert import list_to_range_str +from vyos.utils.convert import human_to_bytes from vyos.utils.cpu import get_cpus from vyos.utils.cpu import get_available_cpus from vyos.utils.dict import dict_search from vyos.utils.file import write_file -from vyos.utils.file import read_file from vyos.utils.kernel import check_kmod +from vyos.utils.kernel import get_kernel_boot_arg +from vyos.utils.kernel import get_crash_kernel_size from vyos.utils.process import cmdl from vyos.utils.process import is_systemd_service_running from vyos.utils.network import is_addr_assigned from vyos.utils.network import is_intf_addr_assigned from vyos.utils.system import sysctl_write +from vyos.utils.memory import get_memory_info from vyos.configdep import set_dependents from vyos.configdep import call_dependents from vyos import ConfigError @@ -73,6 +78,15 @@ tuned_profiles = { 'virtual-host': 'virtual-host', } +# Path to the kdump-tools configuration file controlling dump behaviour +KDUMP_CONFIG_FILE = '/etc/default/kdump-tools' + +# initramfs-tools drop-in configuration applied during kdump initrd +# generation. Named with 'zzzz-' prefix to ensure it is sourced last, +# overriding any conflicting settings from other drop-ins (including the +# `MODULES=dep` override written by the kdump-tools kernel postinst hook). +KDUMP_INITRAMFS_HOOK = '/etc/initramfs-tools/conf.d/zzzz-kdump-vyos-overlay' + MANAGED_PARAMS = { 'hugepages1g': { 'parse': r'hugepagesz=1[Gg]\s+hugepages=(?P\d+)', @@ -168,6 +182,11 @@ MANAGED_PARAMS = { 'clean': r'numa_balancing=\S+', 'type': str, }, + 'crashkernel': { + 'parse': r'crashkernel=(?P\S+)', + 'clean': r'crashkernel=\S+', + 'type': str, + }, } # Compiled regex pattern for parsing command line options @@ -198,9 +217,18 @@ def get_config(config=None): conf = Config() base = ['system', 'option'] options = conf.get_config_dict( - base, key_mangling=('-', '_'), get_first_key=True, with_recursive_defaults=True + base, + key_mangling=('-', '_'), + get_first_key=True, + with_recursive_defaults=True, + with_pki=True, ) + if 'kdump' not in options: + options['kdump'] = {} + options['kdump']['is_live_boot'] = image.is_live_boot() + options['kdump']['disabled'] = not conf.exists(base + ['kdump']) + if 'performance' in options: # Update IPv4/IPv6 and sysctl options after tuned applied it's settings set_dependents('ip_ipv6', conf) @@ -314,6 +342,32 @@ def verify(options): f'({reserved_gb} GB is reserved for system usage and services)' ) + if not options['kdump']['disabled']: + kdump = options['kdump'] + + if kdump.get('is_live_boot'): + raise ConfigError('kdump is not supported on live boot images') + + # Check that the configured crash kernel memory does not exceed the + # total system RAM for the applicable range + memory = kdump.get('memory') + if memory and memory != 'auto': + memory_info = get_memory_info() + # Convert kilobytes to bytes + memory_total = memory_info['MemTotal'] * 1024 + + # 'memory_info' doesn't consider crash kernel reservation size + memory_total += get_crash_kernel_size() + + # Convert scalar value from megabytes to bytes + memory_bytes = human_to_bytes(f'{memory}mb') + + # Ensure that specified memory does not exceed total virtual memory + if memory_bytes >= memory_total: + raise ConfigError( + f'Specified kdump memory {memory!r} exceeds the available system RAM' + ) + return None @@ -401,6 +455,45 @@ def generate(options): if count: cmdline_options.append(f'hugepages={count}') + kdump = options['kdump'] + if not kdump['disabled']: + # Crash kernel memory reservation + kdump_memory = kdump.get('memory') + if kdump_memory: + if kdump_memory == 'auto': + # Tiered auto-sizing based on total system RAM + kdump_memory = KDUMP_DEFAULT_MEMORY_AUTO + else: + # Append suffix (megabytes) to right format + kdump_memory = f'{kdump_memory}M' + + # The `crashkernel=` parameter instructs the boot kernel to reserve a + # dedicated memory region for the capture kernel. + cmdline_options.append(f'crashkernel={kdump_memory}') + + # Render the kdump-tools configuration file from the Jinja2 template + render(KDUMP_CONFIG_FILE, 'system/kdump_tools.conf.j2', kdump) + + # VyOS uses an OverlayFS root filesystem. When mkinitramfs runs with + # `MODULES=dep`, it attempts to determine which kernel modules are needed + # by walking up the device tree from /. On a real block device + # this works correctly, but OverlayFS has no backing block device to walk from, + # causing mkinitramfs to fail with: + # mkinitramfs: failed to determine device for / + write_file(KDUMP_INITRAMFS_HOOK, 'MODULES=most') + else: + # kdump has been disabled - render the + # kdump-tools configuration with the disabled flag so the service + # starts in a no-op state rather than attempting to load a capture + # kernel with stale or missing parameters. + render(KDUMP_CONFIG_FILE, 'system/kdump_tools.conf.j2', kdump) + + # Remove ephemeral files after the feature is disabled + kdump_delete_files = (KDUMP_INITRAMFS_HOOK,) + for delete_file in kdump_delete_files: + if os.path.exists(delete_file): + os.unlink(delete_file) + cmdline_options_str = ' '.join(cmdline_options) grub_util.update_kernel_cmdline_options(cmdline_options_str) @@ -464,7 +557,7 @@ def generate_cmdline_for_kexec(options): - new_cmdline (str): The updated kernel command line string. """ # Read current cmdline and parse it - current_cmdline = read_file('/proc/cmdline').strip() + current_cmdline = get_kernel_boot_arg().strip() current_parsed = parse_cmdline(current_cmdline) # Parse desired options from options['cmdline_options'] diff --git a/src/etc/systemd/system/kdump-tools.service.d/override.conf b/src/etc/systemd/system/kdump-tools.service.d/override.conf new file mode 100644 index 000000000..67419e10b --- /dev/null +++ b/src/etc/systemd/system/kdump-tools.service.d/override.conf @@ -0,0 +1,19 @@ +[Unit] +Description=Kernel crash dump capture service +DefaultDependencies=no +Before= +After= + +[Service] +Type=oneshot +StandardOutput=journal+console +EnvironmentFile=/etc/default/kdump-tools +ExecStart= +ExecStart=/etc/init.d/kdump-tools start +ExecStop= +ExecStop=/etc/init.d/kdump-tools stop +RemainAfterExit=yes + +[Install] +WantedBy= +WantedBy=vyos.target diff --git a/src/op_mode/generate_tech-support_archive.py b/src/op_mode/generate_tech-support_archive.py index 2719d231c..7b539473c 100755 --- a/src/op_mode/generate_tech-support_archive.py +++ b/src/op_mode/generate_tech-support_archive.py @@ -17,6 +17,7 @@ import os import argparse import glob +import json from datetime import datetime from pathlib import Path from shutil import rmtree @@ -24,7 +25,9 @@ from socket import gethostname from sys import exit from tarfile import open as tar_open +from vyos.base import Warning from vyos.defaults import directories +from vyos.utils.process import rc_cmd from vyos.utils.process import call from vyos.utils.process import cmdl from vyos.utils.file import get_name_from_path @@ -38,6 +41,8 @@ ARCHIVE_TMP_DIR_PATTERN = 'drops-debug_' DEFAULT_TMP_DIR = '/tmp' EXCLUDED_ARCHIVE_EXT = ('.iso', '.gz', '.tar', '.zip') +vyos_op_scripts_dir = directories['op_mode'] + def __rotate_logs(path: str, log_pattern:str): files_list = glob.glob(f'{path}/{log_pattern}') @@ -53,7 +58,6 @@ def __save_show_report_files(reports_dir: Path): :type reports_dir: pathlib.Path """ - vyos_op_scripts_dir = directories['op_mode'] script_path = f'{vyos_op_scripts_dir}/show_techsupport_report.py' arguments = [ '--launched-from-generate-archive', @@ -111,6 +115,28 @@ def __generate_archived_files(location_path: str) -> None: 'run': '/run', } + # Get the directory of the last crash dump and add the crash directory to archive dictionary + err_code, out = rc_cmd([f'{vyos_op_scripts_dir}/kdump.py', 'show_dumps', '--raw']) + if not err_code: + try: + dumps = json.loads(out) + except json.decoder.JSONDecodeError as e: + Warning(f'Unable to parse kdump dump list. {e}') + dumps = [] + + if dumps: + last_dump = dumps[-1] + # Before enabling the dump, consider the size as + # it can range from hundreds of MB to several GB + max_dump_archive_bytes = 1 * 1024**3 # 1 GB cap + if last_dump['size_bytes'] > max_dump_archive_bytes: + size_human = last_dump['size_human'] + Warning( + f"Skipping crash dump archival: {size_human} exceeds limit 1 GB" + ) + else: + archive_dict['last-crash-dump'] = last_dump['directory'] + for archive_name, path in archive_dict.items(): if not os.path.exists(path): continue diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 5174ff3d2..fb28f182e 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -47,6 +47,7 @@ from vyos.config_mgmt import unsaved_commits from vyos.defaults import base_dir from vyos.defaults import directories from vyos.defaults import activation_hint +from vyos.defaults import KDUMP_DEFAULT_MEMORY_AUTO from vyos.flavor import get_image_serial_console from vyos.remote import download from vyos.system import disk @@ -525,9 +526,11 @@ def get_cli_kernel_options(config_file: str) -> list: config = ConfigTree(read_file(config_file)) config_dict = loads(config.to_json()) cmdline_options = [] - kernel_options = dict_search('system.option.kernel', config_dict) - if kernel_options is None: + + options = dict_search('system.option', config_dict) + if options is None: return cmdline_options + kernel_options = options.get('kernel', {}) k_cpu_opts = kernel_options.get('cpu', {}) k_memory_opts = kernel_options.get('memory', {}) @@ -591,6 +594,17 @@ def get_cli_kernel_options(config_file: str) -> list: if count: cmdline_options.append(f'hugepages={count}') + # Crash kernel memory reservation for the capture kernel + kdump_memory = options.get('kdump', {}).get('memory') + if kdump_memory: + if kdump_memory == 'auto': + # Tiered auto-sizing based on total system RAM + kdump_memory = KDUMP_DEFAULT_MEMORY_AUTO + else: + # Append suffix (megabytes) to right format + kdump_memory = f'{kdump_memory}M' + cmdline_options.append(f'crashkernel={kdump_memory}') + return cmdline_options def configure_authentication(config_file: str, password: str) -> None: diff --git a/src/op_mode/kdump.py b/src/op_mode/kdump.py new file mode 100644 index 000000000..560650bdf --- /dev/null +++ b/src/op_mode/kdump.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# +# Copyright (C) VyOS Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +from datetime import datetime + +import vyos.opmode +from tabulate import tabulate +from vyos.configquery import ConfigTreeQuery +from vyos.defaults import systemd_services +from vyos.utils.convert import bytes_to_human +from vyos.utils.kernel import get_kernel_boot_arg +from vyos.utils.kernel import is_crash_kernel_loaded +from vyos.utils.kernel import get_crash_kernel_size +from vyos.utils.process import is_systemd_service_active + +# systemd unit managed by kdump-tools that loads the capture kernel +# and handles vmcore saving after a panic +KDUMP_SERVICE = systemd_services['kdump'] + +# Base path to boot files related to kdump +KDUMP_LIB_PATH = '/var/lib/kdump' + +DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S' + +conf = ConfigTreeQuery() +base = ['system', 'option', 'kdump'] + + +def _get_config() -> dict: + """Return the effective kdump config dict, or {} when unconfigured""" + + if not conf.exists(base): + return {} + + return conf.get_config_dict( + base, key_mangling=('-', '_'), get_first_key=True, with_recursive_defaults=True + ) + + +def _read_link(symlink_path: str) -> str | None: + """Read the destination of the symbolic link""" + try: + destination_path = os.readlink(symlink_path) + except OSError: + return None + + # os.readlink() may return a relative target. + # Interpret it relative to the link directory + if not os.path.isabs(destination_path): + destination_path = os.path.join(os.path.dirname(symlink_path), destination_path) + + destination_path = os.path.realpath(destination_path) + return destination_path if os.path.isfile(destination_path) else None + + +def _list_dumps(dump_path: str) -> list[dict]: + """Return a sorted list of vmcore entries found under *dump_path*. + + kdump-tools stores each capture in its own timestamped sub-directory + using the layout: + + //dump. + //dmesg. (optional) + + The reported size is the combined size of both files when the dmesg + companion file is present. + """ + + entries = [] + if not os.path.isdir(dump_path): + return entries + + # Each sub-directory name is the timestamp of the crash event + for name in sorted(os.listdir(dump_path)): + sub = os.path.join(dump_path, name) + if not os.path.isdir(sub): + continue + + dump = os.path.join(sub, f'dump.{name}') + dmesg = os.path.join(sub, f'dmesg.{name}') + + if os.path.isfile(dump): + stat_dump = os.stat(dump) + stat_dmesg = os.stat(dmesg) if os.path.isfile(dmesg) else None + full_size = stat_dump.st_size + (stat_dmesg.st_size if stat_dmesg else 0) + + entry = { + 'directory': sub, + 'size_bytes': full_size, + 'size_human': bytes_to_human(full_size), + 'modify_time': stat_dump.st_mtime, + } + + entries.append(entry) + + return entries + + +def _get_raw_status() -> dict: + """Collect all kdump runtime and configuration data into a single dict""" + + cfg = _get_config() + + memory = cfg.get('memory') + dump_path = cfg.get('dump_path') + dumps = _list_dumps(dump_path) if dump_path else [] + last_dump = dumps[-1] if dumps else None + + return { + 'configured': bool(cfg), + 'service_active': is_systemd_service_active(KDUMP_SERVICE), + 'crash_kernel_loaded': is_crash_kernel_loaded(), + 'crash_kernel_size': get_crash_kernel_size(), + 'cmdline_value': get_kernel_boot_arg('crashkernel') or '', + 'vmlinuz_path': _read_link(f'{KDUMP_LIB_PATH}/vmlinuz'), + 'initrd_path': _read_link(f'{KDUMP_LIB_PATH}/initrd.img'), + 'memory': memory, + 'dump_path': dump_path, + 'last_dump': last_dump, + } + + +def _get_raw_dumps() -> list[dict]: + cfg = _get_config() + dump_path = cfg.get('dump_path') + return _list_dumps(dump_path) if dump_path else [] + + +def _verify_config(): + """Raise UnconfiguredSubsystem when kdump has not been configured""" + + cfg = _get_config() + if not cfg: + conf_command = ' '.join(['set'] + base) + raise vyos.opmode.UnconfiguredSubsystem( + f'kdump is not configured - use "{conf_command}" to enable it.' + ) + + +def _format_status(data: dict) -> str: + """Render the kdump status dict as a human-readable plain-text table""" + + header = 'Kernel crash dump (kdump) status' + + cfg_str = 'configured' if data['configured'] else 'not configured' + svc_str = 'active' if data['service_active'] else 'inactive' + loaded_str = 'loaded' if data['crash_kernel_loaded'] else 'not loaded' + + rows = [ + ('Configuration', cfg_str), + ('Systemd service', svc_str), + ('Crash kernel', loaded_str), + ] + + if data['crash_kernel_loaded']: + # Show reserved memory size and the boot argument that produced it + + raw = data['crash_kernel_size'] + rows.append(('Reserved memory', bytes_to_human(raw))) + + rows.append(("Boot 'crashkernel' value", data['cmdline_value'])) + else: + # Capture kernel is not loaded and memory reservation requires a reboot + rows.append(('Reserved memory', 'none (reboot required to reserve memory)')) + + if data['configured']: + rows.append(('Memory parameter', data['memory'])) + rows.append(('Directory to save dumps', data['dump_path'])) + + # Display the latest date of created dump + last_dump_date = 'none' + if data['last_dump']: + dt = datetime.fromtimestamp(data['last_dump']['modify_time']) + last_dump_date = dt.strftime(DEFAULT_TIME_FORMAT) + rows.append(('Last dump date', last_dump_date)) + + # Show `initrd.img` and `vmlinuz` status which are critical + # for a successful boot process + rows.append(('Initial RAM disk image', data['initrd_path'] or 'none')) + rows.append(('Linux kernel executable file', data['vmlinuz_path'] or 'none')) + + # Expand each row to a three-column tuple so tabulate can align the + # separator colon independently of the label and value columns + rows = [(header, ':', value) for header, value in rows] + table = tabulate(rows, tablefmt='plain') + + return f'{header}\n\n{table}' + + +def _format_dumps(entries: list[dict]) -> str: + """Render the crash dump list as a human-readable table""" + + if not entries: + return 'No kernel crash dumps recorded' + + headers = ['DIRECTORY', 'SIZE', 'TIME'] + rows = [] + + for entry in entries: + dt = datetime.fromtimestamp(entry['modify_time']) + row = ( + entry['directory'], + entry['size_human'], + dt.strftime(DEFAULT_TIME_FORMAT), + ) + rows.append(row) + + # Append a blank separator row followed by an aggregate totals row + total = sum(entry['size_bytes'] for entry in entries) + rows.append(('', '', '')) + rows.append((f'Total: {len(entries)} dump(s)', bytes_to_human(total), '')) + + return tabulate(rows, headers, tablefmt='simple') + + +def show_status(raw: bool): + """Show kdump service status and configuration summary""" + + _verify_config() + + data = _get_raw_status() + return data if raw else _format_status(data) + + +def show_dumps(raw: bool): + """Show recorded kernel crash dumps""" + + _verify_config() + + data = _get_raw_dumps() + return data if raw else _format_dumps(data) + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) diff --git a/src/op_mode/show_techsupport_report.py b/src/op_mode/show_techsupport_report.py index 2434da29a..fa6bff6ba 100644 --- a/src/op_mode/show_techsupport_report.py +++ b/src/op_mode/show_techsupport_report.py @@ -543,6 +543,11 @@ REPORTS: dict[str, tuple[BaseSpec]] = { CommandSpec('', 'vppctl show mpls tunnel'), CommandSpec('', 'vppctl show trace'), ), + 'kernel-dump': ( + CommandSpec('Kdump status and configuration', op('show system kdump')), + CommandSpec('', 'kdump-config show'), + CommandSpec('Recorded kernel crash dumps', op('show system kdump dumps')), + ), } -- cgit v1.2.3 From b1d4de376eb79c932539c5ec2ddb5241fbce8bb7 Mon Sep 17 00:00:00 2001 From: Oleksandr Kuchmystyi Date: Thu, 23 Jul 2026 12:17:36 +0300 Subject: utils: T8868: Refactor kernel command-line arguments and memory info - Replaced direct file reads from `/proc/cmdline` with `get_kernel_boot_arg()` for fetching kernel command-line arguments in multiple modules. - Improved memory information retrieval by utilizing `get_memory_info()`. --- python/vyos/system/image.py | 17 +++++++++++------ python/vyos/utils/kernel.py | 7 +++---- src/helpers/run-config-activation.py | 3 ++- src/helpers/run-config-migration.py | 5 ++--- src/helpers/vyos-boot-config-loader.py | 7 +++---- src/op_mode/memory.py | 19 ++++++------------- 6 files changed, 27 insertions(+), 31 deletions(-) diff --git a/python/vyos/system/image.py b/python/vyos/system/image.py index ed8a96fbb..02f2303b2 100644 --- a/python/vyos/system/image.py +++ b/python/vyos/system/image.py @@ -22,6 +22,7 @@ from json import loads from vyos.defaults import directories from vyos.system import disk, grub +from vyos.utils.kernel import get_kernel_boot_arg # Define variables GRUB_DIR_MAIN: str = '/boot/grub' @@ -29,8 +30,12 @@ GRUB_DIR_VYOS: str = f'{GRUB_DIR_MAIN}/grub.cfg.d' CFG_VYOS_VARS: str = f'{GRUB_DIR_VYOS}/20-vyos-defaults-autoload.cfg' GRUB_DIR_VYOS_VERS: str = f'{GRUB_DIR_VYOS}/vyos-versions' # prepare regexes -REGEX_KERNEL_CMDLINE: str = r'^BOOT_IMAGE=/(?Pboot|live)/((?P.+)/)?vmlinuz.*$' -REGEX_SYSTEM_CFG_VER: str = r'(\r\n|\r|\n)SYSTEM_CFG_VER\s*=\s*(?P\d+)(\r\n|\r|\n)' +REGEX_KERNEL_CMDLINE: str = ( + r'^/(?Pboot|live)/((?P.+)/)?vmlinuz.*$' +) +REGEX_SYSTEM_CFG_VER: str = ( + r'(\r\n|\r|\n)SYSTEM_CFG_VER\s*=\s*(?P\d+)(\r\n|\r|\n)' +) # structures definitions @@ -197,8 +202,8 @@ def get_running_image() -> str: """ running_image: str = '' regex_filter = re_compile(REGEX_KERNEL_CMDLINE) - cmdline: str = Path('/proc/cmdline').read_text() - running_image_result = regex_filter.match(cmdline) + cmdline_arg: str = get_kernel_boot_arg('BOOT_IMAGE') or '' + running_image_result = regex_filter.match(cmdline_arg) if running_image_result: running_image: str = running_image_result.groupdict().get( 'image_version', '') @@ -259,8 +264,8 @@ def is_live_boot() -> bool: bool: True if the system currently booted in live mode """ regex_filter = re_compile(REGEX_KERNEL_CMDLINE) - cmdline: str = Path('/proc/cmdline').read_text() - running_image_result = regex_filter.match(cmdline) + cmdline_arg: str = get_kernel_boot_arg('BOOT_IMAGE') or '' + running_image_result = regex_filter.match(cmdline_arg) if running_image_result: boot_type: str = running_image_result.groupdict().get('boot_type', '') if boot_type == 'boot': diff --git a/python/vyos/utils/kernel.py b/python/vyos/utils/kernel.py index 69574cefc..72c7413cf 100644 --- a/python/vyos/utils/kernel.py +++ b/python/vyos/utils/kernel.py @@ -165,14 +165,13 @@ def get_kernel_serial_console() -> Tuple[Optional[str], Optional[str], Optional[ command line which was used during system boot. """ import re - from vyos.utils.file import read_file cmdline_console_re = re.compile( - r'(?:^|\s)console=(?Ptty(?:S|AMA))(?P\d+),(?P\d+)(?=\s|$)' + r'(?Ptty(?:S|AMA))(?P\d+),(?P\d+)' ) - kernel_cmdline = read_file('/proc/cmdline') - if m := cmdline_console_re.search(kernel_cmdline): + console_value = get_kernel_boot_arg('console') or '' + if m := cmdline_console_re.search(console_value): return ( m.group('console_type'), m.group('console_num'), diff --git a/src/helpers/run-config-activation.py b/src/helpers/run-config-activation.py index 7e7a6571e..9e048e193 100755 --- a/src/helpers/run-config-activation.py +++ b/src/helpers/run-config-activation.py @@ -30,6 +30,7 @@ from vyos.utils.activate import set_activation from vyos.utils.activate import is_active from vyos.utils.system import load_as_module from vyos.utils.func import FalseCallable +from vyos.utils.kernel import get_kernel_boot_arg from vyos.defaults import directories from vyos.defaults import activation_list @@ -54,7 +55,7 @@ fh.setFormatter(formatter) logger.addHandler(fh) -if 'vyos-activate-debug' in Path('/proc/cmdline').read_text(): +if get_kernel_boot_arg('vyos-activate-debug') is not None: print(f'\nactivate-debug enabled: file {checkpoint_file}_* on error') debug = checkpoint_file logger.setLevel(logging.DEBUG) diff --git a/src/helpers/run-config-migration.py b/src/helpers/run-config-migration.py index 6329d2979..f538c1e86 100755 --- a/src/helpers/run-config-migration.py +++ b/src/helpers/run-config-migration.py @@ -19,7 +19,7 @@ import sys import time from argparse import ArgumentParser from shutil import copyfile -from vyos.utils.file import read_file +from vyos.utils.kernel import get_kernel_boot_arg from vyos.migrate import ConfigMigrate from vyos.migrate import ConfigMigrateError @@ -80,6 +80,5 @@ if backup is not None and not config_migrate.config_modified: # T1771: add knob on Kernel command-line to simulate failed config migrator run # used to test if the automatic image reboot works. -kernel_cmdline = read_file('/proc/cmdline') -if 'vyos-fail-migration' in kernel_cmdline.split(): +if get_kernel_boot_arg('vyos-fail-migration') is not None: sys.exit(1) diff --git a/src/helpers/vyos-boot-config-loader.py b/src/helpers/vyos-boot-config-loader.py index a3a66eedc..e01b5bff6 100755 --- a/src/helpers/vyos-boot-config-loader.py +++ b/src/helpers/vyos-boot-config-loader.py @@ -28,6 +28,7 @@ from vyos.configsession import ConfigSessionError from vyos.configtree import ConfigTree from vyos.utils.process import cmdl from vyos.utils.file import write_file +from vyos.utils.kernel import get_kernel_boot_arg STATUS_FILE = config_status TRACE_FILE = '/tmp/boot-config-trace' @@ -44,11 +45,9 @@ else: LOG_FILE = LOG_DIR + '/vyos-boot-config-loader.log' try: - with open('/proc/cmdline', 'r') as f: - cmdline = f.read() - if 'vyos-debug' in cmdline: + if get_kernel_boot_arg('vyos-debug') is not None: os.environ['VYOS_DEBUG'] = 'yes' - if 'vyos-config-debug' in cmdline: + if get_kernel_boot_arg('vyos-config-debug') is not None: os.environ['VYOS_DEBUG'] = 'yes' trace_config = True except Exception as e: diff --git a/src/op_mode/memory.py b/src/op_mode/memory.py index 20d937243..47a9c476e 100755 --- a/src/op_mode/memory.py +++ b/src/op_mode/memory.py @@ -18,23 +18,16 @@ import sys import vyos.opmode +from vyos.utils.memory import get_memory_info def _get_raw_data(): - from re import search as re_search + mem_info = get_memory_info() - def find_value(keyword, mem_data): - regex = keyword + ':\s+(\d+)' - res = re_search(regex, mem_data).group(1) - return int(res) - - with open("/proc/meminfo", "r") as f: - mem_data = f.read() - - total = find_value('MemTotal', mem_data) - available = find_value('MemAvailable', mem_data) - buffers = find_value('Buffers', mem_data) - cached = find_value('Cached', mem_data) + total = mem_info['MemTotal'] + available = mem_info['MemAvailable'] + buffers = mem_info['Buffers'] + cached = mem_info['Cached'] used = total - available -- cgit v1.2.3 From 7b960007e817b722596b7e388577d73ad7a87ed7 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 14 Aug 2026 17:05:13 +0300 Subject: T8868: Change help message for reserved memory Co-authored-by: Daniil Baturin --- interface-definitions/system_option.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/system_option.xml.in b/interface-definitions/system_option.xml.in index beeca74c6..c295b2ebe 100644 --- a/interface-definitions/system_option.xml.in +++ b/interface-definitions/system_option.xml.in @@ -420,7 +420,7 @@ - Amount of memory to reserve for the crash kernel + Amount of memory to reserve for the kernel crash dump auto Automatically determine memory reservation -- cgit v1.2.3