Merge pull request #5130 from natali-rs1985/T8460

vpp: T8460: Use isolated cpus for VPP cpu-cores

8 files changed, 227 insertions, 56 deletions

diff --git a/data/templates/vpp/startup.conf.j2 b/data/templates/vpp/startup.conf.j2
index 5d5406747..82d3753ae 100644
--- a/data/templates/vpp/startup.conf.j2
+++ b/data/templates/vpp/startup.conf.j2
@@ -18,11 +18,8 @@ cpu {
 {%     if cpu.main_core is vyos_defined %}
     main-core {{ cpu.main_core }}
 {%     endif %}
-{%     if cpu.skip_cores is vyos_defined %}
-    skip-cores {{ cpu.skip_cores }}
-{%     endif %}
-{%     if cpu.workers is vyos_defined %}
-    workers {{ cpu.workers }}
+{%     if cpu.corelist_workers is vyos_defined %}
+    corelist-workers {{ cpu.corelist_workers }}
 {%     endif %}
 }
 {% endif %}
diff --git a/python/vyos/utils/convert.py b/python/vyos/utils/convert.py
index ea07f9514..bdc9fd549 100644
--- a/python/vyos/utils/convert.py
+++ b/python/vyos/utils/convert.py
@@ -261,3 +261,56 @@ def encode_to_base64(input_string):
 
     # Decode the base64 bytes back to a string
     return encoded_string.decode('utf-8')
+
+
+def range_str_to_list(data: str) -> list[int]:
+    """
+    Convert a string of ranges to a sorted list of unique integers
+
+    Args:
+        data (str): Comma-separated ranges, e.g. '1-3,5,7-9'
+
+    Returns:
+        Sorted list of unique integers, e.g. ``[1, 2, 3, 5, 7, 8, 9]``.
+    """
+    if not data:
+        return []
+    result = []
+    for part in data.split(','):
+        if '-' in part:
+            start, end = part.split('-')
+            result.extend(range(int(start), int(end) + 1))
+        else:
+            result.append(int(part))
+    return sorted(set(result))
+
+
+def list_to_range_str(nums: list[int]) -> str:
+    """
+    Convert a list of integers to a compact string of ranges
+
+    Args:
+        nums: List of integers, e.g. [1, 2, 3, 5, 7, 8, 9].
+              Duplicates are removed and the list is sorted internally.
+
+    Returns:
+        Compact range string, e.g. '1-3,5,7-9'.
+        Returns '' for an empty input.
+    """
+    nums = sorted(set(nums))
+    if not nums:
+        return ''
+
+    ranges = []
+    start = end = nums[0]
+
+    for n in nums[1:]:
+        if n == end + 1:
+            end = n
+        else:
+            ranges.append(str(start) if start == end else f'{start}-{end}')
+            start = end = n
+
+    ranges.append(str(start) if start == end else f'{start}-{end}')
+
+    return ','.join(ranges)
diff --git a/python/vyos/vpp/config_verify.py b/python/vyos/vpp/config_verify.py
index da8e3d24b..228fcbc09 100644
--- a/python/vyos/vpp/config_verify.py
+++ b/python/vyos/vpp/config_verify.py
@@ -20,8 +20,10 @@ import psutil
 
 from vyos import ConfigError
 from vyos.base import Warning
+from vyos.utils.convert import range_str_to_list
 from vyos.utils.cpu import get_core_count as total_core_count, get_cpus
 from vyos.utils.dict import dict_search
+from vyos.utils.file import read_file
 
 from vyos.vpp.config_resource_checks import memory as mem_checks
 from vyos.vpp.config_resource_checks.resource_defaults import default_resource_map
@@ -54,6 +56,9 @@ _VPP_MEMBER_INTERFACE_REFS = [
 
 _VPP_INTERFACE_REFS = _VPP_FEATURE_INTERFACE_REFS + _VPP_MEMBER_INTERFACE_REFS
 
+# Line width used by ConfigError for message formatting
+LINE_WIDTH = 72
+
 
 def vpp_interface_in_use(
     iface: str, config: dict, match_vlans: bool = False, refs: list = None
@@ -217,12 +222,12 @@ def create_cpu_error_message(cpus_required: int, cpus_available: int = None) ->
 
     available_str = (
         (
-            '---'.ljust(72)
-            + 'Reserved:'.ljust(72)
-            + f'For system: {reserved_cpus}'.ljust(72)
-            + f'VPP main thread: 1'.ljust(72)
-            + '---'.ljust(72)
-            + 'Available:'.ljust(72)
+            '---'.ljust(LINE_WIDTH)
+            + 'Reserved:'.ljust(LINE_WIDTH)
+            + f'For system: {reserved_cpus}'.ljust(LINE_WIDTH)
+            + f'VPP main thread: 1'.ljust(LINE_WIDTH)
+            + '---'.ljust(LINE_WIDTH)
+            + 'Available:'.ljust(LINE_WIDTH)
             + f'Physical cores: {max(cpus_available, 0)}'
         )
         if cpus_available is not None
@@ -230,13 +235,13 @@ def create_cpu_error_message(cpus_required: int, cpus_available: int = None) ->
     )
 
     message = (
-        '---'.ljust(72)
-        + 'Total in the system:'.ljust(72)
-        + f'Physical cores: {total_core_count()}'.ljust(72)
-        + f'Logical cores: {logical_cores}'.ljust(72)
-        + '---'.ljust(72)
-        + 'Required:'.ljust(72)
-        + f'Physical cores: {cpus_required}'.ljust(72)
+        '---'.ljust(LINE_WIDTH)
+        + 'Total in the system:'.ljust(LINE_WIDTH)
+        + f'Physical cores: {total_core_count()}'.ljust(LINE_WIDTH)
+        + f'Logical cores: {logical_cores}'.ljust(LINE_WIDTH)
+        + '---'.ljust(LINE_WIDTH)
+        + 'Required:'.ljust(LINE_WIDTH)
+        + f'Physical cores: {cpus_required}'.ljust(LINE_WIDTH)
         + available_str
     )
 
@@ -251,7 +256,7 @@ def verify_vpp_minimum_cpus():
     min_cpus = default_resource_map.get('min_cpus')
     if total_core_count() < min_cpus:
         raise ConfigError(
-            'This system does not meet minimal requirements for VPP. '.ljust(72)
+            'This system does not meet minimal requirements for VPP. '.ljust(LINE_WIDTH)
             + create_cpu_error_message(min_cpus)
         )
 
@@ -329,10 +334,10 @@ def verify_vpp_memory(config: dict):
 
     if errors:
         raise ConfigError(
-            'Not enough free memory to start VPP! '.ljust(72)
-            + '. '.join([line.ljust(72) for line in errors.values()])
+            'Not enough free memory to start VPP! '.ljust(LINE_WIDTH)
+            + '. '.join([line.ljust(LINE_WIDTH) for line in errors.values()])
             + (
-                'To add HugePages memory please use command '.ljust(72)
+                'To add HugePages memory please use command '.ljust(LINE_WIDTH)
                 + '"set system option kernel memory hugepage-size ..." and reboot!'
                 if any(k in errors for k in ('2M', '1G'))
                 else ''
@@ -342,8 +347,19 @@ def verify_vpp_memory(config: dict):
 
 def verify_vpp_cpu_cores(cpu_cores: int):
     """
-    Verify that the system has enough available CPU cores
-    to run a given amount of worker processes (1 worker/core)
+    Verify that the system has enough available and isolated CPU cores.
+
+    Checks performed:
+      1. The host has enough physical cores (minus reserved) for the requested
+         cpu_cores count.
+      2. The kernel actually has at least cpu_cores isolated CPUs
+         (read from /sys/devices/system/cpu/isolated).
+
+    Args:
+        cpu_cores: Total number of VPP CPU cores (1 main + N-1 workers).
+
+    Raises:
+        ConfigError: When any of the checks fail.
     """
     total_cores = total_core_count()
     reserved_cpus = default_resource_map.get('reserved_cpu_cores')
@@ -351,10 +367,27 @@ def verify_vpp_cpu_cores(cpu_cores: int):
 
     if cpu_cores > available_cores:
         raise ConfigError(
-            f'Not enough free physical CPU cores for {cpu_cores} "cpu-cores" '.ljust(72)
+            f'Not enough free physical CPU cores for {cpu_cores} "cpu-cores" '.ljust(
+                LINE_WIDTH
+            )
             + create_cpu_error_message(cpu_cores, available_cores)
         )
 
+    # Read the set of CPUs the running kernel has actually isolated
+    isolated = read_file('/sys/devices/system/cpu/isolated')
+    isolated_cpus = range_str_to_list(isolated)
+
+    if cpu_cores > len(isolated_cpus):
+        raise ConfigError(
+            'Not enough isolated CPU cores available: '.ljust(LINE_WIDTH)
+            + f'{cpu_cores} requested, but only {len(isolated_cpus)} isolated'
+            f'{f" (CPU#{isolated})" if len(isolated_cpus) > 0 else ""}. '.ljust(
+                LINE_WIDTH
+            )
+            + 'To isolate CPUs please use command '.ljust(LINE_WIDTH)
+            + '"set system option kernel cpu isolate-cpus ..." save and reboot!'
+        )
+
 
 def verify_vpp_statseg_size(settings: dict):
     statseg_size = mem_checks.statseg_size(settings)
diff --git a/smoketest/scripts/cli/test_system_option.py b/smoketest/scripts/cli/test_system_option.py
index 5a8e5b78c..e025c01ab 100755
--- a/smoketest/scripts/cli/test_system_option.py
+++ b/smoketest/scripts/cli/test_system_option.py
@@ -104,12 +104,10 @@ class TestSystemOption(VyOSUnitTestSHIM.TestCase):
 
     def test_kernel_options(self):
         amd_pstate_mode = 'active'
-        isolate_cpus = '1,2,3'
         nohz_full = '2'
         rcu_no_cbs = '1,2,4-5'
 
         self.cli_set(['system', 'option', 'kernel', 'cpu', 'disable-nmi-watchdog'])
-        self.cli_set(['system', 'option', 'kernel', 'cpu', 'isolate-cpus', isolate_cpus])
         self.cli_set(['system', 'option', 'kernel', 'cpu', 'nohz-full', nohz_full])
         self.cli_set(['system', 'option', 'kernel', 'cpu', 'rcu-no-cbs', rcu_no_cbs])
         self.cli_set(['system', 'option', 'kernel', 'disable-hpet'])
@@ -138,7 +136,6 @@ class TestSystemOption(VyOSUnitTestSHIM.TestCase):
         self.assertIn(' hpet=disable', tmp)
         self.assertIn(' mce=off', tmp)
         self.assertIn(' nosoftlockup', tmp)
-        self.assertIn(f' isolcpus={isolate_cpus}', tmp)
         self.assertIn(f' nohz_full={nohz_full}', tmp)
         self.assertIn(f' rcu_nocbs={rcu_no_cbs}', tmp)
         self.assertIn(' numa_balancing=disable', tmp)
diff --git a/smoketest/scripts/cli/test_vpp.py b/smoketest/scripts/cli/test_vpp.py
index 59dbb2272..a506ad4f0 100755
--- a/smoketest/scripts/cli/test_vpp.py
+++ b/smoketest/scripts/cli/test_vpp.py
@@ -26,7 +26,8 @@ from json import loads
 from base_vyostest_shim import VyOSUnitTestSHIM
 
 from vyos.configsession import ConfigSessionError
-from vyos.utils.cpu import get_available_cpus
+from vyos.utils.convert import range_str_to_list
+from vyos.utils.convert import list_to_range_str
 from vyos.utils.process import process_named_running
 from vyos.utils.file import read_file
 from vyos.utils.process import rc_cmd
@@ -86,12 +87,9 @@ def get_address(interface):
             return ip_address
 
 
-def get_vpp_cpu_allocation():
-    reserved_cpus = default_resource_map.get('reserved_cpu_cores')
-    # Get sorted list of available CPU IDs
-    available = sorted({cpu['cpu'] for cpu in get_available_cpus()})
-    main_core = available[reserved_cpus]  # first non-reserved CPU
-    return reserved_cpus, main_core
+def get_isolated_cpus():
+    isolated = read_file('/sys/devices/system/cpu/isolated')
+    return range_str_to_list(isolated)
 
 
 class TestVPP(VyOSUnitTestSHIM.TestCase):
@@ -133,7 +131,7 @@ class TestVPP(VyOSUnitTestSHIM.TestCase):
     def test_01_vpp_basic(self):
         poll_sleep = '0'
         mtu = '2500'
-        skip_cores, main_core = get_vpp_cpu_allocation()
+        isolated_cores = get_isolated_cpus()
 
         self.cli_set(base_path + ['settings', 'poll-sleep-usec', poll_sleep])
 
@@ -142,8 +140,7 @@ class TestVPP(VyOSUnitTestSHIM.TestCase):
 
         config_entries = (
             f'poll-sleep-usec {poll_sleep}',
-            f'skip-cores {skip_cores}',
-            f'main-core {main_core}',
+            f'main-core {str(isolated_cores[0])}',  # first isolated core is set as main-core
             'plugin default { disable }',
             'plugin dpdk_plugin.so { enable }',
             'plugin linux_cp_plugin.so { enable }',
@@ -726,7 +723,9 @@ class TestVPP(VyOSUnitTestSHIM.TestCase):
 
     def test_10_vpp_cpu_cores(self):
         cpu_cores = '2'
-        skip_cores, main_core = get_vpp_cpu_allocation()
+        isolated_cpus = get_isolated_cpus()
+        main_core = str(isolated_cpus[0])  # first isolated core is set as main-core
+        corelist_workers = list_to_range_str(isolated_cpus[1 : int(cpu_cores)])
 
         # verify 'cpu-cores' are set not correctly
         # expect raise ConfigError
@@ -738,9 +737,8 @@ class TestVPP(VyOSUnitTestSHIM.TestCase):
         self.cli_commit()
 
         config_entries = (
-            f'skip-cores {skip_cores}',  # reserved cpus skipped for system use
-            f'main-core {main_core}',  # first available core is set as main-core
-            f'workers {int(cpu_cores) - 1}',
+            f'main-core {main_core}',
+            f'corelist-workers {corelist_workers}',
             'dev 0000:00:00.0',
         )
 
@@ -1218,7 +1216,7 @@ class TestVPP(VyOSUnitTestSHIM.TestCase):
         self.cli_delete(['interfaces', 'ethernet', interface, 'vif-s', vif_s])
         self.cli_commit()
 
-    def test_18_kernel_options_hugepages(self):
+    def test_18_1_kernel_options_hugepages(self):
         default_hp_size = '2M'
         hp_size_1g = '1G'
         hp_size_2m = '2M'
@@ -1251,6 +1249,27 @@ class TestVPP(VyOSUnitTestSHIM.TestCase):
         self.assertIn(f' hugepagesz={hp_size_1g} hugepages={hp_count_1g}', tmp)
         self.assertIn(f' hugepagesz={hp_size_2m} hugepages={hp_count_2m}', tmp)
 
+    def test_18_2_kernel_options_cpu(self):
+        isolate_cpus = '1,2'
+
+        self.cli_set(
+            ['system', 'option', 'kernel', 'cpu', 'isolate-cpus', isolate_cpus]
+        )
+        self.cli_commit()
+
+        # Read GRUB config file for current running image
+        tmp = read_file(
+            f'{image.grub.GRUB_DIR_VYOS_VERS}/{image.get_running_image()}.cfg'
+        )
+        self.assertIn(f' isolcpus={isolate_cpus}', tmp)
+
+        # verify 'isolate-cpus' are set not correctly
+        # expect raise ConfigError
+        self.cli_set(['system', 'option', 'kernel', 'cpu', 'isolate-cpus', '1-99'])
+        with self.assertRaises(ConfigSessionError):
+            self.cli_commit()
+        self.cli_discard()
+
     def test_19_static_arp(self):
         host = '192.0.2.10'
         mac = '00:01:02:03:04:0a'
diff --git a/src/conf_mode/system_option.py b/src/conf_mode/system_option.py
index ac6e7768d..003e8c9f5 100755
--- a/src/conf_mode/system_option.py
+++ b/src/conf_mode/system_option.py
@@ -28,7 +28,10 @@ from vyos.configverify import verify_interface_exists
 from vyos.system import grub_util
 from vyos.template import render
 from vyos.utils.boot import boot_configuration_complete
+from vyos.utils.convert import range_str_to_list
+from vyos.utils.convert import list_to_range_str
 from vyos.utils.cpu import get_cpus
+from vyos.utils.cpu import get_available_cpus
 from vyos.utils.dict import dict_search
 from vyos.utils.file import write_file
 from vyos.utils.file import read_file
@@ -44,6 +47,7 @@ from vyos import ConfigError
 from vyos import airbag
 
 from vyos.vpp.config_resource_checks import memory as mem_check
+from vyos.vpp.config_resource_checks.resource_defaults import default_resource_map
 
 airbag.enable()
 
@@ -239,6 +243,29 @@ def verify(options):
                 f'AMD pstate driver cannot be used with "{cpu_vendor}" CPU!'
             )
 
+        isolate_cpus = dict_search('kernel.cpu.isolate_cpus', options)
+        if isolate_cpus:
+            available_cores = sorted({int(cpu['cpu']) for cpu in get_available_cpus()})
+            cpus_list = range_str_to_list(isolate_cpus)
+            reserved_cpus = default_resource_map.get('reserved_cpu_cores')
+
+            cpus_available = len(available_cores) - reserved_cpus
+            if len(cpus_list) > cpus_available:
+                raise ConfigError(
+                    f'Cannot isolate {len(cpus_list)} CPUs ({isolate_cpus}): '
+                    f'only {cpus_available} of {len(available_cores)} physical cores '
+                    f'are available ({reserved_cpus} reserved for the system)'
+                )
+
+            not_available = [cpu for cpu in cpus_list if cpu not in available_cores]
+            if not_available:
+                not_available_str = list_to_range_str(not_available)
+                available_str = list_to_range_str(available_cores)
+                raise ConfigError(
+                    f'CPU(s) {not_available_str} do not exist on this system. '
+                    f'Available CPUs: {available_str}'
+                )
+
         _, hp_memory_bytes = _get_total_hugepages_and_memory(
             options['kernel'].get('memory', {})
         )
diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py
index 33a4023af..342d58fca 100755
--- a/src/conf_mode/vpp.py
+++ b/src/conf_mode/vpp.py
@@ -38,8 +38,10 @@ from vyos.ifconfig import Section
 from vyos.logger import getLogger
 from vyos.template import render
 from vyos.utils.boot import boot_configuration_complete
-from vyos.utils.cpu import get_available_cpus
+from vyos.utils.convert import range_str_to_list
+from vyos.utils.convert import list_to_range_str
 from vyos.utils.dict import dict_search
+from vyos.utils.file import read_file
 from vyos.utils.kernel import check_kmod
 from vyos.utils.kernel import unload_kmod
 from vyos.utils.kernel import list_loaded_modules
@@ -62,7 +64,6 @@ from vyos.vpp.config_verify import (
     verify_vpp_buffers,
 )
 from vyos.vpp.config_resource_checks import memory
-from vyos.vpp.config_resource_checks.resource_defaults import default_resource_map
 from vyos.vpp.config_filter import iface_filter_eth
 from vyos.vpp.utils import EthtoolGDrvinfo
 from vyos.vpp.utils import cli_ethernet_with_vifs_ifaces
@@ -168,21 +169,27 @@ def _unload_module(module_name: str):
 
 
 def _configure_vpp_cpu_settings(config: dict):
-    """Configure VPP CPU settings: main-core, workers and skip-cores based on 'cpu-cores'"""
+    """Configure VPP CPU settings: main-core and corelist-workers based on 'cpu-cores'.
+
+    Reads the actually-isolated CPUs from the running kernel
+    (/sys/devices/system/cpu/isolated) and assigns:
+      - main_core: the first isolated CPU (index 0)
+      - corelist_workers: the next (cpu_cores - 1) isolated CPUs
+    """
     cpu_cores = int(config['settings']['resource_allocation']['cpu_cores'])
-    reserved_cpus = default_resource_map.get('reserved_cpu_cores')
+    # Use the system's actual isolated CPUs, not config values which may
+    # require a reboot to take effect
+    isolated = read_file('/sys/devices/system/cpu/isolated')
+    cpus_isolated = range_str_to_list(isolated)
 
-    # Get sorted list of available CPU IDs
-    available = sorted({cpu['cpu'] for cpu in get_available_cpus()})
+    if cpu_cores <= len(cpus_isolated):
+        # First isolated CPU is the VPP main thread; remaining are workers
+        config['settings']['cpu'] = {'main_core': str(cpus_isolated[0])}
 
-    if reserved_cpus < len(available):
-        main_core = available[reserved_cpus]  # first non-reserved CPU
-        config['settings']['cpu'] = {
-            'main_core': str(main_core),
-            'skip_cores': str(reserved_cpus),
-        }
         if cpu_cores > 1:
-            config['settings']['cpu']['workers'] = str(cpu_cores - 1)
+            config['settings']['cpu']['corelist_workers'] = list_to_range_str(
+                cpus_isolated[1:cpu_cores]
+            )
 
 
 def _normalize_buffers(config: dict):
diff --git a/src/tests/test_utils.py b/src/tests/test_utils.py
index bbf1cb523..5022b24f6 100644
--- a/src/tests/test_utils.py
+++ b/src/tests/test_utils.py
@@ -48,3 +48,41 @@ class TestVyOSUtils(TestCase):
         self.assertEqual(list_strip(lst, rsb, right=True), ['a', 'b', 'c'])
         self.assertEqual(list_strip(lst, non), [])
         self.assertEqual(list_strip(sub, lst), [])
+
+    def test_range_str_to_list(self):
+        from vyos.utils.convert import range_str_to_list
+
+        # basic cases
+        self.assertEqual(range_str_to_list('1-3'), [1, 2, 3])
+        self.assertEqual(range_str_to_list('1-3,5,7-8'), [1, 2, 3, 5, 7, 8])
+        self.assertEqual(range_str_to_list('3'), [3])
+        # empty string
+        self.assertEqual(range_str_to_list(''), [])
+        # unordered input
+        self.assertEqual(range_str_to_list('5,1-3,4'), [1, 2, 3, 4, 5])
+        self.assertEqual(range_str_to_list('7-9,1-3'), [1, 2, 3, 7, 8, 9])
+        # overlapping ranges
+        self.assertEqual(range_str_to_list('1-5,3-7'), [1, 2, 3, 4, 5, 6, 7])
+        self.assertEqual(range_str_to_list('1-3,2-4,3-5'), [1, 2, 3, 4, 5])
+        # duplicated values
+        self.assertEqual(range_str_to_list('1,1,2,2,3'), [1, 2, 3])
+        self.assertEqual(range_str_to_list('5,1-3,2,3'), [1, 2, 3, 5])
+        # adjacent ranges
+        self.assertEqual(range_str_to_list('1-3,4-6'), [1, 2, 3, 4, 5, 6])
+
+    def test_list_to_range_str(self):
+        from vyos.utils.convert import list_to_range_str
+
+        # basic cases
+        self.assertEqual(list_to_range_str([1, 2, 3]), '1-3')
+        self.assertEqual(list_to_range_str([1, 2, 3, 5, 7, 8]), '1-3,5,7-8')
+        self.assertEqual(list_to_range_str([1, 3]), '1,3')
+        self.assertEqual(list_to_range_str([3]), '3')
+        # empty list
+        self.assertEqual(list_to_range_str([]), '')
+        # unordered input
+        self.assertEqual(list_to_range_str([5, 1, 2, 3, 4]), '1-5')
+        self.assertEqual(list_to_range_str([7, 8, 9, 1, 2, 3]), '1-3,7-9')
+        # duplicated values
+        self.assertEqual(list_to_range_str([1, 1, 2, 2, 3, 3]), '1-3')
+        self.assertEqual(list_to_range_str([5, 1, 2, 2, 3, 5]), '1-3,5')