From 5c6e25f50ddac493ab6c61726fb4a1f96b9bf55c Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Mon, 1 Sep 2025 14:08:21 +0300 Subject: T7757: VPP Verify buffers for rx/tx queues --- python/vyos/vpp/config_resource_checks/memory.py | 34 ++++++++++++++++++++++++ python/vyos/vpp/config_verify.py | 12 +++++++++ 2 files changed, 46 insertions(+) (limited to 'python') diff --git a/python/vyos/vpp/config_resource_checks/memory.py b/python/vyos/vpp/config_resource_checks/memory.py index 9391e21e1..8b201ef87 100644 --- a/python/vyos/vpp/config_resource_checks/memory.py +++ b/python/vyos/vpp/config_resource_checks/memory.py @@ -28,6 +28,10 @@ from vyos.vpp.utils import ( from vyos.vpp.config_resource_checks.resource_defaults import default_resource_map +# VPP buffers default per NUMA node +MIN_BUFFERS = 16_384 + + def classify_page_size(page_size_bytes: int) -> str: """ Returns one of: '4K', '2M', '1G' based on page size. @@ -187,3 +191,33 @@ def total_memory_required(settings: dict) -> dict: memory[classify_page_size(page_size)] += memory_size return memory + + +def buffers_required(settings: dict, workers) -> int: + """ + Calculate total VPP buffer requirements based on interface settings and workers. + """ + buffers_total = 0 + for ifname, iface_config in settings.get('interface', {}).items(): + if iface_config.get('driver') == 'xdp': + continue + dpdk_options = iface_config.get('dpdk_options', {}) + rx_queues = int(dpdk_options.get('num_rx_queues', 1)) + rx_desc = int(dpdk_options.get('num_rx_desc')) + tx_queues = int(dpdk_options.get('num_tx_queues', workers + 1)) + tx_desc = int(dpdk_options.get('num_tx_desc')) + + # buffers for RX/TX queues for interface + buffers_total += rx_queues * rx_desc + tx_queues * tx_desc + + # per-thread buffer caches (approx. 256 buffers per worker) + buffers_total += workers * 256 + + # Safety margin for buffer calculations: + # traffic bursts, alignment/metadata overhead etc. + # The factor 2.5 is derived from VPP’s own calculations: + # https://github.com/FDio/vpp/blob/stable/2506/extras/vpp_config/vpplib/AutoConfig.py#L609 + buffers_total = int(buffers_total * 2.5) + + # Enforce minimum required by VPP (16K buffers) + return max(buffers_total, MIN_BUFFERS) diff --git a/python/vyos/vpp/config_verify.py b/python/vyos/vpp/config_verify.py index cf0b923bb..78c5d420b 100644 --- a/python/vyos/vpp/config_verify.py +++ b/python/vyos/vpp/config_verify.py @@ -442,3 +442,15 @@ def verify_routes_count(settings: dict, workers: int): 'Extensive use of features like ACLs, NAT and others may reduce the numbers above. ' 'Please read the documentation for details: https://docs.vyos.io/' ) + + +def verify_vpp_buffers(settings: dict, workers: int): + buffers_configured = int(settings['buffers']['buffers_per_numa']) + + buffers_required = mem_checks.buffers_required(settings, workers) + + if buffers_required > buffers_configured: + raise ConfigError( + 'Not enough buffers to initialize RX/TX queues for interfaces. ' + f'Set "vpp settings buffers buffers-per-numa" to {buffers_required} or higher' + ) -- cgit v1.2.3 From b0ae930a9fcb3093d3069bb8296a2f572e6a3d93 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Fri, 5 Sep 2025 11:37:55 +0300 Subject: T7757: VPP add auto calculation of buffer-per-numa parameter as default --- interface-definitions/vpp.xml.in | 11 +++++-- python/vyos/vpp/config_resource_checks/memory.py | 7 ++++ python/vyos/vpp/config_verify.py | 8 ++--- src/conf_mode/vpp.py | 41 ++++++++++++++++++++---- 4 files changed, 51 insertions(+), 16 deletions(-) (limited to 'python') diff --git a/interface-definitions/vpp.xml.in b/interface-definitions/vpp.xml.in index 356345f84..e7a21ecf1 100644 --- a/interface-definitions/vpp.xml.in +++ b/interface-definitions/vpp.xml.in @@ -360,14 +360,19 @@ Number of buffers per numa node - u32:0-4294967295 + u32:16384-4294967295 Number of buffers + + auto + Auto calculate number of buffers per numa node + - + + (auto) - 16384 + auto diff --git a/python/vyos/vpp/config_resource_checks/memory.py b/python/vyos/vpp/config_resource_checks/memory.py index 8b201ef87..ea4697ba2 100644 --- a/python/vyos/vpp/config_resource_checks/memory.py +++ b/python/vyos/vpp/config_resource_checks/memory.py @@ -199,11 +199,18 @@ def buffers_required(settings: dict, workers) -> int: """ buffers_total = 0 for ifname, iface_config in settings.get('interface', {}).items(): + # Do not include XDP interfaces in buffer calculations. + # Unlike DPDK, XDP does not use VPP-managed mbufs for RX/TX rings, + # so buffer requirements cannot be derived from descriptors here. + # Buffers for XDP are handled internally by the kernel/XDP layer, + # not by VPP’s buffer allocator. if iface_config.get('driver') == 'xdp': continue dpdk_options = iface_config.get('dpdk_options', {}) rx_queues = int(dpdk_options.get('num_rx_queues', 1)) rx_desc = int(dpdk_options.get('num_rx_desc')) + # default TX queues is equal to number of worker threads + # plus 1 main thread tx_queues = int(dpdk_options.get('num_tx_queues', workers + 1)) tx_desc = int(dpdk_options.get('num_tx_desc')) diff --git a/python/vyos/vpp/config_verify.py b/python/vyos/vpp/config_verify.py index 78c5d420b..71cf97423 100644 --- a/python/vyos/vpp/config_verify.py +++ b/python/vyos/vpp/config_verify.py @@ -317,7 +317,7 @@ def verify_vpp_cpu_main_core(cpu_settings: dict) -> None: ) -def verify_vpp_settings_cpu_workers(cpu_settings: dict) -> int: +def verify_vpp_settings_cpu_workers(cpu_settings: dict): """ Verify that the system has enough available CPU cores to run a given amount of worker processes (1 worker/core) @@ -331,10 +331,8 @@ def verify_vpp_settings_cpu_workers(cpu_settings: dict) -> int: f'(reduce to {available_cores} or less)' ) - return workers - -def verify_vpp_settings_cpu_corelist_workers(cpu_settings: dict) -> int: +def verify_vpp_settings_cpu_corelist_workers(cpu_settings: dict): """ Verify that the CPU cores provided to the config are free and can be used by VPP """ @@ -366,8 +364,6 @@ def verify_vpp_settings_cpu_corelist_workers(cpu_settings: dict) -> int: if len(all_core_nums) > cpu_checks.available_cores_count(cpu_settings): raise ConfigError(f'{error_msg}: Not enough free CPUs in the system.') - return len(all_core_nums) - def verify_vpp_nat44_workers(workers: int, nat44_workers: list): if workers < 1: diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index b205291f2..518860605 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -60,6 +60,7 @@ from vyos.vpp.config_verify import ( verify_vpp_main_heap_size, verify_vpp_buffers, ) +from vyos.vpp.config_resource_checks import memory from vyos.vpp.config_filter import iface_filter_eth from vyos.vpp.utils import EthtoolGDrvinfo from vyos.vpp.configdb import JSONStorage @@ -148,6 +149,28 @@ def _unload_module(module_name: str): raise +def _get_workers_count(cpu_settings: dict) -> int: + if 'corelist_workers' in cpu_settings: + corelist_workers = [] + for worker_range in cpu_settings['corelist_workers']: + core_numbers = worker_range.split('-') + corelist_workers.extend( + range(int(core_numbers[0]), int(core_numbers[-1]) + 1) + ) + + return len(corelist_workers) + + return int(cpu_settings.get('workers', 0)) + + +def _normalize_buffers(config: dict): + """Replace 'auto' buffers_per_numa with calculated value""" + if config['settings']['buffers']['buffers_per_numa'] == 'auto': + workers = _get_workers_count(config['settings'].get('cpu', {})) + buffers = memory.buffers_required(config['settings'], workers) + config['settings']['buffers']['buffers_per_numa'] = str(buffers) + + def get_config(config=None): # use persistent config to store interfaces data between executions # this is required because some interfaces after they are connected @@ -242,9 +265,10 @@ def get_config(config=None): default_values_effective = conf.get_config_defaults( **effective_config.kwargs, recursive=True ) - config['effective'] = config_dict_merge( - default_values_effective, effective_config - ) + effective_config = config_dict_merge(default_values_effective, effective_config) + # Buffer normalization (auto → computed) + _normalize_buffers(effective_config) + config['effective'] = effective_config if 'settings' in config: if 'interface' in config['settings']: @@ -317,6 +341,9 @@ def get_config(config=None): xdp_api_params['flags'] = 'no_syscall_lock' iface_config['xdp_api_params'] = xdp_api_params + # Buffer normalization (auto → computed) + _normalize_buffers(config) + if removed_ifaces: config['removed_ifaces'] = removed_ifaces config['xconn_members'] = xconn_members @@ -373,8 +400,6 @@ def verify(config): raise ConfigError(f'Interface {iface} does not exist or is not Ethernet!') # Resource usage checks - workers = 0 - if 'cpu' in config['settings']: cpu_settings = config['settings']['cpu'] @@ -388,14 +413,16 @@ def verify(config): # Check if there are enough CPU cores to add workers if 'workers' in cpu_settings: - workers = verify_vpp_settings_cpu_workers(cpu_settings) + verify_vpp_settings_cpu_workers(cpu_settings) if 'main_core' in cpu_settings: verify_vpp_cpu_main_core(cpu_settings) # Check the CPU main core not falling to the corelist-workers if 'corelist_workers' in cpu_settings: - workers = verify_vpp_settings_cpu_corelist_workers(cpu_settings) + verify_vpp_settings_cpu_corelist_workers(cpu_settings) + + workers = _get_workers_count(config['settings'].get('cpu', {})) if 'workers' in config['settings']['nat44']: verify_vpp_nat44_workers( -- cgit v1.2.3