#!/usr/bin/env python3 # # Copyright (C) 2023-2024 VyOS Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. from pathlib import Path from psutil import virtual_memory from pyroute2 import IPRoute from vyos import ConfigError from vyos import airbag from vyos.base import Warning from vyos.config import Config from vyos.configdep import set_dependents, call_dependents from vyos.configdict import node_changed, leaf_node_changed from vyos.utils.cpu import get_core_count from vyos.ifconfig import Section from vyos.template import render from vyos.utils.boot import boot_configuration_complete from vyos.utils.process import call from vyos.utils.system import sysctl_read, sysctl_apply from vyos.vpp import VPPControl from vyos.vpp import control_host from vyos.vpp.config_deps import deps_xconnect_dict from vyos.vpp.config_verify import verify_dev_driver from vyos.vpp.config_filter import iface_filter_eth from vyos.vpp.utils import EthtoolGDrvinfo from vyos.vpp.configdb import JSONStorage airbag.enable() service_name = 'vpp' service_conf = Path(f'/run/vpp/{service_name}.conf') systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf' dependency_interface_type_map = { 'vpp_interfaces_bonding': 'bonding', 'vpp_interfaces_bridge': 'bridge', 'vpp_interfaces_ethernet': 'ethernet', 'vpp_interfaces_geneve': 'geneve', 'vpp_interfaces_gre': 'gre', 'vpp_interfaces_ipip': 'ipip', 'vpp_interfaces_loopback': 'loopback', 'vpp_interfaces_vxlan': 'vxlan', 'vpp_interfaces_xconnect': 'xconnect', } # dict of drivers that needs to be overrided override_drivers: dict[str, str] = { 'hv_netvsc': 'uio_hv_generic', 'ena': 'vfio-pci', } # drivers that does not use PCIe addresses not_pci_drv: list[str] = ['hv_netvsc'] def get_config(config=None): # use persistent config to store interfaces data between executions # this is required because some interfaces after they are connected # to VPP is really hard or impossible to restore without knowing # their original parameters (like IDs) persist_config = JSONStorage('vpp_conf') eth_ifaces_persist: dict[str, dict[str, str]] = persist_config.read( 'eth_ifaces', {} ) if config: conf = config else: conf = Config() base = ['vpp'] base_settings = ['vpp', 'settings'] # find interfaces removed from VPP effective_config = conf.get_config_dict( base, key_mangling=('-', '_'), effective=True, get_first_key=True, no_tag_node_value_mangle=True, ) xconn_members = deps_xconnect_dict(conf) removed_ifaces = [] tmp = node_changed(conf, base_settings + ['interface']) if tmp: for removed_iface in tmp: to_append = { 'iface_name': removed_iface, 'driver': effective_config['settings']['interface'][removed_iface][ 'driver' ], } removed_ifaces.append(to_append) # add an interface to a list of interfaces that need # to be reinitialized after the commit set_dependents('ethernet', conf, removed_iface) if not conf.exists(base): return { 'removed_ifaces': removed_ifaces, 'xconn_members': xconn_members, 'persist_config': eth_ifaces_persist, } config = conf.get_config_dict( base, get_first_key=True, key_mangling=('-', '_'), no_tag_node_value_mangle=True, with_recursive_defaults=True, ) # add running config if effective_config: config['effective'] = effective_config if 'settings' in config: if 'interface' in config['settings']: for iface, iface_config in config['settings']['interface'].items(): # Driver must be configured to continue if 'driver' not in iface_config: raise ConfigError( f'"driver" must be configured for {iface} interface!' ) old_driver = leaf_node_changed( conf, base_settings + ['interface', iface, 'driver'] ) # Get current kernel module, required for extra verification and # logic for VMBus interfaces config['settings']['interface'][iface]['kernel_module'] = ( EthtoolGDrvinfo(iface).driver ) # filter unsupported config nodes iface_filter_eth(conf, iface) set_dependents('ethernet', conf, iface) # Interfaces with changed driver should be removed/readded if old_driver and old_driver[0] == 'dpdk': removed_ifaces.append( { 'iface_name': iface, 'driver': 'dpdk', } ) # Get PCI address or device ID if iface_config['driver'] == 'dpdk': if 'dpdk_options' not in iface_config: iface_config['dpdk_options'] = {} # Check in a persistent config first id_from_persisten_conf = eth_ifaces_persist.get(iface, {}).get( 'dev_id' ) if id_from_persisten_conf: iface_config['dpdk_options']['dev_id'] = id_from_persisten_conf else: try: iface_to_search = iface if old_driver and old_driver[0] == 'xdp': iface_to_search = f'defunct_{iface}' iface_config['dpdk_options']['dev_id'] = ( control_host.get_dev_id(iface_to_search) ) except Exception: # Return empty address if all attempts failed # We will catch this in verify() iface_config['dpdk_options']['dev_id'] = '' # prepare XDP interface parameters if iface_config['driver'] == 'xdp': xdp_api_params = { 'rxq_size': int(iface_config['xdp_options']['rx_queue_size']), 'txq_size': int(iface_config['xdp_options']['tx_queue_size']), } if iface_config['xdp_options']['num_rx_queues'] == 'all': xdp_api_params['rxq_num'] = 0 else: xdp_api_params['rxq_num'] = int( iface_config['xdp_options']['num_rx_queues'] ) if 'zero-copy' in iface_config['xdp_options']: xdp_api_params['mode'] = 'zero-copy' if 'zero-copy' in iface_config['xdp_options']: xdp_api_params['flags'] = 'no_syscall_lock' iface_config['xdp_api_params'] = xdp_api_params if removed_ifaces: config['removed_ifaces'] = removed_ifaces config['xconn_members'] = xconn_members # Dependencies for dependency, interface_type in dependency_interface_type_map.items(): # if conf.exists(base + ['interfaces', interface_type]): if effective_config.get('interfaces', {}).get(interface_type): for iface, iface_config in ( config.get('interfaces', {}).get(interface_type, {}).items() ): # filter unsupported config nodes if interface_type == 'ethernet': iface_filter_eth(conf, iface) set_dependents(dependency, conf, iface) # Save important info about all interfaces that cannot be retrieved later # Add new interfaces (only if they are first time seen in a config) for iface, iface_config in config.get('settings', {}).get('interface', {}).items(): if iface not in effective_config.get('settings', {}).get('interface', {}): eth_ifaces_persist[iface] = { 'original_driver': config['settings']['interface'][iface][ 'kernel_module' ], } eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface) eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface) # Return to config dictionary config['persist_config'] = eth_ifaces_persist return config def convert_to_int(val): rates = { 'K': 1024, 'M': 1024**2, 'G': 1024**3, } try: return int(val) except ValueError: return int(val[:-1]) * rates[val[-1]] def verify_memory(settings): memory_available: int = virtual_memory().available cpus: int = get_core_count() nr_hugepages = int(settings['host_resources']['nr_hugepages']) hugepages_memory = nr_hugepages * 2 * 1024**2 memory_required = hugepages_memory buffers_per_numa = int(settings.get('buffers', {}).get('buffers_per_numa', 16384)) data_size = int(settings.get('buffers', {}).get('data_size', 2048)) buffers_memory = buffers_per_numa * data_size * cpus memory_required += buffers_memory netlink_buffer_size = int( settings.get('lcp', {}).get('netlink', {}).get('rx_buffer_size', 212992) ) memory_required += netlink_buffer_size memory_main_heap = convert_to_int( settings.get('memory', {}).get('main_heap_size', '1G') ) memory_required += memory_main_heap statseg_size = convert_to_int(settings.get('statseg', {}).get('size', '96M')) memory_required += statseg_size if memory_available < memory_required: raise ConfigError( 'Not enough free memory to start VPP:\n' f'available: {round(memory_available / 1024 ** 3, 1)}GB\n' f'required: {round(memory_required / 1024 ** 3, 1)}GB' ) def verify(config): # bail out early - looks like removal from running config if not config or ('removed_ifaces' in config and 'settings' not in config): return None if 'settings' not in config: raise ConfigError('"settings interface" is required but not set!') # CPU main-core must be not included to corelist-workers if config.get('settings').get('cpu', {}).get('main_core') and config.get( 'settings' ).get('cpu', {}).get('corelist_workers'): corelist_workers = config['settings']['cpu']['corelist_workers'] main_core = int(config['settings']['cpu']['main_core']) all_core_numbers = [] for worker_range in corelist_workers: core_numbers = worker_range.split('-') all_core_numbers.extend( range(int(core_numbers[0]), int(core_numbers[-1]) + 1) ) if main_core in all_core_numbers: raise ConfigError( f'"cpu main-core {main_core}" must not be included in the corelist-workers!' ) if 'interface' not in config['settings']: raise ConfigError('"settings interface" is required but not set!') # check if Ethernet interfaces exist ethernet_ifaces = Section.interfaces('ethernet') for iface in config['settings']['interface'].keys(): if iface not in ethernet_ifaces: raise ConfigError(f'Interface {iface} does not exist or is not Ethernet!') # ensure DPDK/XDP settings are properly configured for iface, iface_config in config['settings']['interface'].items(): # check if selected driver is supported, but only for new interfaces if iface not in config.get('effective', {}).get('settings', {}).get( 'interface', {} ): if not verify_dev_driver(iface, iface_config['driver']): raise ConfigError( f'Driver {iface_config["driver"]} is not compatible with interface {iface}!' ) if iface_config['driver'] == 'xdp' and 'xdp_options' in iface_config: if iface_config['xdp_options']['num_rx_queues'] != 'all': Warning(f'Not all RX queues will be connected to VPP for {iface}!') # check GRE tunnels as part of the bridge, only tunnel-type teb is allowed # set vpp interfaces bridge br1 member interface gre1 # set vpp interfaces gre gre1 tunnel-type teb if 'interfaces' in config: if 'bridge' in config['interfaces']: for iface, iface_config in config['interfaces']['bridge'].items(): if 'member' in iface_config: for member in iface_config['member'].get('interface', []): if member.startswith('gre'): if ( 'gre' in config['interfaces'] and config['interfaces']['gre'] .get(member, {}) .get('tunnel_type') != 'teb' ): raise ConfigError( f'Only tunnel-type teb is allowed for GRE interfaces in bridge {iface}' ) # Only one multipoint GRE tunnel is allowed from the same source address # set vpp interfaces gre gre0 mode 'point-to-multipoint' # set vpp interfaces gre gre0 remote '0.0.0.0' # set vpp interfaces gre gre0 source-address '192.0.2.1' # set vpp interfaces gre gre1 mode 'point-to-multipoint' # set vpp interfaces gre gre1 remote '0.0.0.0' # set vpp interfaces gre gre1 source-address '192.0.2.1' if 'gre' in config['interfaces']: for iface, iface_config in config['interfaces']['gre'].items(): if iface_config['mode'] == 'point-to-multipoint': for other_iface, other_iface_config in config['interfaces'][ 'gre' ].items(): if ( other_iface_config['mode'] == 'point-to-multipoint' and other_iface_config['source_address'] == iface_config['source_address'] and iface != other_iface ): raise ConfigError( 'Only one multipoint GRE tunnel is allowed from the same source address' ) if 'cpu' in config['settings']: if ( 'corelist_workers' in config['settings']['cpu'] and 'main_core' not in config['settings']['cpu'] ): raise ConfigError('"cpu main-core" is required but not set!') verify_memory(config['settings']) # Check if deleted interfaces are not xconnect memebrs for iface_config in config.get('removed_ifaces', []): if iface_config['iface_name'] in config.get('xconn_members', {}): raise ConfigError( f'Interface {iface_config["iface_name"]} is an xconnect member and cannot be removed' ) def generate(config): if not config or ('removed_ifaces' in config and 'settings' not in config): # Remove old config and return service_conf.unlink(missing_ok=True) return None render(service_conf, 'vpp/startup.conf.j2', config['settings']) render(systemd_override, 'vpp/override.conf.j2', config) # apply sysctl values # default: https://github.com/FDio/vpp/blob/v23.10/src/vpp/conf/80-vpp.conf sysctl_config: dict[str, str] = { 'vm.nr_hugepages': config['settings']['host_resources']['nr_hugepages'], 'vm.max_map_count': config['settings']['host_resources']['max_map_count'], 'vm.hugetlb_shm_group': '0', 'kernel.shmmax': config['settings']['host_resources']['shmmax'], } # we do not want to lower current values for sysctl_key, sysctl_value in sysctl_config.items(): # perform check only for quantitative params if sysctl_key == 'vm.hugetlb_shm_group': pass current_value = sysctl_read(sysctl_key) if int(current_value) > int(sysctl_value): sysctl_config[sysctl_key] = current_value if not sysctl_apply(sysctl_config): raise ConfigError('Cannot configure sysctl parameters for VPP') return None def apply(config): # Open persistent config # It is required for operations with interfaces persist_config = JSONStorage('vpp_conf') if not config or ('removed_ifaces' in config and 'settings' not in config): # Cleanup persistent config persist_config.delete() # And stop the service call(f'systemctl stop {service_name}.service') else: # Some interfaces required extra preparation before VPP can be started if 'settings' in config and 'interface' in config.get('settings'): for iface, iface_config in config['settings']['interface'].items(): if iface_config['driver'] == 'dpdk': # ena interfaces require noiommu mode if iface_config['kernel_module'] == 'ena': control_host.unsafe_noiommu_mode(True) if iface_config['kernel_module'] in override_drivers: control_host.override_driver( config['persist_config'][iface]['bus_id'], config['persist_config'][iface]['dev_id'], override_drivers[iface_config['kernel_module']], ) call('systemctl daemon-reload') call(f'systemctl restart {service_name}.service') # Initialize interfaces removed from VPP for iface in config.get('removed_ifaces', []): # DPDK - rescan PCI to use a proper driver if ( iface['driver'] == 'dpdk' and config['persist_config'][iface['iface_name']]['original_driver'] not in not_pci_drv ): control_host.pci_rescan( config['persist_config'][iface['iface_name']]['dev_id'] ) # rename to the proper name iface_new_name: str = control_host.get_eth_name( config['persist_config'][iface['iface_name']]['dev_id'] ) control_host.rename_iface(iface_new_name, iface['iface_name']) # XDP - rename an interface , disable promisc and XDP if iface['driver'] == 'xdp': control_host.set_promisc(f'defunct_{iface["iface_name"]}', 'off') control_host.rename_iface( f'defunct_{iface["iface_name"]}', iface['iface_name'] ) control_host.xdp_remove(iface['iface_name']) # Rename Mellanox NIC to a normal name try: if ( control_host.get_eth_driver(f'defunct_{iface["iface_name"]}') == 'mlx5_core' ): control_host.rename_iface( f'defunct_{iface["iface_name"]}', iface['iface_name'] ) except FileNotFoundError: pass # Replace a driver with original for VMBus interfaces and rename it if ( iface['driver'] == 'dpdk' and config['persist_config'][iface['iface_name']]['original_driver'] in override_drivers ): control_host.override_driver( config['persist_config'][iface['iface_name']]['bus_id'], config['persist_config'][iface['iface_name']]['dev_id'], ) iface_new_name: str = control_host.get_eth_name( config['persist_config'][iface['iface_name']]['dev_id'] ) control_host.rename_iface(iface_new_name, iface['iface_name']) # Remove what is not in the config anymore if iface['iface_name'] not in config.get('settings', {}).get('interface', {}): del config['persist_config'][iface['iface_name']] if 'settings' in config and 'interface' in config.get('settings'): # connect to VPP # must be performed multiple attempts because API is not available # immediately after the service restart vpp_control = VPPControl(attempts=20, interval=500) # preconfigure LCP plugin if 'ignore_kernel_routes' in config.get('settings', {}).get('lcp', {}): vpp_control.cli_cmd('lcp param route-no-paths off') else: vpp_control.cli_cmd('lcp param route-no-paths on') # add interfaces iproute = IPRoute() for iface, iface_config in config['settings']['interface'].items(): # promisc option for DPDK interfaces if iface_config['driver'] == 'dpdk': if 'promisc' in iface_config['dpdk_options']: if_index = vpp_control.get_sw_if_index(iface) vpp_control.api.sw_interface_set_promisc( sw_if_index=if_index, promisc_on=True ) # add XDP interfaces if iface_config['driver'] == 'xdp': control_host.rename_iface(iface, f'defunct_{iface}') vpp_control.xdp_iface_create( host_if=f'defunct_{iface}', name=iface, **iface_config['xdp_api_params'], ) # replicate MAC address of a real interface real_mac = control_host.get_eth_mac(f'defunct_{iface}') vpp_control.set_iface_mac(iface, real_mac) if 'promisc' in iface_config['xdp_options']: control_host.set_promisc(f'defunct_{iface}', 'on') control_host.set_status(f'defunct_{iface}', 'up') # Rename Mellanox interfaces to hide them and create LCP properly if ( iface in Section.interfaces() and control_host.get_eth_driver(iface) == 'mlx5_core' ): control_host.rename_iface(iface, f'defunct_{iface}') control_host.set_status(f'defunct_{iface}', 'up') # Create lcp if iface not in Section.interfaces(): vpp_control.lcp_pair_add(iface, iface) # Set rx-mode rx_mode = iface_config.get('rx_mode') if rx_mode: # to hardware side vpp_control.iface_rxmode(iface, rx_mode) # to kernel side lcp_name = vpp_control.lcp_pair_find(vpp_name_hw=iface).get( 'vpp_name_kernel' ) vpp_control.iface_rxmode(lcp_name, rx_mode) # For unknown reasons, if multiple interfaces later try to be # initialized by configuration scripts, some of them may stuck # in an endless UP/DOWN loop # We found two workarounds - pause initialization (requires # main code modifications). # And this one dev_index = iproute.link_lookup(ifname=iface)[0] iproute.link('set', index=dev_index, state='up') # Syncronize routes via LCP vpp_control.lcp_resync() # Save persistent config if 'persist_config' in config and config['persist_config']: persist_config.write('eth_ifaces', config['persist_config']) # reinitialize interfaces, but not during the first boot if boot_configuration_complete(): call_dependents() if __name__ == '__main__': try: c = get_config() verify(c) generate(c) apply(c) except ConfigError as e: print(e) exit(1)