From 8f402c2ba47ed3ccbf94f9f037ec6e18d6b975ea Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 26 Jun 2023 11:24:01 +0000 Subject: T1797: Add initial vpp configuration Add initial configuration mode for VPP (PoC) set vpp cpu corelist-workers '2' set vpp cpu main-core '1' set vpp interface eth1 num-rx-desc '256' set vpp interface eth1 num-rx-queues '512' set vpp interface eth1 num-tx-desc '256' set vpp interface eth1 num-tx-queues '512' set vpp interface eth1 pci '0000:02:00.0' set vpp interface eth1 rx-mode 'polling' set vpp interface eth2 pci '0000:08:00.0' Limitation: - 'set vpp interface ethX pci auto' works only per first commit, then interface detached from default stack and creates tun interface 'ethX' to communicate with default stack. In this case we can't get PCI address via ethtool for 'tun' interfaces. But we can set pci address manualy. - Interface sync between default stack and VPP-DPDK stack After vpp change it doesn't trigger iproute2 for changes (should be written later) I.e. if we change something in vpp per each commit it restarts vpp.service it gets empty interface config as we don't configure vpp directly and it should be configured via iproute2 But then if we do any change on interface (for example description) it gets IP address, MTU, state, etc. --- src/conf_mode/vpp.py | 145 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100755 src/conf_mode/vpp.py (limited to 'src') diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py new file mode 100755 index 000000000..aa6c14e89 --- /dev/null +++ b/src/conf_mode/vpp.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2023 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from pathlib import Path + +from vyos.config import Config +from vyos.configdict import dict_merge +from vyos.ifconfig import Section +from vyos.ifconfig import EthernetIf +from vyos.ifconfig import interface +from vyos.util import call +from vyos.util import rc_cmd +from vyos.template import render +from vyos.xml import defaults + +from vyos import ConfigError +from vyos import vpp +from vyos import airbag + +airbag.enable() + +service_name = 'vpp' +service_conf = Path(f'/run/vpp/{service_name}.conf') +systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf' + + +def _get_pci_address_by_interface(iface): + from vyos.util import rc_cmd + rc, out = rc_cmd(f'ethtool -i {iface}') + if rc == 0: + output_lines = out.split('\n') + for line in output_lines: + if 'bus-info' in line: + return line.split(None, 1)[1].strip() + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['vpp'] + base_ethernet = ['interfaces', 'ethernet'] + if not conf.exists(base): + return None + + config = conf.get_config_dict(base, + get_first_key=True, + key_mangling=('-', '_'), + no_tag_node_value_mangle=True) + + # We have gathered the dict representation of the CLI, but there are default + # options which we need to update into the dictionary retrived. + default_values = defaults(base) + if 'interface' in default_values: + del default_values['interface'] + config = dict_merge(default_values, config) + + if 'interface' in config: + for iface, iface_config in config['interface'].items(): + default_values_iface = defaults(base + ['interface']) + config['interface'][iface] = dict_merge(default_values_iface, config['interface'][iface]) + + # Get PCI address auto + for iface, iface_config in config['interface'].items(): + if iface_config['pci'] == 'auto': + config['interface'][iface]['pci'] = _get_pci_address_by_interface(iface) + + config['other_interfaces'] = conf.get_config_dict(base_ethernet, key_mangling=('-', '_'), + get_first_key=True, no_tag_node_value_mangle=True) + + return config + + +def verify(config): + # bail out early - looks like removal from running config + if not config: + return None + + if 'interface' not in config: + raise ConfigError(f'"interface" is required but not set!') + + if 'cpu' in config: + if 'corelist_workers' in config['cpu'] and 'main_core' not in config['cpu']: + raise ConfigError(f'"cpu main-core" is required but not set!') + + +def generate(config): + if not config: + # Remove old config and return + service_conf.unlink(missing_ok=True) + return None + + render(service_conf, 'vpp/startup.conf.j2', config) + render(systemd_override, 'vpp/override.conf.j2', config) + + return None + + +def apply(config): + if not config: + print(f'systemctl stop {service_name}.service') + call(f'systemctl stop {service_name}.service') + return + else: + print(f'systemctl restart {service_name}.service') + call(f'systemctl restart {service_name}.service') + + call('systemctl daemon-reload') + + call('sudo sysctl -w vm.nr_hugepages=4096') + for iface, _ in config['interface'].items(): + # Create lcp + if iface not in Section.interfaces(): + vpp.lcp_create_host_interface(iface) + + # update interface config + #e = EthernetIf(iface) + #e.update(config['other_interfaces'][iface]) + + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) -- cgit v1.2.3 From e30c7fe6a42cd76427432b70a5b629c32be22c47 Mon Sep 17 00:00:00 2001 From: zsdc Date: Tue, 27 Jun 2023 18:00:02 +0300 Subject: VPP: T1797: Replaced CLI with API Replaced CLI commands with API calls. CLI commands still can be used via: ``` vpp_control = VPPControl() vpp_control.cli_cmd('command_here') ``` --- python/vyos/vpp.py | 102 +++++++++++++++++++++++++++++++++++++++++++++++---- src/conf_mode/vpp.py | 5 ++- 2 files changed, 97 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py index decc6c087..9e9471879 100644 --- a/python/vyos/vpp.py +++ b/python/vyos/vpp.py @@ -13,16 +13,102 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . -from vyos.util import call +from vpp_papi import VPPApiClient -def lcp_create_host_interface(ifname): - """LCP reprepsents a connection point between VPP dataplane - and the host stack +class VPPControl: + """Control VPP network stack """ - return call(f'vppctl lcp create {ifname} host-if {ifname}') + def __init__(self) -> None: + """Create VPP API connection + """ + self.vpp_api_client = VPPApiClient() + self.vpp_api_client.connect('vpp-vyos') -def set_interface_rx_mode(ifname, mode): - """Rx mode""" - return call(f'sudo vppctl set interface rx-mode {ifname} {mode}') + def __del__(self) -> None: + """Disconnect from VPP API + """ + self.vpp_api_client.disconnect() + + def cli_cmd(self, command: str) -> None: + """Send raw CLI command + + Args: + command (str): command to send + """ + self.vpp_api_client.api.cli_inband(cmd=command) + + def get_mac(self, ifname: str) -> str: + """Find MAC address by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + str: MAC address + """ + for iface in self.vpp_api_client.api.sw_interface_dump(): + if iface.interface_name == ifname: + return iface.l2_address.mac_string + return '' + + def get_sw_if_index(self, ifname: str) -> int | None: + """Find interface index by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + int | None: Interface index or None (if was not fount) + """ + for iface in self.vpp_api_client.api.sw_interface_dump(): + if iface.interface_name == ifname: + return iface.sw_if_index + return None + + def lcp_pair_add(self, iface_name_vpp: str, iface_name_kernel: str) -> None: + """Create LCP interface pair between VPP and kernel + + Args: + iface_name_vpp (str): interface name in VPP + iface_name_kernel (str): interface name in kernel + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + if iface_index: + self.vpp_api_client.api.lcp_itf_pair_add_del( + is_add=True, + sw_if_index=iface_index, + host_if_name=iface_name_kernel) + + def lcp_pair_del(self, iface_name_vpp: str, iface_name_kernel: str) -> None: + """Delete LCP interface pair between VPP and kernel + + Args: + iface_name_vpp (str): interface name in VPP + iface_name_kernel (str): interface name in kernel + """ + iface_index = self.get_sw_if_index(iface_name_vpp) + if iface_index: + self.vpp_api_client.api.lcp_itf_pair_add_del( + is_add=False, + sw_if_index=iface_index, + host_if_name=iface_name_kernel) + + def iface_rxmode(self, iface_name: str, rx_mode: str) -> None: + """Set interface rx-mode in VPP + + Args: + iface_name (str): interface name in VPP + rx_mode (str): mode (polling, interrupt, adaptive) + """ + modes_dict: dict[str, int] = { + 'polling': 1, + 'interrupt': 2, + 'adaptive': 3 + } + if rx_mode not in modes_dict: + return + iface_index = self.get_sw_if_index(iface_name) + self.vpp_api_client.api.sw_interface_set_rx_mode( + sw_if_index=iface_index, mode=modes_dict[rx_mode]) diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index aa6c14e89..d541e52ba 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -28,8 +28,8 @@ from vyos.template import render from vyos.xml import defaults from vyos import ConfigError -from vyos import vpp from vyos import airbag +from vyos.vpp import VPPControl airbag.enable() @@ -124,10 +124,11 @@ def apply(config): call('systemctl daemon-reload') call('sudo sysctl -w vm.nr_hugepages=4096') + vpp_control = VPPControl() for iface, _ in config['interface'].items(): # Create lcp if iface not in Section.interfaces(): - vpp.lcp_create_host_interface(iface) + vpp_control.lcp_pair_add(iface, iface) # update interface config #e = EthernetIf(iface) -- cgit v1.2.3 From 0bf443acca2985a10ef26c1651992c185d4fd4fa Mon Sep 17 00:00:00 2001 From: zsdc Date: Tue, 27 Jun 2023 23:04:14 +0300 Subject: VPP: T1797: Improved PCI address search Use info from both ethtool and VPP to find PCI address for an interface. --- python/vyos/vpp.py | 40 ++++++++++++++++++++++++++++++++++++++-- src/conf_mode/vpp.py | 25 +++++++++++++++++++------ 2 files changed, 57 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py index 9e9471879..d60ecc1b3 100644 --- a/python/vyos/vpp.py +++ b/python/vyos/vpp.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . +from re import search as re_search, MULTILINE as re_M + from vpp_papi import VPPApiClient @@ -27,17 +29,29 @@ class VPPControl: self.vpp_api_client.connect('vpp-vyos') def __del__(self) -> None: + """Disconnect from VPP API (destructor) + """ + self.disconnect() + + def disconnect(self) -> None: """Disconnect from VPP API """ self.vpp_api_client.disconnect() - def cli_cmd(self, command: str) -> None: + def cli_cmd(self, command: str, return_output: bool = False) -> str: """Send raw CLI command Args: command (str): command to send + return_output (bool, optional): Return command output. Defaults to False. + + Returns: + str: output of the command, only if it was successful """ - self.vpp_api_client.api.cli_inband(cmd=command) + cli_answer = self.vpp_api_client.api.cli_inband(cmd=command) + if return_output and cli_answer.retval == 0: + return cli_answer.reply + return '' def get_mac(self, ifname: str) -> str: """Find MAC address by interface name in VPP @@ -112,3 +126,25 @@ class VPPControl: iface_index = self.get_sw_if_index(iface_name) self.vpp_api_client.api.sw_interface_set_rx_mode( sw_if_index=iface_index, mode=modes_dict[rx_mode]) + + def get_pci_addr(self, ifname: str) -> str: + """Find PCI address of interface by interface name in VPP + + Args: + ifname (str): interface name inside VPP + + Returns: + str: PCI address + """ + hw_info = self.cli_cmd(f'show hardware-interfaces {ifname}', + return_output=True) + + regex_filter = r'^\s+pci: device (?P\w+:\w+) subsystem (?P\w+:\w+) address (?P
\w+:\w+:\w+\.\w+) numa (?P\w+)$' + re_obj = re_search(regex_filter, hw_info, re_M) + + # return empty string if no interface or no PCI info was found + if not hw_info or not re_obj: + return '' + + address = re_obj.groupdict().get('address', '') + return address diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index d541e52ba..54ea54852 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -16,6 +16,7 @@ from pathlib import Path +from re import search as re_search, MULTILINE as re_M from vyos.config import Config from vyos.configdict import dict_merge @@ -38,14 +39,26 @@ service_conf = Path(f'/run/vpp/{service_name}.conf') systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf' -def _get_pci_address_by_interface(iface): +def _get_pci_address_by_interface(iface) -> str: from vyos.util import rc_cmd rc, out = rc_cmd(f'ethtool -i {iface}') - if rc == 0: - output_lines = out.split('\n') - for line in output_lines: - if 'bus-info' in line: - return line.split(None, 1)[1].strip() + # if ethtool command was successful + if rc == 0 and out: + regex_filter = r'^bus-info: (?P
\w+:\w+:\w+\.\w+)$' + re_obj = re_search(regex_filter, out, re_M) + # if bus-info with PCI address found + if re_obj: + address = re_obj.groupdict().get('address', '') + return address + # use VPP - maybe interface already attached to it + vpp_control = VPPControl() + pci_addr = vpp_control.get_pci_addr(iface) + vpp_control.disconnect() + if pci_addr: + return pci_addr + # return empty string if address was not found + return '' + def get_config(config=None): -- cgit v1.2.3 From 4c3fa286a8db7ff099db2d92573d3d47df5e7763 Mon Sep 17 00:00:00 2001 From: zsdc Date: Wed, 28 Jun 2023 17:46:22 +0300 Subject: VPP: T1797: Improved VPP support - added ability to add/remove interfaces without system reboot - added `attempts` and `interval` to the VPP API connection. This is helpful in case of high system load or when VPP was just started and API is not yet available. - added exceptions to API calls. This allows handling errors in communication with API properly in conf-mode scripts. - fixed PCI address search in VPP to match Linux kernel and ethtool style - fixed systemd daemons control - first reload, then restart - removed debug prints - removed `vm.nr_hugepages` configuration. It is not required now but increases RAM requirements a lot. --- python/vyos/vpp.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++--- src/conf_mode/vpp.py | 48 +++++++++++++++++++++---------- 2 files changed, 108 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py index d60ecc1b3..cf0d27eb1 100644 --- a/python/vyos/vpp.py +++ b/python/vyos/vpp.py @@ -13,20 +13,57 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . +from functools import wraps +from pathlib import Path from re import search as re_search, MULTILINE as re_M +from time import sleep from vpp_papi import VPPApiClient +from vpp_papi import VPPIOError class VPPControl: """Control VPP network stack """ - def __init__(self) -> None: + class _Decorators: + """Decorators for VPPControl + """ + + @classmethod + def api_call(cls, decorated_func): + + @wraps(decorated_func) + def api_safe_wrapper(cls, *args, **kwargs): + if not cls.vpp_api_client.transport.connected: + raise VPPIOError(2, 'VPP API is not connected') + return decorated_func(cls, *args, **kwargs) + + return api_safe_wrapper + + def __init__(self, attempts: int = 5, interval: int = 1000) -> None: """Create VPP API connection + + Args: + attempts (int, optional): attempts to connect. Defaults to 5. + interval (int, optional): interval between attempts in ms. Defaults to 1000. + + Raises: + VPPIOError: Connection to API cannot be established """ self.vpp_api_client = VPPApiClient() - self.vpp_api_client.connect('vpp-vyos') + # connect with interval + while attempts: + try: + attempts -= 1 + self.vpp_api_client.connect('vpp-vyos') + break + except (ConnectionRefusedError, FileNotFoundError) as err: + print(f'VPP API connection timeout: {err}') + sleep(interval / 1000) + # raise exception if connection was not successful in the end + if not self.vpp_api_client.transport.connected: + raise VPPIOError(2, 'Cannot connect to VPP API') def __del__(self) -> None: """Disconnect from VPP API (destructor) @@ -36,8 +73,10 @@ class VPPControl: def disconnect(self) -> None: """Disconnect from VPP API """ - self.vpp_api_client.disconnect() + if self.vpp_api_client.transport.connected: + self.vpp_api_client.disconnect() + @_Decorators.api_call def cli_cmd(self, command: str, return_output: bool = False) -> str: """Send raw CLI command @@ -53,6 +92,7 @@ class VPPControl: return cli_answer.reply return '' + @_Decorators.api_call def get_mac(self, ifname: str) -> str: """Find MAC address by interface name in VPP @@ -67,6 +107,7 @@ class VPPControl: return iface.l2_address.mac_string return '' + @_Decorators.api_call def get_sw_if_index(self, ifname: str) -> int | None: """Find interface index by interface name in VPP @@ -81,6 +122,7 @@ class VPPControl: return iface.sw_if_index return None + @_Decorators.api_call def lcp_pair_add(self, iface_name_vpp: str, iface_name_kernel: str) -> None: """Create LCP interface pair between VPP and kernel @@ -95,6 +137,7 @@ class VPPControl: sw_if_index=iface_index, host_if_name=iface_name_kernel) + @_Decorators.api_call def lcp_pair_del(self, iface_name_vpp: str, iface_name_kernel: str) -> None: """Delete LCP interface pair between VPP and kernel @@ -109,6 +152,7 @@ class VPPControl: sw_if_index=iface_index, host_if_name=iface_name_kernel) + @_Decorators.api_call def iface_rxmode(self, iface_name: str, rx_mode: str) -> None: """Set interface rx-mode in VPP @@ -127,6 +171,7 @@ class VPPControl: self.vpp_api_client.api.sw_interface_set_rx_mode( sw_if_index=iface_index, mode=modes_dict[rx_mode]) + @_Decorators.api_call def get_pci_addr(self, ifname: str) -> str: """Find PCI address of interface by interface name in VPP @@ -147,4 +192,30 @@ class VPPControl: return '' address = re_obj.groupdict().get('address', '') - return address + + # we need to modify address to math kernel style + # for example: 0000:06:14.00 -> 0000:06:14.0 + address_chunks: list[str] | Any = address.split('.') + address_normalized: str = f'{address_chunks[0]}.{int(address_chunks[1])}' + + return address_normalized + + +class HostControl: + """Control Linux host + """ + + def pci_rescan(self, address: str = '') -> None: + """Rescan PCI device by removing it and rescan PCI bus + + If PCI address is not defined - just rescan PCI bus + + Args: + address (str, optional): PCI address of device. Defaults to ''. + """ + if address: + device_file = Path(f'/sys/bus/pci/devices/{address}/remove') + if device_file.exists(): + device_file.write_text('1') + rescan_file = Path('/sys/bus/pci/rescan') + rescan_file.write_text('1') diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index 54ea54852..25fe159f8 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -20,6 +20,7 @@ from re import search as re_search, MULTILINE as re_M from vyos.config import Config from vyos.configdict import dict_merge +from vyos.configdict import node_changed from vyos.ifconfig import Section from vyos.ifconfig import EthernetIf from vyos.ifconfig import interface @@ -31,6 +32,7 @@ from vyos.xml import defaults from vyos import ConfigError from vyos import airbag from vyos.vpp import VPPControl +from vyos.vpp import HostControl airbag.enable() @@ -53,7 +55,6 @@ def _get_pci_address_by_interface(iface) -> str: # use VPP - maybe interface already attached to it vpp_control = VPPControl() pci_addr = vpp_control.get_pci_addr(iface) - vpp_control.disconnect() if pci_addr: return pci_addr # return empty string if address was not found @@ -69,8 +70,20 @@ def get_config(config=None): base = ['vpp'] base_ethernet = ['interfaces', 'ethernet'] + + # find interfaces removed from VPP + removed_ifaces = [] + tmp = node_changed(conf, base + ['interface']) + if tmp: + for removed_iface in tmp: + pci_address: str = _get_pci_address_by_interface(removed_iface) + removed_ifaces.append({ + 'iface_name': removed_iface, + 'iface_pci_addr': pci_address + }) + if not conf.exists(base): - return None + return {'removed_ifaces': removed_ifaces} config = conf.get_config_dict(base, get_first_key=True, @@ -97,12 +110,15 @@ def get_config(config=None): config['other_interfaces'] = conf.get_config_dict(base_ethernet, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) + if removed_ifaces: + config['removed_ifaces'] = removed_ifaces + return config def verify(config): # bail out early - looks like removal from running config - if not config: + if not config or (len(config) == 1 and 'removed_ifaces' in config): return None if 'interface' not in config: @@ -114,7 +130,7 @@ def verify(config): def generate(config): - if not config: + if not config or (len(config) == 1 and 'removed_ifaces' in config): # Remove old config and return service_conf.unlink(missing_ok=True) return None @@ -126,22 +142,24 @@ def generate(config): def apply(config): - if not config: - print(f'systemctl stop {service_name}.service') + if not config or (len(config) == 1 and 'removed_ifaces' in config): call(f'systemctl stop {service_name}.service') - return else: - print(f'systemctl restart {service_name}.service') + call('systemctl daemon-reload') call(f'systemctl restart {service_name}.service') - call('systemctl daemon-reload') + for iface in config.get('removed_ifaces', []): + HostControl().pci_rescan(iface['iface_pci_addr']) - call('sudo sysctl -w vm.nr_hugepages=4096') - vpp_control = VPPControl() - for iface, _ in config['interface'].items(): - # Create lcp - if iface not in Section.interfaces(): - vpp_control.lcp_pair_add(iface, iface) + if 'interface' in config: + # connect to VPP + # must be performed multiple attempts because API is not available + # immediately after the service restart + vpp_control = VPPControl(attempts=20, interval=500) + for iface, _ in config['interface'].items(): + # Create lcp + if iface not in Section.interfaces(): + vpp_control.lcp_pair_add(iface, iface) # update interface config #e = EthernetIf(iface) -- cgit v1.2.3 From 199657fc60782961e86af2f3f49b246b29b9723c Mon Sep 17 00:00:00 2001 From: zsdc Date: Thu, 29 Jun 2023 16:31:13 +0300 Subject: VPP: T1797: Optimized interfaces add/remove - added extra renaming operation to be sure that interface has the same name as before in the system after it was moved from VPP to kernel - added extra check after PCI device removal/adding - added check for proper `retval` for CPI calls where it is available - replaced empty return with an error in `_get_pci_address_by_interface()` because not resolved address will lead to inconsistency of the system later --- python/vyos/vpp.py | 132 +++++++++++++++++++++++++++++++++++++++++++-------- src/conf_mode/vpp.py | 14 ++++-- 2 files changed, 123 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py index cf0d27eb1..76e5d29c3 100644 --- a/python/vyos/vpp.py +++ b/python/vyos/vpp.py @@ -15,11 +15,12 @@ from functools import wraps from pathlib import Path -from re import search as re_search, MULTILINE as re_M +from re import search as re_search, fullmatch as re_fullmatch, MULTILINE as re_M +from subprocess import run from time import sleep from vpp_papi import VPPApiClient -from vpp_papi import VPPIOError +from vpp_papi import VPPIOError, VPPValueError class VPPControl: @@ -32,6 +33,14 @@ class VPPControl: @classmethod def api_call(cls, decorated_func): + """Check if API is connected before API call + + Args: + decorated_func: function to decorate + + Raises: + VPPIOError: Connection to API is not established + """ @wraps(decorated_func) def api_safe_wrapper(cls, *args, **kwargs): @@ -41,6 +50,27 @@ class VPPControl: return api_safe_wrapper + @classmethod + def check_retval(cls, decorated_func): + """Check retval from API response + + Args: + decorated_func: function to decorate + + Raises: + VPPValueError: raised when retval is not 0 + """ + + @wraps(decorated_func) + def check_retval_wrapper(cls, *args, **kwargs): + return_value = decorated_func(cls, *args, **kwargs) + if not return_value.retval == 0: + raise VPPValueError( + f'VPP API call failed: {return_value.retval}') + return return_value + + return check_retval_wrapper + def __init__(self, attempts: int = 5, interval: int = 1000) -> None: """Create VPP API connection @@ -76,21 +106,18 @@ class VPPControl: if self.vpp_api_client.transport.connected: self.vpp_api_client.disconnect() + @_Decorators.check_retval @_Decorators.api_call - def cli_cmd(self, command: str, return_output: bool = False) -> str: + def cli_cmd(self, command: str): """Send raw CLI command Args: command (str): command to send - return_output (bool, optional): Return command output. Defaults to False. Returns: - str: output of the command, only if it was successful + vpp_papi.vpp_serializer.cli_inband_reply: CLI reply class """ - cli_answer = self.vpp_api_client.api.cli_inband(cmd=command) - if return_output and cli_answer.retval == 0: - return cli_answer.reply - return '' + return self.vpp_api_client.api.cli_inband(cmd=command) @_Decorators.api_call def get_mac(self, ifname: str) -> str: @@ -122,6 +149,7 @@ class VPPControl: return iface.sw_if_index return None + @_Decorators.check_retval @_Decorators.api_call def lcp_pair_add(self, iface_name_vpp: str, iface_name_kernel: str) -> None: """Create LCP interface pair between VPP and kernel @@ -132,11 +160,12 @@ class VPPControl: """ iface_index = self.get_sw_if_index(iface_name_vpp) if iface_index: - self.vpp_api_client.api.lcp_itf_pair_add_del( + return self.vpp_api_client.api.lcp_itf_pair_add_del( is_add=True, sw_if_index=iface_index, host_if_name=iface_name_kernel) + @_Decorators.check_retval @_Decorators.api_call def lcp_pair_del(self, iface_name_vpp: str, iface_name_kernel: str) -> None: """Delete LCP interface pair between VPP and kernel @@ -147,11 +176,12 @@ class VPPControl: """ iface_index = self.get_sw_if_index(iface_name_vpp) if iface_index: - self.vpp_api_client.api.lcp_itf_pair_add_del( + return self.vpp_api_client.api.lcp_itf_pair_add_del( is_add=False, sw_if_index=iface_index, host_if_name=iface_name_kernel) + @_Decorators.check_retval @_Decorators.api_call def iface_rxmode(self, iface_name: str, rx_mode: str) -> None: """Set interface rx-mode in VPP @@ -166,9 +196,9 @@ class VPPControl: 'adaptive': 3 } if rx_mode not in modes_dict: - return + raise VPPValueError(f'Mode {rx_mode} is not known') iface_index = self.get_sw_if_index(iface_name) - self.vpp_api_client.api.sw_interface_set_rx_mode( + return self.vpp_api_client.api.sw_interface_set_rx_mode( sw_if_index=iface_index, mode=modes_dict[rx_mode]) @_Decorators.api_call @@ -181,8 +211,7 @@ class VPPControl: Returns: str: PCI address """ - hw_info = self.cli_cmd(f'show hardware-interfaces {ifname}', - return_output=True) + hw_info = self.cli_cmd(f'show hardware-interfaces {ifname}').reply regex_filter = r'^\s+pci: device (?P\w+:\w+) subsystem (?P\w+:\w+) address (?P
\w+:\w+:\w+\.\w+) numa (?P\w+)$' re_obj = re_search(regex_filter, hw_info, re_M) @@ -195,7 +224,7 @@ class VPPControl: # we need to modify address to math kernel style # for example: 0000:06:14.00 -> 0000:06:14.0 - address_chunks: list[str] | Any = address.split('.') + address_chunks: list[str] = address.split('.') address_normalized: str = f'{address_chunks[0]}.{int(address_chunks[1])}' return address_normalized @@ -205,7 +234,8 @@ class HostControl: """Control Linux host """ - def pci_rescan(self, address: str = '') -> None: + @staticmethod + def pci_rescan(pci_addr: str = '') -> None: """Rescan PCI device by removing it and rescan PCI bus If PCI address is not defined - just rescan PCI bus @@ -213,9 +243,73 @@ class HostControl: Args: address (str, optional): PCI address of device. Defaults to ''. """ - if address: - device_file = Path(f'/sys/bus/pci/devices/{address}/remove') + if pci_addr: + device_file = Path(f'/sys/bus/pci/devices/{pci_addr}/remove') if device_file.exists(): device_file.write_text('1') + # wait 10 seconds max until device will be removed + attempts = 100 + while device_file.exists() and attempts: + attempts -= 1 + sleep(0.1) + if device_file.exists(): + raise TimeoutError( + f'Timeout was reached for removing PCI device {pci_addr}' + ) + else: + raise FileNotFoundError(f'PCI device {pci_addr} does not exist') rescan_file = Path('/sys/bus/pci/rescan') rescan_file.write_text('1') + if pci_addr: + # wait 10 seconds max until device will be installed + attempts = 100 + while not device_file.exists() and attempts: + attempts -= 1 + sleep(0.1) + if not device_file.exists(): + raise TimeoutError( + f'Timeout was reached for installing PCI device {pci_addr}') + + @staticmethod + def get_eth_name(pci_addr: str) -> str: + """Find Ethernet interface name by PCI address + + Args: + pci_addr (str): PCI address + + Raises: + FileNotFoundError: no Ethernet interface was found + + Returns: + str: Ethernet interface name + """ + # find all PCI devices with eth* names + net_devs: dict[str, str] = {} + net_devs_dir = Path('/sys/class/net') + regex_filter = r'^/sys/devices/pci[\w/:\.]+/(?P\w+:\w+:\w+\.\w+)/[\w/:\.]+/(?Peth\d+)$' + for dir in net_devs_dir.iterdir(): + real_dir: str = dir.resolve().as_posix() + re_obj = re_fullmatch(regex_filter, real_dir) + if re_obj: + iface_name: str = re_obj.group('iface_name') + iface_addr: str = re_obj.group('pci_addr') + net_devs.update({iface_addr: iface_name}) + # match to provided PCI address and return a name if found + if pci_addr in net_devs: + return net_devs[pci_addr] + # raise error if device was not found + raise FileNotFoundError( + f'PCI device {pci_addr} not found in ethernet interfaces') + + @staticmethod + def rename_iface(name_old: str, name_new: str) -> None: + """Rename interface + + Args: + name_old (str): old name + name_new (str): new name + """ + rename_cmd: list[str] = [ + 'ip', 'link', 'set', name_old, 'name', name_new + ] + run(rename_cmd) diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index 25fe159f8..dd01da87e 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -53,12 +53,12 @@ def _get_pci_address_by_interface(iface) -> str: address = re_obj.groupdict().get('address', '') return address # use VPP - maybe interface already attached to it - vpp_control = VPPControl() + vpp_control = VPPControl(attempts=20, interval=500) pci_addr = vpp_control.get_pci_addr(iface) if pci_addr: return pci_addr - # return empty string if address was not found - return '' + # raise error if PCI address was not found + raise ConfigError(f'Cannot find PCI address for interface {iface}') @@ -148,8 +148,14 @@ def apply(config): call('systemctl daemon-reload') call(f'systemctl restart {service_name}.service') + # Initialize interfaces removed from VPP for iface in config.get('removed_ifaces', []): - HostControl().pci_rescan(iface['iface_pci_addr']) + host_control = HostControl() + # rescan PCI to use a proper driver + host_control.pci_rescan(iface['iface_pci_addr']) + # rename to the proper name + iface_new_name: str = host_control.get_eth_name(iface['iface_pci_addr']) + host_control.rename_iface(iface_new_name, iface['iface_name']) if 'interface' in config: # connect to VPP -- cgit v1.2.3