From 0bf443acca2985a10ef26c1651992c185d4fd4fa Mon Sep 17 00:00:00 2001 From: zsdc Date: Tue, 27 Jun 2023 23:04:14 +0300 Subject: VPP: T1797: Improved PCI address search Use info from both ethtool and VPP to find PCI address for an interface. --- src/conf_mode/vpp.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index d541e52ba..54ea54852 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -16,6 +16,7 @@ from pathlib import Path +from re import search as re_search, MULTILINE as re_M from vyos.config import Config from vyos.configdict import dict_merge @@ -38,14 +39,26 @@ service_conf = Path(f'/run/vpp/{service_name}.conf') systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf' -def _get_pci_address_by_interface(iface): +def _get_pci_address_by_interface(iface) -> str: from vyos.util import rc_cmd rc, out = rc_cmd(f'ethtool -i {iface}') - if rc == 0: - output_lines = out.split('\n') - for line in output_lines: - if 'bus-info' in line: - return line.split(None, 1)[1].strip() + # if ethtool command was successful + if rc == 0 and out: + regex_filter = r'^bus-info: (?P
\w+:\w+:\w+\.\w+)$' + re_obj = re_search(regex_filter, out, re_M) + # if bus-info with PCI address found + if re_obj: + address = re_obj.groupdict().get('address', '') + return address + # use VPP - maybe interface already attached to it + vpp_control = VPPControl() + pci_addr = vpp_control.get_pci_addr(iface) + vpp_control.disconnect() + if pci_addr: + return pci_addr + # return empty string if address was not found + return '' + def get_config(config=None): -- cgit v1.2.3 From 4c3fa286a8db7ff099db2d92573d3d47df5e7763 Mon Sep 17 00:00:00 2001 From: zsdc Date: Wed, 28 Jun 2023 17:46:22 +0300 Subject: VPP: T1797: Improved VPP support - added ability to add/remove interfaces without system reboot - added `attempts` and `interval` to the VPP API connection. This is helpful in case of high system load or when VPP was just started and API is not yet available. - added exceptions to API calls. This allows handling errors in communication with API properly in conf-mode scripts. - fixed PCI address search in VPP to match Linux kernel and ethtool style - fixed systemd daemons control - first reload, then restart - removed debug prints - removed `vm.nr_hugepages` configuration. It is not required now but increases RAM requirements a lot. --- python/vyos/vpp.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++--- src/conf_mode/vpp.py | 48 +++++++++++++++++++++---------- 2 files changed, 108 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py index d60ecc1b3..cf0d27eb1 100644 --- a/python/vyos/vpp.py +++ b/python/vyos/vpp.py @@ -13,20 +13,57 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . +from functools import wraps +from pathlib import Path from re import search as re_search, MULTILINE as re_M +from time import sleep from vpp_papi import VPPApiClient +from vpp_papi import VPPIOError class VPPControl: """Control VPP network stack """ - def __init__(self) -> None: + class _Decorators: + """Decorators for VPPControl + """ + + @classmethod + def api_call(cls, decorated_func): + + @wraps(decorated_func) + def api_safe_wrapper(cls, *args, **kwargs): + if not cls.vpp_api_client.transport.connected: + raise VPPIOError(2, 'VPP API is not connected') + return decorated_func(cls, *args, **kwargs) + + return api_safe_wrapper + + def __init__(self, attempts: int = 5, interval: int = 1000) -> None: """Create VPP API connection + + Args: + attempts (int, optional): attempts to connect. Defaults to 5. + interval (int, optional): interval between attempts in ms. Defaults to 1000. + + Raises: + VPPIOError: Connection to API cannot be established """ self.vpp_api_client = VPPApiClient() - self.vpp_api_client.connect('vpp-vyos') + # connect with interval + while attempts: + try: + attempts -= 1 + self.vpp_api_client.connect('vpp-vyos') + break + except (ConnectionRefusedError, FileNotFoundError) as err: + print(f'VPP API connection timeout: {err}') + sleep(interval / 1000) + # raise exception if connection was not successful in the end + if not self.vpp_api_client.transport.connected: + raise VPPIOError(2, 'Cannot connect to VPP API') def __del__(self) -> None: """Disconnect from VPP API (destructor) @@ -36,8 +73,10 @@ class VPPControl: def disconnect(self) -> None: """Disconnect from VPP API """ - self.vpp_api_client.disconnect() + if self.vpp_api_client.transport.connected: + self.vpp_api_client.disconnect() + @_Decorators.api_call def cli_cmd(self, command: str, return_output: bool = False) -> str: """Send raw CLI command @@ -53,6 +92,7 @@ class VPPControl: return cli_answer.reply return '' + @_Decorators.api_call def get_mac(self, ifname: str) -> str: """Find MAC address by interface name in VPP @@ -67,6 +107,7 @@ class VPPControl: return iface.l2_address.mac_string return '' + @_Decorators.api_call def get_sw_if_index(self, ifname: str) -> int | None: """Find interface index by interface name in VPP @@ -81,6 +122,7 @@ class VPPControl: return iface.sw_if_index return None + @_Decorators.api_call def lcp_pair_add(self, iface_name_vpp: str, iface_name_kernel: str) -> None: """Create LCP interface pair between VPP and kernel @@ -95,6 +137,7 @@ class VPPControl: sw_if_index=iface_index, host_if_name=iface_name_kernel) + @_Decorators.api_call def lcp_pair_del(self, iface_name_vpp: str, iface_name_kernel: str) -> None: """Delete LCP interface pair between VPP and kernel @@ -109,6 +152,7 @@ class VPPControl: sw_if_index=iface_index, host_if_name=iface_name_kernel) + @_Decorators.api_call def iface_rxmode(self, iface_name: str, rx_mode: str) -> None: """Set interface rx-mode in VPP @@ -127,6 +171,7 @@ class VPPControl: self.vpp_api_client.api.sw_interface_set_rx_mode( sw_if_index=iface_index, mode=modes_dict[rx_mode]) + @_Decorators.api_call def get_pci_addr(self, ifname: str) -> str: """Find PCI address of interface by interface name in VPP @@ -147,4 +192,30 @@ class VPPControl: return '' address = re_obj.groupdict().get('address', '') - return address + + # we need to modify address to math kernel style + # for example: 0000:06:14.00 -> 0000:06:14.0 + address_chunks: list[str] | Any = address.split('.') + address_normalized: str = f'{address_chunks[0]}.{int(address_chunks[1])}' + + return address_normalized + + +class HostControl: + """Control Linux host + """ + + def pci_rescan(self, address: str = '') -> None: + """Rescan PCI device by removing it and rescan PCI bus + + If PCI address is not defined - just rescan PCI bus + + Args: + address (str, optional): PCI address of device. Defaults to ''. + """ + if address: + device_file = Path(f'/sys/bus/pci/devices/{address}/remove') + if device_file.exists(): + device_file.write_text('1') + rescan_file = Path('/sys/bus/pci/rescan') + rescan_file.write_text('1') diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py index 54ea54852..25fe159f8 100755 --- a/src/conf_mode/vpp.py +++ b/src/conf_mode/vpp.py @@ -20,6 +20,7 @@ from re import search as re_search, MULTILINE as re_M from vyos.config import Config from vyos.configdict import dict_merge +from vyos.configdict import node_changed from vyos.ifconfig import Section from vyos.ifconfig import EthernetIf from vyos.ifconfig import interface @@ -31,6 +32,7 @@ from vyos.xml import defaults from vyos import ConfigError from vyos import airbag from vyos.vpp import VPPControl +from vyos.vpp import HostControl airbag.enable() @@ -53,7 +55,6 @@ def _get_pci_address_by_interface(iface) -> str: # use VPP - maybe interface already attached to it vpp_control = VPPControl() pci_addr = vpp_control.get_pci_addr(iface) - vpp_control.disconnect() if pci_addr: return pci_addr # return empty string if address was not found @@ -69,8 +70,20 @@ def get_config(config=None): base = ['vpp'] base_ethernet = ['interfaces', 'ethernet'] + + # find interfaces removed from VPP + removed_ifaces = [] + tmp = node_changed(conf, base + ['interface']) + if tmp: + for removed_iface in tmp: + pci_address: str = _get_pci_address_by_interface(removed_iface) + removed_ifaces.append({ + 'iface_name': removed_iface, + 'iface_pci_addr': pci_address + }) + if not conf.exists(base): - return None + return {'removed_ifaces': removed_ifaces} config = conf.get_config_dict(base, get_first_key=True, @@ -97,12 +110,15 @@ def get_config(config=None): config['other_interfaces'] = conf.get_config_dict(base_ethernet, key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) + if removed_ifaces: + config['removed_ifaces'] = removed_ifaces + return config def verify(config): # bail out early - looks like removal from running config - if not config: + if not config or (len(config) == 1 and 'removed_ifaces' in config): return None if 'interface' not in config: @@ -114,7 +130,7 @@ def verify(config): def generate(config): - if not config: + if not config or (len(config) == 1 and 'removed_ifaces' in config): # Remove old config and return service_conf.unlink(missing_ok=True) return None @@ -126,22 +142,24 @@ def generate(config): def apply(config): - if not config: - print(f'systemctl stop {service_name}.service') + if not config or (len(config) == 1 and 'removed_ifaces' in config): call(f'systemctl stop {service_name}.service') - return else: - print(f'systemctl restart {service_name}.service') + call('systemctl daemon-reload') call(f'systemctl restart {service_name}.service') - call('systemctl daemon-reload') + for iface in config.get('removed_ifaces', []): + HostControl().pci_rescan(iface['iface_pci_addr']) - call('sudo sysctl -w vm.nr_hugepages=4096') - vpp_control = VPPControl() - for iface, _ in config['interface'].items(): - # Create lcp - if iface not in Section.interfaces(): - vpp_control.lcp_pair_add(iface, iface) + if 'interface' in config: + # connect to VPP + # must be performed multiple attempts because API is not available + # immediately after the service restart + vpp_control = VPPControl(attempts=20, interval=500) + for iface, _ in config['interface'].items(): + # Create lcp + if iface not in Section.interfaces(): + vpp_control.lcp_pair_add(iface, iface) # update interface config #e = EthernetIf(iface) -- cgit v1.2.3