From 0bf443acca2985a10ef26c1651992c185d4fd4fa Mon Sep 17 00:00:00 2001
From: zsdc <taras@vyos.io>
Date: Tue, 27 Jun 2023 23:04:14 +0300
Subject: VPP: T1797: Improved PCI address search
Use info from both ethtool and VPP to find PCI address for an
interface.
---
python/vyos/vpp.py | 40 ++++++++++++++++++++++++++++++++++++++--
src/conf_mode/vpp.py | 25 +++++++++++++++++++------
2 files changed, 57 insertions(+), 8 deletions(-)
diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py
index 9e9471879..d60ecc1b3 100644
--- a/python/vyos/vpp.py
+++ b/python/vyos/vpp.py
@@ -13,6 +13,8 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+from re import search as re_search, MULTILINE as re_M
+
from vpp_papi import VPPApiClient
@@ -27,17 +29,29 @@ class VPPControl:
self.vpp_api_client.connect('vpp-vyos')
def __del__(self) -> None:
+ """Disconnect from VPP API (destructor)
+ """
+ self.disconnect()
+
+ def disconnect(self) -> None:
"""Disconnect from VPP API
"""
self.vpp_api_client.disconnect()
- def cli_cmd(self, command: str) -> None:
+ def cli_cmd(self, command: str, return_output: bool = False) -> str:
"""Send raw CLI command
Args:
command (str): command to send
+ return_output (bool, optional): Return command output. Defaults to False.
+
+ Returns:
+ str: output of the command, only if it was successful
"""
- self.vpp_api_client.api.cli_inband(cmd=command)
+ cli_answer = self.vpp_api_client.api.cli_inband(cmd=command)
+ if return_output and cli_answer.retval == 0:
+ return cli_answer.reply
+ return ''
def get_mac(self, ifname: str) -> str:
"""Find MAC address by interface name in VPP
@@ -112,3 +126,25 @@ class VPPControl:
iface_index = self.get_sw_if_index(iface_name)
self.vpp_api_client.api.sw_interface_set_rx_mode(
sw_if_index=iface_index, mode=modes_dict[rx_mode])
+
+ def get_pci_addr(self, ifname: str) -> str:
+ """Find PCI address of interface by interface name in VPP
+
+ Args:
+ ifname (str): interface name inside VPP
+
+ Returns:
+ str: PCI address
+ """
+ hw_info = self.cli_cmd(f'show hardware-interfaces {ifname}',
+ return_output=True)
+
+ regex_filter = r'^\s+pci: device (?P<device>\w+:\w+) subsystem (?P<subsystem>\w+:\w+) address (?P<address>\w+:\w+:\w+\.\w+) numa (?P<numa>\w+)$'
+ re_obj = re_search(regex_filter, hw_info, re_M)
+
+ # return empty string if no interface or no PCI info was found
+ if not hw_info or not re_obj:
+ return ''
+
+ address = re_obj.groupdict().get('address', '')
+ return address
diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py
index d541e52ba..54ea54852 100755
--- a/src/conf_mode/vpp.py
+++ b/src/conf_mode/vpp.py
@@ -16,6 +16,7 @@
from pathlib import Path
+from re import search as re_search, MULTILINE as re_M
from vyos.config import Config
from vyos.configdict import dict_merge
@@ -38,14 +39,26 @@ service_conf = Path(f'/run/vpp/{service_name}.conf')
systemd_override = '/run/systemd/system/vpp.service.d/10-override.conf'
-def _get_pci_address_by_interface(iface):
+def _get_pci_address_by_interface(iface) -> str:
from vyos.util import rc_cmd
rc, out = rc_cmd(f'ethtool -i {iface}')
- if rc == 0:
- output_lines = out.split('\n')
- for line in output_lines:
- if 'bus-info' in line:
- return line.split(None, 1)[1].strip()
+ # if ethtool command was successful
+ if rc == 0 and out:
+ regex_filter = r'^bus-info: (?P<address>\w+:\w+:\w+\.\w+)$'
+ re_obj = re_search(regex_filter, out, re_M)
+ # if bus-info with PCI address found
+ if re_obj:
+ address = re_obj.groupdict().get('address', '')
+ return address
+ # use VPP - maybe interface already attached to it
+ vpp_control = VPPControl()
+ pci_addr = vpp_control.get_pci_addr(iface)
+ vpp_control.disconnect()
+ if pci_addr:
+ return pci_addr
+ # return empty string if address was not found
+ return ''
+
def get_config(config=None):
--
cgit v1.2.3
From 4c3fa286a8db7ff099db2d92573d3d47df5e7763 Mon Sep 17 00:00:00 2001
From: zsdc <taras@vyos.io>
Date: Wed, 28 Jun 2023 17:46:22 +0300
Subject: VPP: T1797: Improved VPP support
- added ability to add/remove interfaces without system reboot
- added `attempts` and `interval` to the VPP API connection. This is helpful in
case of high system load or when VPP was just started and API is not yet
available.
- added exceptions to API calls. This allows handling errors in communication
with API properly in conf-mode scripts.
- fixed PCI address search in VPP to match Linux kernel and ethtool style
- fixed systemd daemons control - first reload, then restart
- removed debug prints
- removed `vm.nr_hugepages` configuration. It is not required now but increases
RAM requirements a lot.
---
python/vyos/vpp.py | 79 +++++++++++++++++++++++++++++++++++++++++++++++++---
src/conf_mode/vpp.py | 48 +++++++++++++++++++++----------
2 files changed, 108 insertions(+), 19 deletions(-)
diff --git a/python/vyos/vpp.py b/python/vyos/vpp.py
index d60ecc1b3..cf0d27eb1 100644
--- a/python/vyos/vpp.py
+++ b/python/vyos/vpp.py
@@ -13,20 +13,57 @@
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+from functools import wraps
+from pathlib import Path
from re import search as re_search, MULTILINE as re_M
+from time import sleep
from vpp_papi import VPPApiClient
+from vpp_papi import VPPIOError
class VPPControl:
"""Control VPP network stack
"""
- def __init__(self) -> None:
+ class _Decorators:
+ """Decorators for VPPControl
+ """
+
+ @classmethod
+ def api_call(cls, decorated_func):
+
+ @wraps(decorated_func)
+ def api_safe_wrapper(cls, *args, **kwargs):
+ if not cls.vpp_api_client.transport.connected:
+ raise VPPIOError(2, 'VPP API is not connected')
+ return decorated_func(cls, *args, **kwargs)
+
+ return api_safe_wrapper
+
+ def __init__(self, attempts: int = 5, interval: int = 1000) -> None:
"""Create VPP API connection
+
+ Args:
+ attempts (int, optional): attempts to connect. Defaults to 5.
+ interval (int, optional): interval between attempts in ms. Defaults to 1000.
+
+ Raises:
+ VPPIOError: Connection to API cannot be established
"""
self.vpp_api_client = VPPApiClient()
- self.vpp_api_client.connect('vpp-vyos')
+ # connect with interval
+ while attempts:
+ try:
+ attempts -= 1
+ self.vpp_api_client.connect('vpp-vyos')
+ break
+ except (ConnectionRefusedError, FileNotFoundError) as err:
+ print(f'VPP API connection timeout: {err}')
+ sleep(interval / 1000)
+ # raise exception if connection was not successful in the end
+ if not self.vpp_api_client.transport.connected:
+ raise VPPIOError(2, 'Cannot connect to VPP API')
def __del__(self) -> None:
"""Disconnect from VPP API (destructor)
@@ -36,8 +73,10 @@ class VPPControl:
def disconnect(self) -> None:
"""Disconnect from VPP API
"""
- self.vpp_api_client.disconnect()
+ if self.vpp_api_client.transport.connected:
+ self.vpp_api_client.disconnect()
+ @_Decorators.api_call
def cli_cmd(self, command: str, return_output: bool = False) -> str:
"""Send raw CLI command
@@ -53,6 +92,7 @@ class VPPControl:
return cli_answer.reply
return ''
+ @_Decorators.api_call
def get_mac(self, ifname: str) -> str:
"""Find MAC address by interface name in VPP
@@ -67,6 +107,7 @@ class VPPControl:
return iface.l2_address.mac_string
return ''
+ @_Decorators.api_call
def get_sw_if_index(self, ifname: str) -> int | None:
"""Find interface index by interface name in VPP
@@ -81,6 +122,7 @@ class VPPControl:
return iface.sw_if_index
return None
+ @_Decorators.api_call
def lcp_pair_add(self, iface_name_vpp: str, iface_name_kernel: str) -> None:
"""Create LCP interface pair between VPP and kernel
@@ -95,6 +137,7 @@ class VPPControl:
sw_if_index=iface_index,
host_if_name=iface_name_kernel)
+ @_Decorators.api_call
def lcp_pair_del(self, iface_name_vpp: str, iface_name_kernel: str) -> None:
"""Delete LCP interface pair between VPP and kernel
@@ -109,6 +152,7 @@ class VPPControl:
sw_if_index=iface_index,
host_if_name=iface_name_kernel)
+ @_Decorators.api_call
def iface_rxmode(self, iface_name: str, rx_mode: str) -> None:
"""Set interface rx-mode in VPP
@@ -127,6 +171,7 @@ class VPPControl:
self.vpp_api_client.api.sw_interface_set_rx_mode(
sw_if_index=iface_index, mode=modes_dict[rx_mode])
+ @_Decorators.api_call
def get_pci_addr(self, ifname: str) -> str:
"""Find PCI address of interface by interface name in VPP
@@ -147,4 +192,30 @@ class VPPControl:
return ''
address = re_obj.groupdict().get('address', '')
- return address
+
+ # we need to modify address to math kernel style
+ # for example: 0000:06:14.00 -> 0000:06:14.0
+ address_chunks: list[str] | Any = address.split('.')
+ address_normalized: str = f'{address_chunks[0]}.{int(address_chunks[1])}'
+
+ return address_normalized
+
+
+class HostControl:
+ """Control Linux host
+ """
+
+ def pci_rescan(self, address: str = '') -> None:
+ """Rescan PCI device by removing it and rescan PCI bus
+
+ If PCI address is not defined - just rescan PCI bus
+
+ Args:
+ address (str, optional): PCI address of device. Defaults to ''.
+ """
+ if address:
+ device_file = Path(f'/sys/bus/pci/devices/{address}/remove')
+ if device_file.exists():
+ device_file.write_text('1')
+ rescan_file = Path('/sys/bus/pci/rescan')
+ rescan_file.write_text('1')
diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py
index 54ea54852..25fe159f8 100755
--- a/src/conf_mode/vpp.py
+++ b/src/conf_mode/vpp.py
@@ -20,6 +20,7 @@ from re import search as re_search, MULTILINE as re_M
from vyos.config import Config
from vyos.configdict import dict_merge
+from vyos.configdict import node_changed
from vyos.ifconfig import Section
from vyos.ifconfig import EthernetIf
from vyos.ifconfig import interface
@@ -31,6 +32,7 @@ from vyos.xml import defaults
from vyos import ConfigError
from vyos import airbag
from vyos.vpp import VPPControl
+from vyos.vpp import HostControl
airbag.enable()
@@ -53,7 +55,6 @@ def _get_pci_address_by_interface(iface) -> str:
# use VPP - maybe interface already attached to it
vpp_control = VPPControl()
pci_addr = vpp_control.get_pci_addr(iface)
- vpp_control.disconnect()
if pci_addr:
return pci_addr
# return empty string if address was not found
@@ -69,8 +70,20 @@ def get_config(config=None):
base = ['vpp']
base_ethernet = ['interfaces', 'ethernet']
+
+ # find interfaces removed from VPP
+ removed_ifaces = []
+ tmp = node_changed(conf, base + ['interface'])
+ if tmp:
+ for removed_iface in tmp:
+ pci_address: str = _get_pci_address_by_interface(removed_iface)
+ removed_ifaces.append({
+ 'iface_name': removed_iface,
+ 'iface_pci_addr': pci_address
+ })
+
if not conf.exists(base):
- return None
+ return {'removed_ifaces': removed_ifaces}
config = conf.get_config_dict(base,
get_first_key=True,
@@ -97,12 +110,15 @@ def get_config(config=None):
config['other_interfaces'] = conf.get_config_dict(base_ethernet, key_mangling=('-', '_'),
get_first_key=True, no_tag_node_value_mangle=True)
+ if removed_ifaces:
+ config['removed_ifaces'] = removed_ifaces
+
return config
def verify(config):
# bail out early - looks like removal from running config
- if not config:
+ if not config or (len(config) == 1 and 'removed_ifaces' in config):
return None
if 'interface' not in config:
@@ -114,7 +130,7 @@ def verify(config):
def generate(config):
- if not config:
+ if not config or (len(config) == 1 and 'removed_ifaces' in config):
# Remove old config and return
service_conf.unlink(missing_ok=True)
return None
@@ -126,22 +142,24 @@ def generate(config):
def apply(config):
- if not config:
- print(f'systemctl stop {service_name}.service')
+ if not config or (len(config) == 1 and 'removed_ifaces' in config):
call(f'systemctl stop {service_name}.service')
- return
else:
- print(f'systemctl restart {service_name}.service')
+ call('systemctl daemon-reload')
call(f'systemctl restart {service_name}.service')
- call('systemctl daemon-reload')
+ for iface in config.get('removed_ifaces', []):
+ HostControl().pci_rescan(iface['iface_pci_addr'])
- call('sudo sysctl -w vm.nr_hugepages=4096')
- vpp_control = VPPControl()
- for iface, _ in config['interface'].items():
- # Create lcp
- if iface not in Section.interfaces():
- vpp_control.lcp_pair_add(iface, iface)
+ if 'interface' in config:
+ # connect to VPP
+ # must be performed multiple attempts because API is not available
+ # immediately after the service restart
+ vpp_control = VPPControl(attempts=20, interval=500)
+ for iface, _ in config['interface'].items():
+ # Create lcp
+ if iface not in Section.interfaces():
+ vpp_control.lcp_pair_add(iface, iface)
# update interface config
#e = EthernetIf(iface)
--
cgit v1.2.3