Merge pull request #4747 from natali-rs1985/T7789

T7789: T7661: VPP prevent failing to load XDP in clouds (ena/gve drivers)

1 files changed, 68 insertions, 17 deletions

diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py
index 61b10406f..bd7ff173f 100755
--- a/src/conf_mode/vpp.py
+++ b/src/conf_mode/vpp.py
@@ -112,6 +112,9 @@ drivers_support_interrupt: dict[str, list] = {
     'virtio_net': ['xdp'],
 }
 
+# drivers that require changing channels (half the maximum number of RX/TX queues)
+ethtool_channels_change_drv: list[str] = ['ena', 'gve']
+
 
 def _load_module(module_name: str):
     """
@@ -171,6 +174,25 @@ def _normalize_buffers(config: dict):
         config['settings']['buffers']['buffers_per_numa'] = str(buffers)
 
 
+def _get_max_xdp_rx_queues(config: dict):
+    """
+    Count max number of RX queues for XDP driver
+    - If the interface driver is in `ethtool_channels_change_drv`
+      only half of the available queues are used (to avoid NIC issues)
+    - For other interface drivers the full number of queues is returned.
+    - If neither `rx` nor `combined` is set, return 1.
+    """
+    for key in ('rx', 'combined'):
+        value = config['channels'].get(key)
+        if value:
+            if config['original_driver'] in ethtool_channels_change_drv:
+                return max(1, int(value) // 2)
+            else:
+                return int(value)
+
+    return 1
+
+
 def get_config(config=None):
     # use persistent config to store interfaces data between executions
     # this is required because some interfaces after they are connected
@@ -270,6 +292,20 @@ def get_config(config=None):
         _normalize_buffers(effective_config)
         config['effective'] = effective_config
 
+    # Save important info about all interfaces that cannot be retrieved later
+    # Add new interfaces (only if they are first time seen in a config)
+    for iface, iface_config in config.get('settings', {}).get('interface', {}).items():
+        if iface not in effective_config.get('settings', {}).get('interface', {}):
+            eth_ifaces_persist[iface] = {
+                'original_driver': EthtoolGDrvinfo(iface).driver,
+            }
+            eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface)
+            eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface)
+            eth_ifaces_persist[iface]['channels'] = control_host.get_eth_channels(iface)
+
+    # Return to config dictionary
+    config['persist_config'] = eth_ifaces_persist
+
     if 'settings' in config:
         if 'interface' in config['settings']:
             for iface, iface_config in config['settings']['interface'].items():
@@ -330,7 +366,8 @@ def get_config(config=None):
                         'txq_size': int(iface_config['xdp_options']['tx_queue_size']),
                     }
                     if iface_config['xdp_options']['num_rx_queues'] == 'all':
-                        xdp_api_params['rxq_num'] = 0
+                        # 65535 is used as special value to request all available queues
+                        xdp_api_params['rxq_num'] = 65535
                     else:
                         xdp_api_params['rxq_num'] = int(
                             iface_config['xdp_options']['num_rx_queues']
@@ -363,18 +400,6 @@ def get_config(config=None):
                     iface_filter_eth(conf, iface)
                 set_dependents(dependency, conf, iface)
 
-    # Save important info about all interfaces that cannot be retrieved later
-    # Add new interfaces (only if they are first time seen in a config)
-    for iface, iface_config in config.get('settings', {}).get('interface', {}).items():
-        if iface not in effective_config.get('settings', {}).get('interface', {}):
-            eth_ifaces_persist[iface] = {
-                'original_driver': config['settings']['interface'][iface][
-                    'kernel_module'
-                ],
-            }
-            eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface)
-            eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface)
-
     # kernel-interfaces dependency
     if effective_config.get('kernel_interfaces'):
         for iface in config.get('kernel_interfaces', {}):
@@ -399,9 +424,6 @@ def get_config(config=None):
         config['pppoe_ifaces'] = pppoe_map_ifaces
         set_dependents('pppoe_server', conf)
 
-    # Return to config dictionary
-    config['persist_config'] = eth_ifaces_persist
-
     return config
 
 
@@ -493,6 +515,14 @@ def verify(config):
                 )
         if iface_config['driver'] == 'xdp' and 'xdp_options' in iface_config:
             if iface_config['xdp_options']['num_rx_queues'] != 'all':
+                rx_queues = iface_config['xdp_api_params']['rxq_num']
+                max_rx_queues = _get_max_xdp_rx_queues(config['persist_config'][iface])
+                if rx_queues > max_rx_queues:
+                    raise ConfigError(
+                        f'Maximum supported number of RX queues for interface {iface} is {max_rx_queues}. '
+                        f'Please set "xdp-options num-rx-queues" to {max_rx_queues} or fewer'
+                    )
+
                 Warning(f'Not all RX queues will be connected to VPP for {iface}!')
 
         if iface_config['driver'] == 'xdp' and 'dpdk_options' in iface_config:
@@ -601,11 +631,13 @@ def initialize_interface(iface, driver, iface_config) -> None:
         iface_new_name: str = control_host.get_eth_name(iface_config['dev_id'])
         control_host.rename_iface(iface_new_name, iface)
 
-    # XDP - rename an interface, disable promisc and XDP
+    # XDP - rename an interface, disable promisc and XDP, set original channels
     if driver == 'xdp':
         control_host.set_promisc(f'defunct_{iface}', 'off')
         control_host.rename_iface(f'defunct_{iface}', iface)
         control_host.xdp_remove(iface)
+        if iface_config['original_driver'] in ethtool_channels_change_drv:
+            control_host.set_eth_channels(iface, iface_config['channels'])
 
     # Rename Mellanox NIC to a normal name
     try:
@@ -720,6 +752,25 @@ def apply(config):
                 # add XDP interfaces
                 if iface_config['driver'] == 'xdp':
                     control_host.rename_iface(iface, f'defunct_{iface}')
+
+                    # Some cloud NICs fail to load XDP if all RX queues are configured. To avoid this,
+                    # we limit the number of queues to half of the maximum supported by the driver.
+                    if (
+                        config['persist_config'][iface]['original_driver']
+                        in ethtool_channels_change_drv
+                    ):
+                        max_rx_queues = _get_max_xdp_rx_queues(
+                            config['persist_config'][iface]
+                        )
+                        channels_orig = config['persist_config'][iface]['channels']
+                        channels = {}
+                        if channels_orig.get('rx'):
+                            channels = {'rx': max_rx_queues, 'tx': max_rx_queues}
+                        if channels_orig.get('combined'):
+                            channels['combined'] = max_rx_queues
+                        if channels:
+                            control_host.set_eth_channels(f'defunct_{iface}', channels)
+
                     vpp_control.xdp_iface_create(
                         host_if=f'defunct_{iface}',
                         name=iface,