T7789: T7661: VPP prevent failing to set XDP driver on clouds

Some cloud NICs (ena, gve) fail to load XDP if all RX queues are configured. To
avoid this, we limit the number of queues to half of the maximum supported by the driver.

1 files changed, 68 insertions, 17 deletions

diff --git a/src/conf_mode/vpp.py b/src/conf_mode/vpp.py
index 61b10406f..bd7ff173f 100755
--- a/src/conf_mode/vpp.py
+++ b/src/conf_mode/vpp.py
@@ -112,6 +112,9 @@ drivers_support_interrupt: dict[str, list] = {
     'virtio_net': ['xdp'],
 }
 
+# drivers that require changing channels (half the maximum number of RX/TX queues)
+ethtool_channels_change_drv: list[str] = ['ena', 'gve']
+
 
 def _load_module(module_name: str):
     """
@@ -171,6 +174,25 @@ def _normalize_buffers(config: dict):
         config['settings']['buffers']['buffers_per_numa'] = str(buffers)
 
 
+def _get_max_xdp_rx_queues(config: dict):
+    """
+    Count max number of RX queues for XDP driver
+    - If the interface driver is in `ethtool_channels_change_drv`
+      only half of the available queues are used (to avoid NIC issues)
+    - For other interface drivers the full number of queues is returned.
+    - If neither `rx` nor `combined` is set, return 1.
+    """
+    for key in ('rx', 'combined'):
+        value = config['channels'].get(key)
+        if value:
+            if config['original_driver'] in ethtool_channels_change_drv:
+                return max(1, int(value) // 2)
+            else:
+                return int(value)
+
+    return 1
+
+
 def get_config(config=None):
     # use persistent config to store interfaces data between executions
     # this is required because some interfaces after they are connected
@@ -270,6 +292,20 @@ def get_config(config=None):
         _normalize_buffers(effective_config)
         config['effective'] = effective_config
 
+    # Save important info about all interfaces that cannot be retrieved later
+    # Add new interfaces (only if they are first time seen in a config)
+    for iface, iface_config in config.get('settings', {}).get('interface', {}).items():
+        if iface not in effective_config.get('settings', {}).get('interface', {}):
+            eth_ifaces_persist[iface] = {
+                'original_driver': EthtoolGDrvinfo(iface).driver,
+            }
+            eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface)
+            eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface)
+            eth_ifaces_persist[iface]['channels'] = control_host.get_eth_channels(iface)
+
+    # Return to config dictionary
+    config['persist_config'] = eth_ifaces_persist
+
     if 'settings' in config:
         if 'interface' in config['settings']:
             for iface, iface_config in config['settings']['interface'].items():
@@ -330,7 +366,8 @@ def get_config(config=None):
                         'txq_size': int(iface_config['xdp_options']['tx_queue_size']),
                     }
                     if iface_config['xdp_options']['num_rx_queues'] == 'all':
-                        xdp_api_params['rxq_num'] = 0
+                        # 65535 is used as special value to request all available queues
+                        xdp_api_params['rxq_num'] = 65535
                     else:
                         xdp_api_params['rxq_num'] = int(
                             iface_config['xdp_options']['num_rx_queues']
@@ -363,18 +400,6 @@ def get_config(config=None):
                     iface_filter_eth(conf, iface)
                 set_dependents(dependency, conf, iface)
 
-    # Save important info about all interfaces that cannot be retrieved later
-    # Add new interfaces (only if they are first time seen in a config)
-    for iface, iface_config in config.get('settings', {}).get('interface', {}).items():
-        if iface not in effective_config.get('settings', {}).get('interface', {}):
-            eth_ifaces_persist[iface] = {
-                'original_driver': config['settings']['interface'][iface][
-                    'kernel_module'
-                ],
-            }
-            eth_ifaces_persist[iface]['bus_id'] = control_host.get_bus_name(iface)
-            eth_ifaces_persist[iface]['dev_id'] = control_host.get_dev_id(iface)
-
     # kernel-interfaces dependency
     if effective_config.get('kernel_interfaces'):
         for iface in config.get('kernel_interfaces', {}):
@@ -399,9 +424,6 @@ def get_config(config=None):
         config['pppoe_ifaces'] = pppoe_map_ifaces
         set_dependents('pppoe_server', conf)
 
-    # Return to config dictionary
-    config['persist_config'] = eth_ifaces_persist
-
     return config
 
 
@@ -493,6 +515,14 @@ def verify(config):
                 )
         if iface_config['driver'] == 'xdp' and 'xdp_options' in iface_config:
             if iface_config['xdp_options']['num_rx_queues'] != 'all':
+                rx_queues = iface_config['xdp_api_params']['rxq_num']
+                max_rx_queues = _get_max_xdp_rx_queues(config['persist_config'][iface])
+                if rx_queues > max_rx_queues:
+                    raise ConfigError(
+                        f'Maximum supported number of RX queues for interface {iface} is {max_rx_queues}. '
+                        f'Please set "xdp-options num-rx-queues" to {max_rx_queues} or fewer'
+                    )
+
                 Warning(f'Not all RX queues will be connected to VPP for {iface}!')
 
         if iface_config['driver'] == 'xdp' and 'dpdk_options' in iface_config:
@@ -601,11 +631,13 @@ def initialize_interface(iface, driver, iface_config) -> None:
         iface_new_name: str = control_host.get_eth_name(iface_config['dev_id'])
         control_host.rename_iface(iface_new_name, iface)
 
-    # XDP - rename an interface, disable promisc and XDP
+    # XDP - rename an interface, disable promisc and XDP, set original channels
     if driver == 'xdp':
         control_host.set_promisc(f'defunct_{iface}', 'off')
         control_host.rename_iface(f'defunct_{iface}', iface)
         control_host.xdp_remove(iface)
+        if iface_config['original_driver'] in ethtool_channels_change_drv:
+            control_host.set_eth_channels(iface, iface_config['channels'])
 
     # Rename Mellanox NIC to a normal name
     try:
@@ -720,6 +752,25 @@ def apply(config):
                 # add XDP interfaces
                 if iface_config['driver'] == 'xdp':
                     control_host.rename_iface(iface, f'defunct_{iface}')
+
+                    # Some cloud NICs fail to load XDP if all RX queues are configured. To avoid this,
+                    # we limit the number of queues to half of the maximum supported by the driver.
+                    if (
+                        config['persist_config'][iface]['original_driver']
+                        in ethtool_channels_change_drv
+                    ):
+                        max_rx_queues = _get_max_xdp_rx_queues(
+                            config['persist_config'][iface]
+                        )
+                        channels_orig = config['persist_config'][iface]['channels']
+                        channels = {}
+                        if channels_orig.get('rx'):
+                            channels = {'rx': max_rx_queues, 'tx': max_rx_queues}
+                        if channels_orig.get('combined'):
+                            channels['combined'] = max_rx_queues
+                        if channels:
+                            control_host.set_eth_channels(f'defunct_{iface}', channels)
+
                     vpp_control.xdp_iface_create(
                         host_if=f'defunct_{iface}',
                         name=iface,