diff options
| author | Christian Breunig <christian@breunig.cc> | 2026-10-31 11:51:02 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-10-31 11:51:02 +0100 |
| commit | 140b1c1a89431f28ee2401255e006d1f4f516af2 (patch) | |
| tree | 2763b7d6af2fcf4e366c9fc4dfe878e87e0d4bd3 /scripts | |
| parent | 4a0045b21845c4f5e1266ecfdc6d29588764787c (diff) | |
| parent | 5093c2eab0d0963e68f5850d100daa6a3a2d55c8 (diff) | |
| download | vyos-build-140b1c1a89431f28ee2401255e006d1f4f516af2.tar.gz vyos-build-140b1c1a89431f28ee2401255e006d1f4f516af2.zip | |
Merge pull request #1305 from c-po/T3871-ifname-store
T3871: bind interface names to hardware slots instead of hw-id
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/check-qemu-install | 777 |
1 files changed, 604 insertions, 173 deletions
diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install index 373587e4..f710158b 100755 --- a/scripts/check-qemu-install +++ b/scripts/check-qemu-install @@ -55,6 +55,10 @@ import pexpect EXCEPTION = 0 DISK_IMAGE_EXTENSION = '.raw' +# T3871: interface names are resolved from this store, so dump it on every +# testrun - it is what makes a naming failure reproducible after the fact +IFNAME_MAPPING_FILE = '/config/interface-mapping.json' + # Nested installer-ISO QEMU test: mkisofs payload (see --nested-installer-iso-test) NESTED_ISO_DATA_DIR = 'nested_iso_data' NESTED_INSTALLER_PAYLOAD_ISO = 'nested_installer_payload.iso' @@ -69,11 +73,18 @@ NESTED_HTTP_VRF_TABLE = '42042' NESTED_HTTP_VRF_ADDR = '198.51.100.99' # RFC 5737 TEST-NET-3: bind address for http.server inside VRF NESTED_HTTP_MOUNT_POINT = '/mnt/nested_installer_iso' +# RFC7042 section 2.1.2 MAC addresses used for documentation +macbase = '00:00:5E:00:53' + # Cloud-Init data CI_CONFIG_ARGS = { 'hostname': 'vyos-CLOUD-INIT', + 'eth0_mac': f'{macbase}:02'.lower(), + 'eth1_mac': f'{macbase}:00'.lower(), + 'eth2_mac': f'{macbase}:01'.lower(), 'eth0_ipv4': '192.0.2.1/25', 'eth0_ipv6': '2001:db8::1/64', + 'eth2_ipv4': '198.51.100.1/24', 'default_nexthop' : '192.0.2.126', 'default_nexthop6' : '2001:db8::ffff', } @@ -82,8 +93,9 @@ tpm_folder = '/tmp/vyos_tpm_test' tpm_sock = f'{tpm_folder}/swtpm-sock' qemu_name = 'VyOS-QEMU' -# RFC7042 section 2.1.2 MAC addresses used for documentation -macbase = '00:00:5E:00:53' +# NICs wired up at boot by get_qemu_cmd() - the VMware variant labels +# every one of them +VMWARE_PORT_LABEL_COUNT = 8 test_timeout = 5 *3600 # 5 hours (in seconds) to complete individual testcases @@ -157,7 +169,10 @@ parser.add_argument('--configtest', help='Execute load/commit config tests', action='store_true', default=False) parser.add_argument('--tpmtest', help='Execute TPM encrypted config tests', action='store_true', default=False) -parser.add_argument('--ifnametest', help='Execute interface naming/hw-id persistence tests', +parser.add_argument('--ifnametest', help='Execute interface naming persistence tests', + action='store_true', default=False) +parser.add_argument('--vmware', help='Present the VM as VMware hardware (DMI vendor and ' + 'firmware port labels)', action='store_true', default=False) parser.add_argument('--secure-boot-test', help='Execute Secure Boot tests (enrolls MOK only if custom ' 'signing certificates exist in data/certificates)', @@ -236,7 +251,8 @@ def _kvm_exists(): return os.path.exists("/dev/kvm") def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False, - vnc_enabled=False, secure_boot=False, nested_cdrom_iso=None): + vnc_enabled=False, secure_boot=False, nested_cdrom_iso=None, + vmware=False): uefi = "" uuid = "f48b60b2-e6ad-49ef-9d09-4245d0585e52" accel = ',accel=kvm' if _kvm_exists() else '' @@ -300,6 +316,13 @@ def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False # Set QEmu disk image format - this differs if VyOS was installed via smoketest # or we use an already ewxisting image disk_format = 'qcow2' if args.disk.endswith('.qcow2') else 'raw' + + # A virtio NIC cannot carry a firmware port label: its sysfs device is the + # virtio bus device, not the PCI function the label hangs off. ESXi has no + # virtio anyway, so the VMware variant uses the adapters it does present. + # host_mtu is a virtio option and goes with them. + nic_model = 'vmxnet3' if vmware else 'virtio-net-pci' + nic_opts = '' if vmware else ',host_mtu=1500' cmd = f'{qemu_bin} \ -name "{name}" \ -smp {args.cpu},sockets=1,cores={args.cpu},threads=1 \ @@ -312,14 +335,14 @@ def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False -uuid {uuid} \ {enable_kvm} \ -monitor unix:/tmp/qemu-monitor-socket-{disk_img},server,nowait \ - -netdev user,id=n0,net=192.0.2.0/24,dhcpstart=192.0.2.101,dns=192.0.2.10 -device virtio-net-pci,netdev=n0,mac={macbase}:00,romfile="",host_mtu=1500 \ - -netdev user,id=n1 -device virtio-net-pci,netdev=n1,mac={macbase}:01,romfile="",host_mtu=1500 \ - -netdev user,id=n2 -device virtio-net-pci,netdev=n2,mac={macbase}:02,romfile="",host_mtu=1500 \ - -netdev user,id=n3 -device virtio-net-pci,netdev=n3,mac={macbase}:03,romfile="",host_mtu=1500 \ - -netdev user,id=n4 -device e1000e,netdev=n4,mac={macbase}:04,romfile="" \ - -netdev user,id=n5 -device e1000e,netdev=n5,mac={macbase}:05,romfile="" \ - -netdev user,id=n6 -device vmxnet3,netdev=n6,mac={macbase}:06,romfile="" \ - -netdev user,id=n7 -device vmxnet3,netdev=n7,mac={macbase}:07,romfile="" \ + -netdev user,id=n0,net=192.0.2.0/24,dhcpstart=192.0.2.101,dns=192.0.2.10 -device {nic_model},id=nic0,netdev=n0,mac={macbase}:00,romfile="",bus=pci.0,addr=0x4{nic_opts} \ + -netdev user,id=n1 -device {nic_model},id=nic1,netdev=n1,mac={macbase}:01,romfile="",bus=pci.0,addr=0x5{nic_opts} \ + -netdev user,id=n2 -device {nic_model},id=nic2,netdev=n2,mac={macbase}:02,romfile="",bus=pci.0,addr=0x6{nic_opts} \ + -netdev user,id=n3 -device {nic_model},id=nic3,netdev=n3,mac={macbase}:03,romfile="",bus=pci.0,addr=0x7{nic_opts} \ + -netdev user,id=n4 -device e1000e,id=nic4,netdev=n4,mac={macbase}:04,romfile="",bus=pci.0,addr=0xa \ + -netdev user,id=n5 -device e1000e,id=nic5,netdev=n5,mac={macbase}:05,romfile="",bus=pci.0,addr=0xb \ + -netdev user,id=n6 -device vmxnet3,id=nic6,netdev=n6,mac={macbase}:06,romfile="",bus=pci.0,addr=0xe \ + -netdev user,id=n7 -device vmxnet3,id=nic7,netdev=n7,mac={macbase}:07,romfile="",bus=pci.0,addr=0xf \ -device virtio-scsi-pci,id=scsi0 \ {cdrom}{nested_cdrom} \ -drive format={disk_format},file={disk_img},if=none,media=disk,id=drive-hd1,readonly=off \ @@ -344,6 +367,19 @@ def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False ' -tpmdev emulator,id=tpm0,chardev=chrtpm' \ f' -device {tpm_device},tpmdev=tpm0' + if vmware: + # ESXi is the one platform where the firmware, not the PCI topology, + # decides which adapter the operator calls the first one. Two SMBIOS + # tables reproduce that: type 1 is what /sys/class/dmi/id/sys_vendor + # reports, type 41 is where the kernel gets <device>/label from, which + # is the "Network adapter N" order as vSphere shows it. The comma in + # the vendor string has to reach QEMU doubled or its option parser + # treats it as a separator. + cmd += ' -smbios "type=1,manufacturer=VMware,, Inc.,product=VMware Virtual Platform"' + for index in range(VMWARE_PORT_LABEL_COUNT): + cmd += (f' -smbios type=41,designation=Ethernet{index},kind=ethernet,' + f'instance={index + 1},pcidev=nic{index}') + return cmd def shutdownVM(c, log, message=''): @@ -677,6 +713,9 @@ def basic_cli_tests(c): c.expect(op_mode_prompt) c.sendline('show interfaces') c.expect(op_mode_prompt) + # 2>&1 so a missing store is visible rather than an empty result + c.sendline(f'cat {IFNAME_MAPPING_FILE} 2>&1 || true') + c.expect(op_mode_prompt) # c.sendline('systemd-detect-virt') # c.expect('kvm') # c.expect(op_mode_prompt) @@ -716,7 +755,13 @@ def basic_cli_tests(c): if not re.fullmatch(pattern, value): raise Exception(f'Invalid {name} "{value}" read from {flavor_json}') + # These are scraped off a serial console, so they can pick up stray + # control characters. Escape them: they are literal values, never + # patterns, and an unescaped '[' makes re.compile() raise before the + # expect even runs - which reads as a mysterious harness crash rather + # than the scrape problem it actually is. console_dev = re.escape(f'{console_type}{console_num}') + log.debug(f'console from flavor.json: {console_dev!r},{console_speed!r}') # Ensure serial console with kernel option is set in CLI, done automatically by image installer c.sendline('show configuration commands | match "system console"') @@ -736,12 +781,176 @@ def basic_cli_tests(c): c.expect(rf'set console_type="{re.escape(console_type)}"') c.expect(op_mode_prompt) -def verify_eth_mac_mapping(c, log): - """ NICs are attached with a mix of drivers (virtio/e1000e/vmxnet3, see - get_qemu_cmd()) to cover different naming schemes. Regardless of - driver, udev must always enumerate them in ascending order: eth0 - carries mac0, eth1 mac1, ... up to eth7 mac7 - never scrambled. """ - log.info('Verify eth0..eth7 are enumerated in ascending MAC order') +def qemu_monitor(commands): + """ Feed commands to the QEMU monitor of the running VM. """ + os.system(f"printf '{commands}' | " + f'socat - unix-connect:/tmp/qemu-monitor-socket-{args.disk}') + + +def hotplug_nic(log, index, pci_addr, mac=None): + """ Plug a NIC into the running VM through the QEMU monitor. The slot is + deliberately one that sits between the slots already in use, so + canonical hardware order would place the card in the middle of the + existing interfaces - it must still be named after them. """ + netdev = f'hotplug{index}' + mac = mac or f'{macbase}:{index:02x}'.lower() + log.info(f'Hot-plugging a NIC with MAC {mac} into PCI slot {pci_addr}') + qemu_monitor(f'netdev_add user,id={netdev}\n' + f'device_add virtio-net-pci,id=nic{index},netdev={netdev},' + f'mac={mac},bus=pci.0,addr={pci_addr}\n') + return mac + + +def unplug_nic(log, index, settle=10): + """ Pull a card out of the running VM. PCI removal is asynchronous - the + guest has to acknowledge it - so give it a moment to complete, then + drop the backend too or its id cannot be reused. The boot-time cards + are wired to netdev n0..n7, anything plugged in later to hotplugN. """ + log.info(f'Unplugging the card in nic{index}') + qemu_monitor(f'device_del nic{index}\n') + time.sleep(settle) + netdev = f'n{index}' if index < 8 else f'hotplug{index}' + qemu_monitor(f'netdev_del {netdev}\n') + + +def move_nic(log, index, pci_addr, mac): + """ Take a card out and put the SAME card back in a different slot, which + is what a machine looks like after someone re-seated it. """ + unplug_nic(log, index) + return hotplug_nic(log, index, pci_addr, mac=mac) + + +def verify_hotplug_was_recorded(c, log, mapping_file, ifname, mac, + timeout=60): + """ The hot-plug unit has to run and record the card straight away. If the + udev rule does not match, nothing starts the unit, the card is absent + from the store and no .link file is written for it - and a later + reboot hides all of that, because the boot-time pass picks the card up + anyway. Assert it here, before any reboot can paper over it. """ + log.info(f'Verify {ifname} was recorded without waiting for a reboot') + unit = 'vyos-net-name-resolve-hotplug.service' + + # The card is registered by the Kernel well before the unit udev starts + # has finished recording it, so poll rather than read the map once. No + # reboot happens in between, which is what this is here to prove. + deadline = time.time() + timeout + tmp = 'NONE' + while time.time() < deadline: + tmp = get_cmd_output( + c, f'sudo jq -r \'.hardware["{ifname}"] // "NONE"\' ' + f'{mapping_file}').lower() + if tmp == mac: + break + time.sleep(2) + + if tmp != mac: + started = get_cmd_output( + c, f'systemctl show {unit} -p ExecMainStartTimestamp --value ' + '| grep -c . || true') + if started != '1': + cond = get_cmd_output( + c, f'systemctl show {unit} -p ConditionTimestamp --value ' + '| grep -c . || true') + reason = ('its condition was not met' if cond == '1' else + 'the udev rule did not match the new card') + raise Exception(f'{unit} never ran - {reason}') + # it ran and still did not record the card - show what it decided + c.sendline('cat /run/vyos-net-name-resolve.json 2>&1 || true') + c.expect(op_mode_prompt) + c.sendline(f'sudo journalctl -u {unit} --no-pager | tail -20') + c.expect(op_mode_prompt) + raise Exception(f'{mapping_file} still records {ifname} on {tmp!r} ' + f'{timeout}s after hot-plug, expected {mac}') + + tmp = get_cmd_output( + c, f'sudo jq -r \'.interfaces["{ifname}"] // "NONE"\' {mapping_file}') + if tmp == 'NONE': + raise Exception(f'{ifname} is absent from {mapping_file} after hot-plug') + log.info(f'{ifname} is in the store, on {mac}, and the unit ran') + + +def wait_for_ifname(c, log, ifname, timeout=60): + """ Wait for the Kernel to register the card. This says nothing about the + naming pass udev starts for it - that is still running afterwards. """ + deadline = time.time() + timeout + while time.time() < deadline: + if get_cmd_output(c, f'ip -br link show {ifname} 2>/dev/null | wc -l') == '1': + log.info(f"'{ifname}' appeared") + return + time.sleep(2) + raise Exception(f"'{ifname}' did not appear within {timeout}s after hot-plug") + + +def verify_ifname_has_mac(c, log, ifname, expected_mac): + """ Assert a single interface sits on the NIC we expect. """ + tmp = get_cmd_output( + c, f"ip -json link show {ifname} | jq -r '.[0].address'").lower() + if tmp != expected_mac: + raise Exception(f'Interface {ifname} has MAC {tmp}, ' + f'expected {expected_mac}') + log.info(f"'{ifname}' is on the expected NIC ({expected_mac})") + + +def verify_vmware_firmware(c, log, count=VMWARE_PORT_LABEL_COUNT): + """ The VMware variant of the naming tests is only worth anything if the + guest really believes it runs on VMware. Both signals the store looks + at have to be there: the DMI vendor, and a firmware port label per + NIC. Without the SMBIOS tables the guest is an ordinary machine and + the run would silently repeat the plain naming test. """ + log.info('Verify the guest sees VMware firmware') + vendor = get_cmd_output(c, 'cat /sys/class/dmi/id/sys_vendor') + if 'VMware' not in vendor: + raise Exception(f'DMI vendor is {vendor!r}, expected VMware - the ' + 'SMBIOS type 1 table did not reach the guest') + + c.sendline('for i in /sys/class/net/eth*; do echo "$(basename $i) ' + '$(cat $i/device/label 2>/dev/null)"; done') + c.expect(op_mode_prompt) + labels = {} + for line in ansi_escape.sub('', c.before.decode(errors='replace')).splitlines(): + parts = line.split() + if len(parts) == 2 and re.fullmatch(r'eth\d+', parts[0]): + labels[parts[0]] = parts[1] + + missing = [f'eth{i}' for i in range(count) + if not re.fullmatch(r'Ethernet\d+', labels.get(f'eth{i}', ''))] + if missing: + raise Exception(f'No firmware port label on {", ".join(missing)} - the ' + 'SMBIOS type 41 tables did not reach the guest') + log.info(f'VMware DMI vendor and {count} port labels present') + + +def verify_bootup_scripts(c, log): + """ The pre/post-config hooks normally ship in the config directory, but a + config volume mounted over it has none - the encrypted one, for + instance - and the boot then copies them out of the running image. + Assert the hooks are usable either way, and that the copy, where it + ran, did not quietly go wrong: it used to be handed a split path and + try to copy a directory. The mode is not asserted - the packaged + scripts and the restored ones differ, and both are fine. """ + log.info('Verify the bootup hook scripts are in place') + for script in ('vyos-preconfig-bootup.script', + 'vyos-postconfig-bootup.script'): + path = f'/opt/vyatta/etc/config/scripts/{script}' + tmp = get_cmd_output(c, f'sudo test -x {path} && echo OK || echo NO') + if tmp != 'OK': + raise Exception(f'{script} is missing or not executable') + + tmp = get_cmd_output( + c, 'journalctl -b --no-pager | grep -c "omitting directory" || true') + if tmp != '0': + raise Exception(f'the boot logged {tmp} "omitting directory" errors - ' + 'a bootup script was copied from a split path') + log.info('Both bootup hook scripts are usable, nothing was mis-copied') + + +def verify_eth_mac_mapping(c, log, count=8): + """ NICs are attached with a mix of drivers (see get_qemu_cmd()) to cover + different naming schemes. Regardless of driver, udev must always + enumerate them in ascending order: eth0 carries mac0, eth1 mac1, ... - + never scrambled. Hot-plugged cards extend the same run, so 'count' + covers them too. """ + log.info(f'Verify eth0..eth{count - 1} are enumerated in ascending MAC order') c.sendline('ip -json link show | jq -r \'.[] | select(.ifname|test("^eth[0-9]+$")) | "\(.ifname) \(.address)"\'') c.expect(op_mode_prompt) lines = [l.strip() for l in c.before.decode(errors='replace').splitlines() if l.strip()] @@ -752,25 +961,137 @@ def verify_eth_mac_mapping(c, log): if len(parts) == 2 and re.fullmatch(r'eth\d+', parts[0]): macs[parts[0]] = parts[1].lower() - for i in range(8): + for i in range(count): ifname = f'eth{i}' expected_mac = f'{macbase}:{i:02x}'.lower() if ifname not in macs: raise Exception(f'Interface {ifname} not found on installed system') if macs[ifname] != expected_mac: raise Exception(f'Interface {ifname} has MAC {macs[ifname]}, expected {expected_mac} - naming race?') - log.info('eth0..eth7 MAC mapping verified') - -def verify_swapped_hwid_assignment(c, log, mac1, mac2): - """ eth1/eth2 were just explicitly reassigned to the opposite of their - default MAC order (eth1 -> the numerically higher MAC, eth2 -> the - lower one) - an ordinary, unambiguous rightful-owner rename that - must take effect cleanly. This is the precondition for the next - check: an existing box whose interface names do not follow - ascending PCIe/MAC order, exactly like a real, already-provisioned - system (its hw-id came from historical probe-order rescan, not - from this sort). """ - log.info('Verify the swapped eth1/eth2 hw-id assignment took effect') + log.info(f'eth0..eth{count - 1} MAC mapping verified') + +def verify_radio_has_physical_device(c, log): + """ A radio is not in the mapping file - every interface on a phy carries + that phy's address, so there is nothing there to anchor a name to. The + rescan has to find it on the system instead, or the interface never + gets a node and 'interfaces wireless' shows nothing at all. """ + log.info('Verify a radio present at boot was given its physical-device') + c.sendline('for i in /sys/class/net/wlan*; do [ -e "$i" ] && ' + 'echo "$(basename $i) $(basename $(readlink $i/phy80211))"; done') + c.expect(op_mode_prompt) + + radios = {} + for line in ansi_escape.sub('', c.before.decode(errors='replace')).splitlines(): + parts = line.split() + if len(parts) == 2 and re.fullmatch(r'wlan\d+', parts[0]): + radios[parts[0]] = parts[1] + + # without one the rest of this proves nothing + if not radios: + raise Exception('no radio present after reboot - mac80211_hwsim did ' + 'not load at boot, so the rescan had nothing to find') + + for name, phy in sorted(radios.items()): + tmp = get_cmd_output( + c, f'show configuration commands | match "wireless {name} ' + 'physical-device"') + if phy not in tmp: + raise Exception(f'{name} has no physical-device {phy} in the ' + f'configuration, got {tmp!r}') + log.info(f"'{name}' was given physical-device {phy}") + + +def firmware_diagnostics(c): + """ Dump what the machine claims to be. The store reads the DMI vendor to + decide whether a port label is worth looking at, and the labels + themselves come from SMBIOS type 41 - so when a naming decision looks + wrong, these tables say whether the input was there at all. Static for + the life of the VM, hence once per run. """ + for command in ('cat /sys/class/dmi/id/sys_vendor', + 'cat /sys/class/dmi/id/product_name', + 'sudo dmidecode -t 1', + 'sudo dmidecode -t 41'): + c.sendline(f'{command} 2>/dev/null || true') + c.expect(op_mode_prompt) + + +def verify_cloud_init_mapping(c, log, mapping_file): + """ Which NIC the cloud considers its first one is not visible in the + hardware, so the order the datasource gives has to be recorded in the + mapping file before the naming pass reads it. The seed names the + cards against the order they sit in, so deriving names from the + hardware alone would put the other card on eth0. """ + log.info('Verify the datasource order reached the mapping file') + expected = {'eth0': CI_CONFIG_ARGS['eth0_mac'], + 'eth1': CI_CONFIG_ARGS['eth1_mac'], + 'eth2': CI_CONFIG_ARGS['eth2_mac']} + + for ifname, mac in sorted(expected.items()): + tmp = get_cmd_output(c, f'cat /sys/class/net/{ifname}/address').lower() + if tmp != mac: + raise Exception(f'{ifname} is on {tmp!r}, expected {mac} - the ' + 'names were derived from the hardware instead of ' + 'the order the datasource gave') + log.info(f"'{ifname}' is on {mac}") + + for ifname, mac in sorted(expected.items()): + tmp = get_cmd_output( + c, f'sudo jq -r \'.hardware["{ifname}"] // "NONE"\' ' + f'{mapping_file}').lower() + if tmp != mac: + raise Exception(f'{mapping_file} records {ifname} on {tmp!r}, ' + f'expected {mac}') + + tmp = get_cmd_output( + c, f'sudo jq -r \'.interfaces["{ifname}"] // "NONE"\' ' + f'{mapping_file}') + if tmp == 'NONE': + raise Exception(f'{ifname} is absent from {mapping_file}') + # these cards sit in a PCI slot, so a name anchored to the MAC would + # mean the slot was never resolved + if tmp.startswith('ID_NET_NAME_MAC='): + raise Exception(f'{ifname} is recorded on {tmp!r}, which is its ' + 'MAC rather than the slot it sits in') + log.info(f"'{ifname}' is recorded on {tmp}") + + +def ifname_diagnostics(c, mapping_file): + """ Dump everything needed to reconstruct a naming decision after the + fact. Cheap, and the difference between a one-line failure and an + unreproducible one. """ + # every sysfs attribute and udev property the store keys a name on - + # repeated per checkpoint because hot-plug changes them + sysfs_dump = ( + 'for d in /sys/class/net/*/device; do [ -e "$d" ] || continue; ' + 'n=$(basename $(dirname $d)); ' + 'echo "$n bus=$(basename $(readlink -f $d)) ' + 'vendor=$(cat $d/vendor 2>/dev/null) ' + 'subsystem_vendor=$(cat $d/subsystem_vendor 2>/dev/null) ' + 'label=$(cat $d/label 2>/dev/null)"; done') + udev_dump = ( + 'for i in /sys/class/net/*; do n=$(basename $i); ' + 'udevadm info -q property -p /sys/class/net/$n 2>/dev/null | ' + 'grep -E "^(DEVTYPE|ID_BUS|ID_PATH|ID_NET_NAME_)" | ' + 'sed "s/^/$n /"; done') + + for command in ('show interfaces ethernet', + 'ip -br link show', + sysfs_dump, + udev_dump, + f'cat {mapping_file}', + 'cat /run/vyos-net-name-resolve.json', + 'ls /etc/systemd/network/', + 'show log | match net-name-resolve'): + c.sendline(f'{command} 2>/dev/null || true') + c.expect(op_mode_prompt) + + +def verify_swapped_assignment(c, log, mac1, mac2): + """ eth1 and eth2 were just swapped in the mapping store, so each must + now be on the other's NIC. This proves the store alone decides names, + and sets up the next check: a box whose names do not follow slot + order, exactly like one first named by an older release. """ + log.info('Verify the swapped eth1/eth2 mapping took effect') c.sendline('ip -json link show | jq -r \'.[] | select(.ifname|test("^eth[0-9]+$")) | "\(.ifname) \(.address)"\'') c.expect(op_mode_prompt) lines = [l.strip() for l in c.before.decode(errors='replace').splitlines() if l.strip()] @@ -783,25 +1104,28 @@ def verify_swapped_hwid_assignment(c, log, mac1, mac2): if macs.get('eth1') != mac2: raise Exception(f'Interface eth1 has MAC {macs.get("eth1")}, expected {mac2} ' - '- swapped hw-id assignment did not take effect') + '- the mapping store was not applied') if macs.get('eth2') != mac1: raise Exception(f'Interface eth2 has MAC {macs.get("eth2")}, expected {mac1} ' - '- swapped hw-id assignment did not take effect') - log.info('Swapped hw-id assignment confirmed - eth1/eth2 no longer follow ascending MAC order') - -def verify_pending_node_never_gets_wrong_hardware(c, log, ifname, wrong_mac): - """ Regression reported against an earlier PCIe/MAC-sorted replacement - fill: with an unrelated interface fully removed in the same boot - that `ifname`'s hw-id alone was cleared, there is no way to tell - which of the freed candidates is genuinely `ifname`'s own - hardware once its hw-id is gone - so the fix leaves `ifname` - pending (safely unresolved) rather than guessing. `ifname` must - never end up bound to `wrong_mac` - the OTHER freed interface's - hardware - which would silently apply a setting configured on - `ifname` (e.g. address) to a different physical NIC. `ifname` - simply not existing (still pending) is the expected, safe - outcome here, not a failure. """ - log.info(f'Verify {ifname} was never bound to the wrong physical NIC') + '- the mapping store was not applied') + log.info('Mapping store is authoritative - eth1/eth2 no longer follow slot order') + +def verify_interface_keeps_own_hardware(c, log, ifname, expected_mac): + """ An interface that still carries settings (an address) must come back + bound to its OWN hardware after a different, unrelated interface's + configuration is removed - while its own name does not follow slot + order, which is what used to make names get re-derived and two + interfaces end up on each other's cable. + + The check for this could once only be negative ("never the wrong + MAC"), because leaving the node unresolved was the safe outcome at + the time - so a run where the interface was missing entirely counted + as a pass. Names no longer come from the configuration, so deleting + one cannot put a name up for grabs and the guarantee is absolute: the + interface must be present, and on its own hardware. Assert that + directly - a missing interface is now a failure. + """ + log.info(f'Verify {ifname} still owns its own physical NIC ({expected_mac})') c.sendline('ip -json link show | jq -r \'.[] | select(.ifname|test("^eth[0-9]+$")) | "\(.ifname) \(.address)"\'') c.expect(op_mode_prompt) lines = [l.strip() for l in c.before.decode(errors='replace').splitlines() if l.strip()] @@ -812,12 +1136,16 @@ def verify_pending_node_never_gets_wrong_hardware(c, log, ifname, wrong_mac): if len(parts) == 2 and re.fullmatch(r'eth\d+', parts[0]): macs[parts[0]] = parts[1].lower() - if macs.get(ifname) == wrong_mac: - raise Exception(f'Interface {ifname} has MAC {wrong_mac} - bound to the ' - "OTHER freed interface's hardware instead of its own " - f'(a setting configured on {ifname} is now applied to ' - 'the wrong wire)') - log.info(f'{ifname} was not bound to the wrong physical NIC') + if ifname not in macs: + raise Exception(f'Interface {ifname} does not exist after reboot - its ' + 'name must stay bound to its own hardware, not be left ' + 'unresolved') + if macs[ifname] != expected_mac: + raise Exception(f'Interface {ifname} has MAC {macs[ifname]}, expected ' + f'{expected_mac} - it is bound to the wrong physical ' + 'NIC, so settings configured on it now apply to the ' + 'wrong wire') + log.info(f'{ifname} correctly kept its own physical NIC') def _image_update_cli_sequence(c, log, new_image_name, server_bind_host='127.0.0.1', use_vrf=False): """One add-system-image/delete cycle for nested ISO over HTTP (optional Linux VRF + VyOS vrf arg).""" @@ -931,7 +1259,7 @@ def _image_update_cli_sequence(c, log, new_image_name, server_bind_host='127.0.0 if args.qemu_cmd: tmp = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, iso_img=args.iso, vnc_enabled=args.vnc, secure_boot=args.secure_boot_test, - nested_cdrom_iso=nested_payload_iso_path) + nested_cdrom_iso=nested_payload_iso_path, vmware=args.vmware) os.system(tmp) exit(0) @@ -948,12 +1276,31 @@ if args.cloud_init: # create "empty" meta-data file open(f'{CI_DATA_DIR}/meta-data', mode='w').close() + # Name the cards against the order they sit in, the way a cloud hands out + # an order of its own. It is a three-cycle: not one of them keeps the name + # its slot would give it, so none of this holds unless the order the + # datasource gave reached the mapping file. network_config = """version: 2 ethernets: eth0: + match: + macaddress: {eth0_mac} + set-name: eth0 + dhcp4: false + dhcp6: false + eth1: + match: + macaddress: {eth1_mac} + set-name: eth1 dhcp4: false dhcp6: false -""" + eth2: + match: + macaddress: {eth2_mac} + set-name: eth2 + dhcp4: false + dhcp6: false +""".format(**CI_CONFIG_ARGS) user_data = """#cloud-config vyos_config_commands: @@ -963,6 +1310,7 @@ vyos_config_commands: - delete interfaces ethernet eth0 address 'dhcp' - set interfaces ethernet eth0 address '{eth0_ipv4}' - set interfaces ethernet eth0 address '{eth0_ipv6}' + - set interfaces ethernet eth2 address '{eth2_ipv4}' - set protocols static route 0.0.0.0/0 next-hop '{default_nexthop}' - set protocols static route6 ::/0 next-hop '{default_nexthop6}' """.format(**CI_CONFIG_ARGS) @@ -990,7 +1338,8 @@ try: log.info('Installing system') cmd = get_qemu_cmd(qemu_name, args.uefi, args.disk, raid=diskname_raid, tpm=args.tpmtest, iso_img=args.iso, vnc_enabled=args.vnc, - secure_boot=args.secure_boot_test, nested_cdrom_iso=nested_payload_iso_path) + secure_boot=args.secure_boot_test, nested_cdrom_iso=nested_payload_iso_path, + vmware=args.vmware) log.debug(f'Executing command: {cmd}') c = pexpect.spawn(cmd, logfile=stl, timeout=60) @@ -1026,6 +1375,12 @@ try: c.expect(CI_CONFIG_ARGS['eth0_ipv6']) c.expect(op_mode_prompt) + # the last card of the cycle carries an address of its own, so a name + # on the wrong wire shows up as an address on the wrong wire + c.sendline('ip --json addr show dev eth2 | jq -r \'.[0].addr_info[] | select(.family=="inet") | "\(.local)/\(.prefixlen)"\'') + c.expect(CI_CONFIG_ARGS['eth2_ipv4']) + c.expect(op_mode_prompt) + # Valida default route nexthops c.sendline('ip -4 --json route show default | jq -r ".[0].gateway"') c.expect(CI_CONFIG_ARGS['default_nexthop']) @@ -1035,6 +1390,9 @@ try: c.expect(CI_CONFIG_ARGS['default_nexthop6']) c.expect(op_mode_prompt) + ifname_diagnostics(c, IFNAME_MAPPING_FILE) + verify_cloud_init_mapping(c, log, IFNAME_MAPPING_FILE) + shutdownVM(c, log, 'Powering off system') c.close() raise EarlyExit @@ -1186,6 +1544,11 @@ try: loginVM(c, log) ################################################# + # Bootup hook scripts must be usable on a plain installed system + ################################################# + verify_bootup_scripts(c, log) + + ################################################# # Boot options require a reboot ################################################# if args.huge_page_size and args.huge_page_count: @@ -1329,6 +1692,11 @@ try: loginVM(c, log) + # The encrypted volume is mounted over the config directory, so when it + # stays shut the hooks are absent and the boot has to copy them out of + # the running image. This is the one boot that exercises that path. + verify_bootup_scripts(c, log) + # Test loading config with recovery key c.sendline('encryption load') c.expect('Enter recovery key: ') @@ -1422,23 +1790,33 @@ try: c.expect(op_mode_prompt) elif args.ifnametest: - # A missing/deleted hw-id binding, or a fully deleted interface - # config, must not change the eth0..eth7 <-> MAC mapping after - # the next reboot (regression check for the boot-time naming - # race). Deliberately run as two INDEPENDENT reboots rather than - # one combined one: a pending node (hw-id cleared, node kept) - # only ever recovers its own hardware automatically when it's - # the sole candidate of its type this boot - if a different, - # unrelated interface's config were ALSO fully removed in the - # same boot, the two freed NICs become genuinely indistinguishable - # candidates and neither auto-resolves (see - # verify_pending_node_never_gets_wrong_hardware() below for why - # guessing there is unsafe). Testing each mechanism in its own - # boot is what each can actually guarantee. - log.info('Running interface naming/hw-id persistence tests') - del_idx, hwid_idx = random.sample(range(8), 2) - - log.info(f'Deleting eth{del_idx} entirely') + # Interface names come from the mapping store, which is keyed on where + # the hardware sits. Nothing in the configuration decides a name any + # more, so the guarantees under test are: + # + # 1. deleting an interface's configuration does not free its name + # 2. the store is authoritative - edit it and names follow + # 3. losing the store entirely reproduces the same names from + # hardware alone, because bootstrap order is a property of the + # machine and not of probe timing + # + # The NICs are pinned to ascending PCI slots with ascending MACs, so + # eth0..eth7 must map to macbase:00..07 whenever naming is re-derived. + log.info('Running interface naming persistence tests') + mapping_file = IFNAME_MAPPING_FILE + + firmware_diagnostics(c) + if args.vmware: + verify_vmware_firmware(c, log) + + log.info('Verify the mapping store was created on first boot') + c.sendline(f'cat {mapping_file}') + c.expect(op_mode_prompt) + if 'interfaces' not in c.before.decode(errors='replace'): + raise Exception(f'{mapping_file} missing or empty after first boot') + + del_idx = random.choice(range(8)) + log.info(f'Deleting eth{del_idx} configuration entirely') c.sendline('configure') c.expect(cfg_mode_prompt) c.sendline(f'delete interfaces ethernet eth{del_idx}') @@ -1450,88 +1828,42 @@ try: c.sendline('exit') c.expect(op_mode_prompt) - log.info('Rebooting to verify the fully deleted interface backfills its own gap') + log.info('Rebooting - a deleted configuration must not free the name') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) - - log.info('Collecting interface naming diagnostics') - c.sendline('show configuration commands | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('show interfaces ethernet') - c.expect(op_mode_prompt) - c.sendline('ip link show') - c.expect(op_mode_prompt) - c.sendline('show log | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true') - c.expect(op_mode_prompt) - c.sendline('show log kernel | match "eth"') - c.expect(op_mode_prompt) - + ifname_diagnostics(c, mapping_file) verify_eth_mac_mapping(c, log) - log.info(f"Removing hw-id only on eth{hwid_idx}, keeping its node") - c.sendline('configure') - c.expect(cfg_mode_prompt) - c.sendline(f'delete interfaces ethernet eth{hwid_idx} hw-id') - c.expect(cfg_mode_prompt) - c.sendline('commit') - c.expect(cfg_mode_prompt) - c.sendline('save') - c.expect(cfg_mode_prompt) - c.sendline('exit') + # The store is the only thing that decides names, so editing it must + # be enough to move an interface onto different hardware - and the + # change must stick across the reboot after that. + log.info('Swapping eth1 and eth2 in the mapping store') + c.sendline( + "sudo python3 -c \"import json;" + f"p='{mapping_file}';d=json.load(open(p));" + "i=d['interfaces'];i['eth1'],i['eth2']=i['eth2'],i['eth1'];" + "json.dump(d,open(p,'w'))\"") c.expect(op_mode_prompt) - log.info('Rebooting to verify the pending node reclaims its own hardware') + log.info('Rebooting to verify the store is authoritative') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) + ifname_diagnostics(c, mapping_file) - log.info('Collecting interface naming diagnostics') - c.sendline('show configuration commands | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('show interfaces ethernet') - c.expect(op_mode_prompt) - c.sendline('ip link show') - c.expect(op_mode_prompt) - c.sendline('show log | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true') - c.expect(op_mode_prompt) - c.sendline('show log kernel | match "eth"') - c.expect(op_mode_prompt) - - verify_eth_mac_mapping(c, log) - - # Second, separate regression: a settings-bearing node must - # reclaim its OWN hardware, not whatever an ascending PCIe/MAC - # sort hands it, when an unrelated interface is fully removed in - # the same boot. The harness's own MACs are sequential - # (macbase:00..07), so a random del_idx/hwid_idx pair here would - # never expose this - a fresh install's initial bootstrap already - # sorts names and MACs together. Deterministically invert eth1's - # and eth2's hw-id first, so their names no longer follow - # ascending MAC order - exactly like any already-provisioned box, - # whose hw-id came from historical probe-order rescan rather than - # this sort. - log.info('Simulating an existing box whose interface names do not ' - 'follow PCIe/MAC order, then replacing one NIC while a ' - 'different, unrelated interface is fully removed') mac1 = f'{macbase}:01'.lower() mac2 = f'{macbase}:02'.lower() + verify_swapped_assignment(c, log, mac1, mac2) + # eth1 and eth2 no longer follow slot order, exactly like a box first + # named by an older release. Removing an unrelated interface's + # configuration in that state used to be enough to re-derive names and + # put two interfaces on each other's cable. + log.info("Deleting eth1's configuration while eth2 is off slot order") c.sendline('configure') c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth1 hw-id') - c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth2 hw-id') - c.expect(cfg_mode_prompt) - c.sendline(f"set interfaces ethernet eth1 hw-id '{mac2}'") - c.expect(cfg_mode_prompt) - c.sendline(f"set interfaces ethernet eth2 hw-id '{mac1}'") - c.expect(cfg_mode_prompt) - c.sendline("set interfaces ethernet eth2 address '10.99.2.1/24'") + c.sendline('delete interfaces ethernet eth1') c.expect(cfg_mode_prompt) c.sendline('commit') c.expect(cfg_mode_prompt) @@ -1540,48 +1872,147 @@ try: c.sendline('exit') c.expect(op_mode_prompt) - log.info('Rebooting to establish the swapped hw-id assignment as ' - "this box's existing state") + log.info('Rebooting to check eth2 stays on its own physical NIC') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) - - verify_swapped_hwid_assignment(c, log, mac1, mac2) - - log.info('Fully removing eth1 while only clearing eth2\'s hw-id, ' - 'keeping its address - the exact shape reported to ' - 'silently bind a configured node to the wrong physical NIC') - c.sendline('configure') - c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth1') - c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth2 hw-id') - c.expect(cfg_mode_prompt) - c.sendline('commit') - c.expect(cfg_mode_prompt) - c.sendline('save') - c.expect(cfg_mode_prompt) - c.sendline('exit') + ifname_diagnostics(c, mapping_file) + verify_interface_keeps_own_hardware(c, log, 'eth2', mac1) + + # Finally, lose the store. Names must come back identical, because + # bootstrap order is derived from PCI topology rather than from the + # order drivers happened to finish probing. + log.info('Deleting the mapping store to verify names are reproducible') + c.sendline(f'sudo rm -f {mapping_file} /etc/systemd/network/*-vyos-*.link') c.expect(op_mode_prompt) - - log.info('Rebooting to check eth2 is never bound to the wrong physical NIC') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) + ifname_diagnostics(c, mapping_file) + verify_eth_mac_mapping(c, log) - log.info('Collecting mac-order-mismatch diagnostics') - c.sendline('show configuration commands | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('show interfaces ethernet') - c.expect(op_mode_prompt) - c.sendline('ip link show') - c.expect(op_mode_prompt) - c.sendline('show log | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true') - c.expect(op_mode_prompt) + # A card which appears after boot must be named after the interfaces + # already known, never among them. Both slots used here sit between + # slots already in use, so canonical hardware order would place the + # new cards at eth4 and eth6 - the names in the store are reserved, + # which is what has to stop that from happening. + if platform.machine() in ['amd64', 'x86_64']: + mac8 = hotplug_nic(log, 8, '0x8') + wait_for_ifname(c, log, 'eth8') + verify_hotplug_was_recorded(c, log, mapping_file, 'eth8', mac8) + mac9 = hotplug_nic(log, 9, '0xc') + wait_for_ifname(c, log, 'eth9') + verify_hotplug_was_recorded(c, log, mapping_file, 'eth9', mac9) + + ifname_diagnostics(c, mapping_file) + log.info('Verify the hot-plugged cards were appended, not inserted') + verify_ifname_has_mac(c, log, 'eth8', mac8) + verify_ifname_has_mac(c, log, 'eth9', mac9) + verify_eth_mac_mapping(c, log, count=10) + + # The real corner case: with every card present from the start, + # the one in slot 0x8 is fifth in hardware order. It must still be + # eth8, or five interfaces move onto the wrong cable. + log.info('Rebooting - hot-plugged names must survive re-enumeration') + c.sendline('reboot now') + waitForLogin(c, log) + loginVM(c, log) + ifname_diagnostics(c, mapping_file) + verify_eth_mac_mapping(c, log, count=10) + + # ... and the boot-time rescan must have added a node for them. + # config.boot is what the rescan writes, so assert on that rather + # than on the running config - a node carrying no settings at all + # does not necessarily survive into it. + for ifname in ('eth8', 'eth9'): + tmp = get_cmd_output( + c, f'sudo grep -c "ethernet {ifname}" /config/config.boot') + if tmp == '0': + raise Exception(f'{ifname} was not added to the ' + 'configuration by the interface rescan') + log.info('Hot-plugged interfaces were added to the configuration') + + # A card carries its name with it. Re-seat eth9's card in a free + # slot: the name has to follow the card, not stay behind with the + # position. Matching on the slot alone would hand eth9 - and the + # address configured under it - to whatever turns up there next. + log.info("Moving eth9's card to a different PCI slot") + move_nic(log, 9, '0x9', mac9) + wait_for_ifname(c, log, 'eth9') + c.sendline('reboot now') + waitForLogin(c, log) + loginVM(c, log) + ifname_diagnostics(c, mapping_file) + verify_ifname_has_mac(c, log, 'eth9', mac9) + log.info('eth9 followed its own card to the new slot') + + # ... and the slot it vacated must not lend its name out. A card + # added for an unrelated purpose, dropped into the freed slot, was + # reported from the field coming up on another segment's address. + log.info('Adding an unrelated card to the slot eth9 left behind') + mac10 = f'{macbase}:0a'.lower() + hotplug_nic(log, 10, '0xc', mac=mac10) + wait_for_ifname(c, log, 'eth10') + verify_hotplug_was_recorded(c, log, mapping_file, 'eth10', mac10) + verify_ifname_has_mac(c, log, 'eth9', mac9) + verify_ifname_has_mac(c, log, 'eth10', mac10) + log.info('The newcomer was named separately, eth9 kept its card') + + # With the original card gone for good, nothing distinguishes a + # replacement in the same slot from an unrelated card that landed + # there. The slot decides, and the boot has to say so - that + # warning is the only thing standing between the operator and an + # address quietly moving to another cable. + log.info("Replacing eth8's card with a different one in its slot") + mac11 = f'{macbase}:0b'.lower() + unplug_nic(log, 8) + hotplug_nic(log, 11, '0x8', mac=mac11) + verify_hotplug_was_recorded(c, log, mapping_file, 'eth8', mac11) + verify_ifname_has_mac(c, log, 'eth8', mac11) + + # The pass which takes this decision is the one udev starts for + # the new card, so look at what it reported. Rebooting first would + # find the card already on record and nothing left to say about + # it, which is not the same thing as the case never arising. + tmp = get_cmd_output( + c, 'sudo jq -r \'.matched_by_path["eth8"] // "NONE"\' ' + '/run/vyos-net-name-resolve.json').lower() + if tmp != mac11: + raise Exception('the status file does not report eth8 as ' + f'matched by slot to {mac11}, got {tmp!r}') + tmp = get_cmd_output( + c, 'journalctl -b --no-pager | ' + 'grep -c "eth8. was matched to new hardware" || true') + if tmp == '0': + raise Exception('the replacement was not warned about - eth8 ' + 'was matched to different hardware by slot') + log.info('The replacement was taken, reported and warned about') + + # ... and it has to still be there after a reboot + c.sendline('reboot now') + waitForLogin(c, log) + loginVM(c, log) + ifname_diagnostics(c, mapping_file) + verify_ifname_has_mac(c, log, 'eth8', mac11) + log.info('eth8 kept the replacement card across a reboot') + + # A radio has to be there before the naming pass, not after it, or + # the rescan has nothing to find - which is why this went + # unnoticed: the harness loads the module once the system is + # already up. Last in the testcase on purpose, as the node it + # checks for is committed on the next boot and a radio which + # failed to come up would disturb the naming checks above. + log.info('Making a radio present at boot') + c.sendline('echo mac80211_hwsim | sudo tee ' + '/etc/modules-load.d/vyos-hwsim.conf') + c.expect(op_mode_prompt) + c.sendline('reboot now') + waitForLogin(c, log) + loginVM(c, log) + verify_radio_has_physical_device(c, log) + else: + log.info('Skipping NIC hot-plug tests - not supported on this machine type') - verify_pending_node_never_gets_wrong_hardware(c, log, 'eth2', mac2) elif args.raid: # Verify RAID subsystem - by deleting a disk and re-create the array |
