diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/check-qemu-install | 278 |
1 files changed, 119 insertions, 159 deletions
diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install index 373587e4..7e03623e 100755 --- a/scripts/check-qemu-install +++ b/scripts/check-qemu-install @@ -157,7 +157,7 @@ parser.add_argument('--configtest', help='Execute load/commit config tests', action='store_true', default=False) parser.add_argument('--tpmtest', help='Execute TPM encrypted config tests', action='store_true', default=False) -parser.add_argument('--ifnametest', help='Execute interface naming/hw-id persistence tests', +parser.add_argument('--ifnametest', help='Execute interface naming persistence tests', action='store_true', default=False) parser.add_argument('--secure-boot-test', help='Execute Secure Boot tests (enrolls MOK only if custom ' 'signing certificates exist in data/certificates)', @@ -312,14 +312,14 @@ def get_qemu_cmd(name, enable_uefi, disk_img, raid=None, iso_img=None, tpm=False -uuid {uuid} \ {enable_kvm} \ -monitor unix:/tmp/qemu-monitor-socket-{disk_img},server,nowait \ - -netdev user,id=n0,net=192.0.2.0/24,dhcpstart=192.0.2.101,dns=192.0.2.10 -device virtio-net-pci,netdev=n0,mac={macbase}:00,romfile="",host_mtu=1500 \ - -netdev user,id=n1 -device virtio-net-pci,netdev=n1,mac={macbase}:01,romfile="",host_mtu=1500 \ - -netdev user,id=n2 -device virtio-net-pci,netdev=n2,mac={macbase}:02,romfile="",host_mtu=1500 \ - -netdev user,id=n3 -device virtio-net-pci,netdev=n3,mac={macbase}:03,romfile="",host_mtu=1500 \ - -netdev user,id=n4 -device e1000e,netdev=n4,mac={macbase}:04,romfile="" \ - -netdev user,id=n5 -device e1000e,netdev=n5,mac={macbase}:05,romfile="" \ - -netdev user,id=n6 -device vmxnet3,netdev=n6,mac={macbase}:06,romfile="" \ - -netdev user,id=n7 -device vmxnet3,netdev=n7,mac={macbase}:07,romfile="" \ + -netdev user,id=n0,net=192.0.2.0/24,dhcpstart=192.0.2.101,dns=192.0.2.10 -device virtio-net-pci,netdev=n0,mac={macbase}:00,romfile="",bus=pci.0,addr=0x4,host_mtu=1500 \ + -netdev user,id=n1 -device virtio-net-pci,netdev=n1,mac={macbase}:01,romfile="",bus=pci.0,addr=0x5,host_mtu=1500 \ + -netdev user,id=n2 -device virtio-net-pci,netdev=n2,mac={macbase}:02,romfile="",bus=pci.0,addr=0x6,host_mtu=1500 \ + -netdev user,id=n3 -device virtio-net-pci,netdev=n3,mac={macbase}:03,romfile="",bus=pci.0,addr=0x7,host_mtu=1500 \ + -netdev user,id=n4 -device e1000e,netdev=n4,mac={macbase}:04,romfile="",bus=pci.0,addr=0x8 \ + -netdev user,id=n5 -device e1000e,netdev=n5,mac={macbase}:05,romfile="",bus=pci.0,addr=0x9 \ + -netdev user,id=n6 -device vmxnet3,netdev=n6,mac={macbase}:06,romfile="",bus=pci.0,addr=0xa \ + -netdev user,id=n7 -device vmxnet3,netdev=n7,mac={macbase}:07,romfile="",bus=pci.0,addr=0xb \ -device virtio-scsi-pci,id=scsi0 \ {cdrom}{nested_cdrom} \ -drive format={disk_format},file={disk_img},if=none,media=disk,id=drive-hd1,readonly=off \ @@ -716,7 +716,13 @@ def basic_cli_tests(c): if not re.fullmatch(pattern, value): raise Exception(f'Invalid {name} "{value}" read from {flavor_json}') + # These are scraped off a serial console, so they can pick up stray + # control characters. Escape them: they are literal values, never + # patterns, and an unescaped '[' makes re.compile() raise before the + # expect even runs - which reads as a mysterious harness crash rather + # than the scrape problem it actually is. console_dev = re.escape(f'{console_type}{console_num}') + log.debug(f'console from flavor.json: {console_dev!r},{console_speed!r}') # Ensure serial console with kernel option is set in CLI, done automatically by image installer c.sendline('show configuration commands | match "system console"') @@ -761,16 +767,26 @@ def verify_eth_mac_mapping(c, log): raise Exception(f'Interface {ifname} has MAC {macs[ifname]}, expected {expected_mac} - naming race?') log.info('eth0..eth7 MAC mapping verified') -def verify_swapped_hwid_assignment(c, log, mac1, mac2): - """ eth1/eth2 were just explicitly reassigned to the opposite of their - default MAC order (eth1 -> the numerically higher MAC, eth2 -> the - lower one) - an ordinary, unambiguous rightful-owner rename that - must take effect cleanly. This is the precondition for the next - check: an existing box whose interface names do not follow - ascending PCIe/MAC order, exactly like a real, already-provisioned - system (its hw-id came from historical probe-order rescan, not - from this sort). """ - log.info('Verify the swapped eth1/eth2 hw-id assignment took effect') +def ifname_diagnostics(c, mapping_file): + """ Dump everything needed to reconstruct a naming decision after the + fact. Cheap, and the difference between a one-line failure and an + unreproducible one. """ + for command in ('show interfaces ethernet', + 'ip -br link show', + f'cat {mapping_file}', + 'cat /run/vyos-net-name-resolve.json', + 'ls /etc/systemd/network/', + 'show log | match net-name-resolve'): + c.sendline(f'{command} 2>/dev/null || true') + c.expect(op_mode_prompt) + + +def verify_swapped_assignment(c, log, mac1, mac2): + """ eth1 and eth2 were just swapped in the mapping store, so each must + now be on the other's NIC. This proves the store alone decides names, + and sets up the next check: a box whose names do not follow slot + order, exactly like one first named by an older release. """ + log.info('Verify the swapped eth1/eth2 mapping took effect') c.sendline('ip -json link show | jq -r \'.[] | select(.ifname|test("^eth[0-9]+$")) | "\(.ifname) \(.address)"\'') c.expect(op_mode_prompt) lines = [l.strip() for l in c.before.decode(errors='replace').splitlines() if l.strip()] @@ -783,25 +799,28 @@ def verify_swapped_hwid_assignment(c, log, mac1, mac2): if macs.get('eth1') != mac2: raise Exception(f'Interface eth1 has MAC {macs.get("eth1")}, expected {mac2} ' - '- swapped hw-id assignment did not take effect') + '- the mapping store was not applied') if macs.get('eth2') != mac1: raise Exception(f'Interface eth2 has MAC {macs.get("eth2")}, expected {mac1} ' - '- swapped hw-id assignment did not take effect') - log.info('Swapped hw-id assignment confirmed - eth1/eth2 no longer follow ascending MAC order') - -def verify_pending_node_never_gets_wrong_hardware(c, log, ifname, wrong_mac): - """ Regression reported against an earlier PCIe/MAC-sorted replacement - fill: with an unrelated interface fully removed in the same boot - that `ifname`'s hw-id alone was cleared, there is no way to tell - which of the freed candidates is genuinely `ifname`'s own - hardware once its hw-id is gone - so the fix leaves `ifname` - pending (safely unresolved) rather than guessing. `ifname` must - never end up bound to `wrong_mac` - the OTHER freed interface's - hardware - which would silently apply a setting configured on - `ifname` (e.g. address) to a different physical NIC. `ifname` - simply not existing (still pending) is the expected, safe - outcome here, not a failure. """ - log.info(f'Verify {ifname} was never bound to the wrong physical NIC') + '- the mapping store was not applied') + log.info('Mapping store is authoritative - eth1/eth2 no longer follow slot order') + +def verify_interface_keeps_own_hardware(c, log, ifname, expected_mac): + """ An interface that still carries settings (an address) must come back + bound to its OWN hardware after a different, unrelated interface's + configuration is removed - while its own name does not follow slot + order, which is what used to make names get re-derived and two + interfaces end up on each other's cable. + + The check for this could once only be negative ("never the wrong + MAC"), because leaving the node unresolved was the safe outcome at + the time - so a run where the interface was missing entirely counted + as a pass. Names no longer come from the configuration, so deleting + one cannot put a name up for grabs and the guarantee is absolute: the + interface must be present, and on its own hardware. Assert that + directly - a missing interface is now a failure. + """ + log.info(f'Verify {ifname} still owns its own physical NIC ({expected_mac})') c.sendline('ip -json link show | jq -r \'.[] | select(.ifname|test("^eth[0-9]+$")) | "\(.ifname) \(.address)"\'') c.expect(op_mode_prompt) lines = [l.strip() for l in c.before.decode(errors='replace').splitlines() if l.strip()] @@ -812,12 +831,16 @@ def verify_pending_node_never_gets_wrong_hardware(c, log, ifname, wrong_mac): if len(parts) == 2 and re.fullmatch(r'eth\d+', parts[0]): macs[parts[0]] = parts[1].lower() - if macs.get(ifname) == wrong_mac: - raise Exception(f'Interface {ifname} has MAC {wrong_mac} - bound to the ' - "OTHER freed interface's hardware instead of its own " - f'(a setting configured on {ifname} is now applied to ' - 'the wrong wire)') - log.info(f'{ifname} was not bound to the wrong physical NIC') + if ifname not in macs: + raise Exception(f'Interface {ifname} does not exist after reboot - its ' + 'name must stay bound to its own hardware, not be left ' + 'unresolved') + if macs[ifname] != expected_mac: + raise Exception(f'Interface {ifname} has MAC {macs[ifname]}, expected ' + f'{expected_mac} - it is bound to the wrong physical ' + 'NIC, so settings configured on it now apply to the ' + 'wrong wire') + log.info(f'{ifname} correctly kept its own physical NIC') def _image_update_cli_sequence(c, log, new_image_name, server_bind_host='127.0.0.1', use_vrf=False): """One add-system-image/delete cycle for nested ISO over HTTP (optional Linux VRF + VyOS vrf arg).""" @@ -1422,23 +1445,29 @@ try: c.expect(op_mode_prompt) elif args.ifnametest: - # A missing/deleted hw-id binding, or a fully deleted interface - # config, must not change the eth0..eth7 <-> MAC mapping after - # the next reboot (regression check for the boot-time naming - # race). Deliberately run as two INDEPENDENT reboots rather than - # one combined one: a pending node (hw-id cleared, node kept) - # only ever recovers its own hardware automatically when it's - # the sole candidate of its type this boot - if a different, - # unrelated interface's config were ALSO fully removed in the - # same boot, the two freed NICs become genuinely indistinguishable - # candidates and neither auto-resolves (see - # verify_pending_node_never_gets_wrong_hardware() below for why - # guessing there is unsafe). Testing each mechanism in its own - # boot is what each can actually guarantee. - log.info('Running interface naming/hw-id persistence tests') - del_idx, hwid_idx = random.sample(range(8), 2) - - log.info(f'Deleting eth{del_idx} entirely') + # Interface names come from the mapping store, which is keyed on where + # the hardware sits. Nothing in the configuration decides a name any + # more, so the guarantees under test are: + # + # 1. deleting an interface's configuration does not free its name + # 2. the store is authoritative - edit it and names follow + # 3. losing the store entirely reproduces the same names from + # hardware alone, because bootstrap order is a property of the + # machine and not of probe timing + # + # The NICs are pinned to ascending PCI slots with ascending MACs, so + # eth0..eth7 must map to macbase:00..07 whenever naming is re-derived. + log.info('Running interface naming persistence tests') + mapping_file = '/config/interface-mapping.json' + + log.info('Verify the mapping store was created on first boot') + c.sendline(f'cat {mapping_file}') + c.expect(op_mode_prompt) + if 'interfaces' not in c.before.decode(errors='replace'): + raise Exception(f'{mapping_file} missing or empty after first boot') + + del_idx = random.choice(range(8)) + log.info(f'Deleting eth{del_idx} configuration entirely') c.sendline('configure') c.expect(cfg_mode_prompt) c.sendline(f'delete interfaces ethernet eth{del_idx}') @@ -1450,88 +1479,42 @@ try: c.sendline('exit') c.expect(op_mode_prompt) - log.info('Rebooting to verify the fully deleted interface backfills its own gap') + log.info('Rebooting - a deleted configuration must not free the name') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) - - log.info('Collecting interface naming diagnostics') - c.sendline('show configuration commands | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('show interfaces ethernet') - c.expect(op_mode_prompt) - c.sendline('ip link show') - c.expect(op_mode_prompt) - c.sendline('show log | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true') - c.expect(op_mode_prompt) - c.sendline('show log kernel | match "eth"') - c.expect(op_mode_prompt) - + ifname_diagnostics(c, mapping_file) verify_eth_mac_mapping(c, log) - log.info(f"Removing hw-id only on eth{hwid_idx}, keeping its node") - c.sendline('configure') - c.expect(cfg_mode_prompt) - c.sendline(f'delete interfaces ethernet eth{hwid_idx} hw-id') - c.expect(cfg_mode_prompt) - c.sendline('commit') - c.expect(cfg_mode_prompt) - c.sendline('save') - c.expect(cfg_mode_prompt) - c.sendline('exit') + # The store is the only thing that decides names, so editing it must + # be enough to move an interface onto different hardware - and the + # change must stick across the reboot after that. + log.info('Swapping eth1 and eth2 in the mapping store') + c.sendline( + "sudo python3 -c \"import json;" + f"p='{mapping_file}';d=json.load(open(p));" + "i=d['interfaces'];i['eth1'],i['eth2']=i['eth2'],i['eth1'];" + "json.dump(d,open(p,'w'))\"") c.expect(op_mode_prompt) - log.info('Rebooting to verify the pending node reclaims its own hardware') + log.info('Rebooting to verify the store is authoritative') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) + ifname_diagnostics(c, mapping_file) - log.info('Collecting interface naming diagnostics') - c.sendline('show configuration commands | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('show interfaces ethernet') - c.expect(op_mode_prompt) - c.sendline('ip link show') - c.expect(op_mode_prompt) - c.sendline('show log | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true') - c.expect(op_mode_prompt) - c.sendline('show log kernel | match "eth"') - c.expect(op_mode_prompt) - - verify_eth_mac_mapping(c, log) - - # Second, separate regression: a settings-bearing node must - # reclaim its OWN hardware, not whatever an ascending PCIe/MAC - # sort hands it, when an unrelated interface is fully removed in - # the same boot. The harness's own MACs are sequential - # (macbase:00..07), so a random del_idx/hwid_idx pair here would - # never expose this - a fresh install's initial bootstrap already - # sorts names and MACs together. Deterministically invert eth1's - # and eth2's hw-id first, so their names no longer follow - # ascending MAC order - exactly like any already-provisioned box, - # whose hw-id came from historical probe-order rescan rather than - # this sort. - log.info('Simulating an existing box whose interface names do not ' - 'follow PCIe/MAC order, then replacing one NIC while a ' - 'different, unrelated interface is fully removed') mac1 = f'{macbase}:01'.lower() mac2 = f'{macbase}:02'.lower() + verify_swapped_assignment(c, log, mac1, mac2) + # eth1 and eth2 no longer follow slot order, exactly like a box first + # named by an older release. Removing an unrelated interface's + # configuration in that state used to be enough to re-derive names and + # put two interfaces on each other's cable. + log.info("Deleting eth1's configuration while eth2 is off slot order") c.sendline('configure') c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth1 hw-id') - c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth2 hw-id') - c.expect(cfg_mode_prompt) - c.sendline(f"set interfaces ethernet eth1 hw-id '{mac2}'") - c.expect(cfg_mode_prompt) - c.sendline(f"set interfaces ethernet eth2 hw-id '{mac1}'") - c.expect(cfg_mode_prompt) - c.sendline("set interfaces ethernet eth2 address '10.99.2.1/24'") + c.sendline('delete interfaces ethernet eth1') c.expect(cfg_mode_prompt) c.sendline('commit') c.expect(cfg_mode_prompt) @@ -1540,48 +1523,25 @@ try: c.sendline('exit') c.expect(op_mode_prompt) - log.info('Rebooting to establish the swapped hw-id assignment as ' - "this box's existing state") + log.info('Rebooting to check eth2 stays on its own physical NIC') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) - - verify_swapped_hwid_assignment(c, log, mac1, mac2) - - log.info('Fully removing eth1 while only clearing eth2\'s hw-id, ' - 'keeping its address - the exact shape reported to ' - 'silently bind a configured node to the wrong physical NIC') - c.sendline('configure') - c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth1') - c.expect(cfg_mode_prompt) - c.sendline('delete interfaces ethernet eth2 hw-id') - c.expect(cfg_mode_prompt) - c.sendline('commit') - c.expect(cfg_mode_prompt) - c.sendline('save') - c.expect(cfg_mode_prompt) - c.sendline('exit') + ifname_diagnostics(c, mapping_file) + verify_interface_keeps_own_hardware(c, log, 'eth2', mac1) + + # Finally, lose the store. Names must come back identical, because + # bootstrap order is derived from PCI topology rather than from the + # order drivers happened to finish probing. + log.info('Deleting the mapping store to verify names are reproducible') + c.sendline(f'sudo rm -f {mapping_file} /etc/systemd/network/*-vyos-*.link') c.expect(op_mode_prompt) - - log.info('Rebooting to check eth2 is never bound to the wrong physical NIC') c.sendline('reboot now') waitForLogin(c, log) loginVM(c, log) + ifname_diagnostics(c, mapping_file) + verify_eth_mac_mapping(c, log) - log.info('Collecting mac-order-mismatch diagnostics') - c.sendline('show configuration commands | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('show interfaces ethernet') - c.expect(op_mode_prompt) - c.sendline('ip link show') - c.expect(op_mode_prompt) - c.sendline('show log | match "hw-id"') - c.expect(op_mode_prompt) - c.sendline('cat /run/vyos-net-name-resolve.json 2>/dev/null || true') - c.expect(op_mode_prompt) - - verify_pending_node_never_gets_wrong_hardware(c, log, 'eth2', mac2) elif args.raid: # Verify RAID subsystem - by deleting a disk and re-create the array |
