From 67a6f3e3c9fe830bd5f9878ab39786073c94b13d Mon Sep 17 00:00:00 2001
From: Christian Poessinger <christian@poessinger.com>
Date: Sat, 25 Dec 2021 22:01:46 +0100
Subject: Testsuite: add RAID-1 test case for disk recovery
(cherry picked from commit 986fd18e11ac19470e17ad786d5cc908339ccb43)
---
scripts/check-qemu-install | 161 +++++++++++++++++++++++++++++++++++----------
1 file changed, 127 insertions(+), 34 deletions(-)
diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install
index e1a97c85..5c099a3b 100755
--- a/scripts/check-qemu-install
+++ b/scripts/check-qemu-install
@@ -42,6 +42,7 @@ import random
import traceback
import logging
import re
+import json
from io import BytesIO
from io import StringIO
@@ -65,13 +66,18 @@ parser.add_argument('--logfile', help='Log to file')
parser.add_argument('--uefi', help='Boot using UEFI', action='store_true', default=False)
parser.add_argument('--raid', help='Perform a RAID-1 install', action='store_true', default=False)
parser.add_argument('--no-kvm', help='Disable use of kvm', action='store_true', default=False)
-parser.add_argument('--configd', help='Execute testsuite with config daemon',
- action='store_true', default=False)
+parser.add_argument('--configd', help='Execute testsuite with config daemon', action='store_true',
+ default=False)
+parser.add_argument('--no-interfaces', help='Execute testsuite without interface tests to save time',
+ action='store_true', default=False)
parser.add_argument('--configtest', help='Execute load/commit config tests',
action='store_true', default=False)
args = parser.parse_args()
+with open('data/defaults.json') as f:
+ vyos_defaults = json.load(f)
+
class StreamToLogger(object):
"""
Fake file-like stream object that redirects writes to a logger instance.
@@ -119,7 +125,7 @@ def get_qemu_cmd(name, enable_kvm, enable_uefi, disk_img, raid=None, iso_img=Non
cdrom = ""
if iso_img:
- cdrom = "-boot d -cdrom {}".format(iso_img)
+ cdrom = f"-boot d -cdrom {iso_img}"
# test using half of the available CPUs on the system
cpucount = get_half_cpus()
@@ -143,10 +149,13 @@ def get_qemu_cmd(name, enable_kvm, enable_uefi, disk_img, raid=None, iso_img=Non
-machine accel=kvm \
-uuid {uuid} \
-nographic {cpu} {cdrom} {kvm} \
- -drive format=raw,file={disk_img}'
+ -device virtio-scsi-pci,id=scsi0 \
+ -drive format=raw,file={disk_img},if=none,media=disk,id=drive-hd1,readonly=off \
+ -device scsi-hd,bus=scsi0.0,drive=drive-hd1,id=hd1,bootindex=1'
if raid:
- cmd += f' -drive format=raw,file={raid}'
+ cmd += f' -drive format=raw,file={raid},if=none,media=disk,id=drive-hd2,readonly=off \
+ -device scsi-hd,bus=scsi0.0,drive=drive-hd2,id=hd2,bootindex=2'
return cmd
@@ -157,9 +166,7 @@ def shutdownVM(c, log, message=''):
if message:
log.info(message)
- c.sendline('poweroff')
- c.expect(r'\nAre you sure you want to poweroff this system.*\]')
- c.sendline('Y')
+ c.sendline('poweroff now')
log.info('Shutting down virtual machine')
for i in range(30):
log.info('Waiting for shutdown...')
@@ -243,6 +250,7 @@ if args.raid:
# must be called after the raid disk as args.disk name is altered in the RAID path
gen_disk(args.disk)
+test_timeout = 3 *3600 # 3 hours (in seconds)
try:
#################################################
# Installing image to disk
@@ -341,7 +349,7 @@ try:
# Always load the WiFi simulation module
################################################
c.sendline('sudo modprobe mac80211_hwsim')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
#################################################
# Start/stop config daemon
@@ -350,35 +358,116 @@ try:
c.sendline('sudo systemctl start vyos-configd.service &> /dev/null')
else:
c.sendline('sudo systemctl stop vyos-configd.service &> /dev/null')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
#################################################
# Basic Configmode/Opmode switch
#################################################
log.info('Basic CLI configuration mode test')
c.sendline('configure')
- c.expect(r'vyos@vyos#')
+ c.expect(cfg_mode_prompt)
c.sendline('exit')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
c.sendline('show version')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
+ c.sendline('show version kernel')
+ c.expect(f'{vyos_defaults["kernel_version"]}-{vyos_defaults["kernel_flavor"]}')
+ c.expect(op_mode_prompt)
+ c.sendline('show version frr')
+ c.expect(op_mode_prompt)
c.sendline('show interfaces')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
+ #################################################
+ # Executing test-suite
+ #################################################
if args.raid:
+ # Verify RAID subsystem - by deleting a disk and re-create the array
+ # from scratch
c.sendline('cat /proc/mdstat')
c.expect(op_mode_prompt)
- #################################################
- # Executing test-suite
- #################################################
+ shutdownVM(c, log, f'Shutdown VM and start with empty RAID member "{args.disk}"')
+
+ if os.path.exists(args.disk):
+ os.unlink(args.disk)
+
+ gen_disk(args.disk)
+
+ #################################################
+ # Booting RAID-1 system with one missing disk
+ #################################################
+ log.info('Booting RAID-1 system')
+ cmd = get_qemu_cmd('TESTVM', kvm, args.uefi, args.disk, diskname_raid)
+
+ # We need to swap boot indexes to boot from second harddisk so we can
+ # recreate the RAID on the first disk
+ cmd = cmd.replace('bootindex=1', 'bootindex=X')
+ cmd = cmd.replace('bootindex=2', 'bootindex=1')
+ cmd = cmd.replace('bootindex=X', 'bootindex=2')
+
+ log.debug(f'Executing command: {cmd}')
+ c = pexpect.spawn(cmd, logfile=stl)
+
+
+ #################################################
+ # Logging into VyOS system
+ #################################################
+ try:
+ c.expect('The highlighted entry will be executed automatically in', timeout=10)
+ c.sendline('')
+ except pexpect.TIMEOUT:
+ log.warning('Did not find GRUB countdown window, ignoring')
+
+ loginVM(c, log)
+
+ c.sendline('cat /proc/mdstat')
+ c.expect(op_mode_prompt)
+
+ log.info('Re-format new RAID member')
+ c.sendline('format disk sda like sdb')
+ c.sendline('yes')
+ c.expect(op_mode_prompt)
+
+ log.info('Add member to RAID1 (md0)')
+ c.sendline('add raid md0 member sda1')
+ c.expect(op_mode_prompt)
+
+ log.info('Now we need to wait for re-sync to complete')
+
+ start_time = time.time()
+ timeout = 60
+ while True:
+ if (start_time + timeout) < time.time():
+ break
+ c.sendline('cat /proc/mdstat')
+ c.expect(op_mode_prompt)
+ time.sleep(20)
+
+ # Reboot system with new primary RAID1 disk
+ shutdownVM(c, log, f'Shutdown VM and start from recovered RAID member "{args.disk}"')
+
+ log.info('Booting RAID-1 system')
+ cmd = get_qemu_cmd('TESTVM', kvm, args.uefi, args.disk, diskname_raid)
+ log.debug(f'Executing command: {cmd}')
+ c = pexpect.spawn(cmd, logfile=stl)
+
+ loginVM(c, log)
+
+ c.sendline('cat /proc/mdstat')
+ c.expect(op_mode_prompt)
+
+ elif not args.configtest:
+ # run default smoketest suite
+ if args.no_interfaces:
+ # remove interface tests as they consume a lot of time
+ c.sendline('sudo rm -f /usr/libexec/vyos/tests/smoke/cli/test_interfaces_*')
+ c.expect(op_mode_prompt)
- # run default smoketest suite
- if not args.raid and not args.configtest:
log.info('Executing VyOS smoketests')
c.sendline('/usr/bin/vyos-smoketest')
i = c.expect(['\n +Invalid command:', '\n +Set failed',
- 'No such file or directory', r'\n\S+@\S+[$#]'], timeout=7200)
+ 'No such file or directory', r'\n\S+@\S+[$#]'], timeout=test_timeout)
if i == 0:
raise Exception('Invalid command detected')
@@ -399,10 +488,14 @@ try:
raise Exception("Smoketest-failed, please look into debug output")
# else, run configtest suite
- elif not args.raid:
- log.info('Generating a WireGuard default keypair')
- c.sendline('generate wireguard default-keypair')
- c.expect(r'vyos@vyos:~\$')
+ else:
+ log.info('Adding a legacy WireGuard default keypair for migrations')
+ c.sendline('sudo mkdir -p /config/auth/wireguard/default')
+ c.expect(op_mode_prompt)
+ c.sendline('echo "aGx+fvW916Ej7QRnBbW3QMoldhNv1u95/WHz45zDmF0=" | sudo tee /config/auth/wireguard/default/private.key')
+ c.expect(op_mode_prompt)
+ c.sendline('echo "x39C77eavJNpvYbNzPSG3n1D68rHYei6q3AEBEyL1z8=" | sudo tee /config/auth/wireguard/default/public.key')
+ c.expect(op_mode_prompt)
log.info('Generating some OpenVPN keys')
subject = '/C=DE/ST=BY/O=VyOS/localityName=Cloud/commonName=vyos/' \
@@ -416,28 +509,28 @@ try:
c.sendline(f'openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 '\
f'-keyout {ssl_key} -out {ssl_cert} -subj {subject}')
- c.expect(r'vyos@vyos:~\$', timeout=600)
+ c.expect(op_mode_prompt, timeout=600)
c.sendline(f'openssl req -new -x509 -key {ssl_key} -out {ca_cert} -subj {subject}')
- c.expect(r'vyos@vyos:~\$', timeout=600)
+ c.expect(op_mode_prompt, timeout=600)
c.sendline(f'openssl dhparam -out {dh_pem} 2048')
- c.expect(r'vyos@vyos:~\$', timeout=600)
- c.sendline(f'generate openvpn key {s2s_key}')
- c.expect(r'vyos@vyos:~\$')
- c.sendline(f'generate openvpn key {auth_key}')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt, timeout=600)
+ c.sendline(f'openvpn --genkey secret {s2s_key}')
+ c.expect(op_mode_prompt)
+ c.sendline(f'openvpn --genkey secret {auth_key}')
+ c.expect(op_mode_prompt)
script_file = '/config/scripts/vyos-foo-update.script'
c.sendline(f'echo "#!/bin/sh" > {script_file}; chmod 775 {script_file}')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
for file in [ca_cert, ssl_cert, ssl_key, dh_pem, s2s_key, auth_key]:
c.sendline(f'sudo chown openvpn:openvpn {file}')
- c.expect(r'vyos@vyos:~\$')
+ c.expect(op_mode_prompt)
log.info('Executing load config tests')
c.sendline('/usr/bin/vyos-configtest')
i = c.expect(['\n +Invalid command:', 'No such file or directory',
- r'\n\S+@\S+[$#]'], timeout=3600)
+ r'\n\S+@\S+[$#]'], timeout=test_timeout)
if i==0:
raise Exception('Invalid command detected')
--
cgit v1.2.3