Testsuite: add RAID-1 test case for disk recovery

(cherry picked from commit 986fd18e11ac19470e17ad786d5cc908339ccb43)

1 files changed, 127 insertions, 34 deletions

diff --git a/scripts/check-qemu-install b/scripts/check-qemu-install
index e1a97c85..5c099a3b 100755
--- a/scripts/check-qemu-install
+++ b/scripts/check-qemu-install
@@ -42,6 +42,7 @@ import random
 import traceback
 import logging
 import re
+import json
 
 from io import BytesIO
 from io import StringIO
@@ -65,13 +66,18 @@ parser.add_argument('--logfile', help='Log to file')
 parser.add_argument('--uefi', help='Boot using UEFI', action='store_true', default=False)
 parser.add_argument('--raid', help='Perform a RAID-1 install', action='store_true', default=False)
 parser.add_argument('--no-kvm', help='Disable use of kvm', action='store_true', default=False)
-parser.add_argument('--configd', help='Execute testsuite with config daemon',
-				action='store_true', default=False)
+parser.add_argument('--configd', help='Execute testsuite with config daemon', action='store_true',
+				default=False)
+parser.add_argument('--no-interfaces', help='Execute testsuite without interface tests to save time',
+                action='store_true', default=False)
 parser.add_argument('--configtest', help='Execute load/commit config tests',
 				action='store_true', default=False)
 
 args = parser.parse_args()
 
+with open('data/defaults.json') as f:
+    vyos_defaults = json.load(f)
+
 class StreamToLogger(object):
     """
     Fake file-like stream object that redirects writes to a logger instance.
@@ -119,7 +125,7 @@ def get_qemu_cmd(name, enable_kvm, enable_uefi, disk_img, raid=None, iso_img=Non
 
     cdrom = ""
     if iso_img:
-        cdrom = "-boot d -cdrom {}".format(iso_img)
+        cdrom = f"-boot d -cdrom {iso_img}"
 
     # test using half of the available CPUs on the system
     cpucount = get_half_cpus()
@@ -143,10 +149,13 @@ def get_qemu_cmd(name, enable_kvm, enable_uefi, disk_img, raid=None, iso_img=Non
         -machine accel=kvm \
         -uuid {uuid} \
         -nographic {cpu} {cdrom} {kvm} \
-        -drive format=raw,file={disk_img}'
+        -device virtio-scsi-pci,id=scsi0 \
+        -drive format=raw,file={disk_img},if=none,media=disk,id=drive-hd1,readonly=off \
+        -device scsi-hd,bus=scsi0.0,drive=drive-hd1,id=hd1,bootindex=1'
 
     if raid:
-        cmd += f' -drive format=raw,file={raid}'
+        cmd += f' -drive format=raw,file={raid},if=none,media=disk,id=drive-hd2,readonly=off \
+                  -device scsi-hd,bus=scsi0.0,drive=drive-hd2,id=hd2,bootindex=2'
 
     return cmd
 
@@ -157,9 +166,7 @@ def shutdownVM(c, log, message=''):
     if message:
         log.info(message)
 
-    c.sendline('poweroff')
-    c.expect(r'\nAre you sure you want to poweroff this system.*\]')
-    c.sendline('Y')
+    c.sendline('poweroff now')
     log.info('Shutting down virtual machine')
     for i in range(30):
         log.info('Waiting for shutdown...')
@@ -243,6 +250,7 @@ if args.raid:
 # must be called after the raid disk as args.disk name is altered in the RAID path
 gen_disk(args.disk)
 
+test_timeout = 3 *3600 # 3 hours (in seconds)
 try:
     #################################################
     # Installing image to disk
@@ -341,7 +349,7 @@ try:
     # Always load the WiFi simulation module
     ################################################
     c.sendline('sudo modprobe mac80211_hwsim')
-    c.expect(r'vyos@vyos:~\$')
+    c.expect(op_mode_prompt)
 
     #################################################
     # Start/stop config daemon
@@ -350,35 +358,116 @@ try:
         c.sendline('sudo systemctl start vyos-configd.service &> /dev/null')
     else:
         c.sendline('sudo systemctl stop vyos-configd.service &> /dev/null')
-    c.expect(r'vyos@vyos:~\$')
+    c.expect(op_mode_prompt)
 
     #################################################
     # Basic Configmode/Opmode switch
     #################################################
     log.info('Basic CLI configuration mode test')
     c.sendline('configure')
-    c.expect(r'vyos@vyos#')
+    c.expect(cfg_mode_prompt)
     c.sendline('exit')
-    c.expect(r'vyos@vyos:~\$')
+    c.expect(op_mode_prompt)
     c.sendline('show version')
-    c.expect(r'vyos@vyos:~\$')
+    c.expect(op_mode_prompt)
+    c.sendline('show version kernel')
+    c.expect(f'{vyos_defaults["kernel_version"]}-{vyos_defaults["kernel_flavor"]}')
+    c.expect(op_mode_prompt)
+    c.sendline('show version frr')
+    c.expect(op_mode_prompt)
     c.sendline('show interfaces')
-    c.expect(r'vyos@vyos:~\$')
+    c.expect(op_mode_prompt)
 
+    #################################################
+    # Executing test-suite
+    #################################################
     if args.raid:
+        # Verify RAID subsystem - by deleting a disk and re-create the array
+        # from scratch
         c.sendline('cat /proc/mdstat')
         c.expect(op_mode_prompt)
 
-    #################################################
-    # Executing test-suite
-    #################################################
+        shutdownVM(c, log, f'Shutdown VM and start with empty RAID member "{args.disk}"')
+
+        if os.path.exists(args.disk):
+            os.unlink(args.disk)
+
+        gen_disk(args.disk)
+
+        #################################################
+        # Booting RAID-1 system with one missing disk
+        #################################################
+        log.info('Booting RAID-1 system')
+        cmd = get_qemu_cmd('TESTVM', kvm, args.uefi, args.disk, diskname_raid)
+
+        # We need to swap boot indexes to boot from second harddisk so we can
+        # recreate the RAID on the first disk
+        cmd = cmd.replace('bootindex=1', 'bootindex=X')
+        cmd = cmd.replace('bootindex=2', 'bootindex=1')
+        cmd = cmd.replace('bootindex=X', 'bootindex=2')
+
+        log.debug(f'Executing command: {cmd}')
+        c = pexpect.spawn(cmd, logfile=stl)
+
+
+        #################################################
+        # Logging into VyOS system
+        #################################################
+        try:
+            c.expect('The highlighted entry will be executed automatically in', timeout=10)
+            c.sendline('')
+        except pexpect.TIMEOUT:
+            log.warning('Did not find GRUB countdown window, ignoring')
+
+        loginVM(c, log)
+
+        c.sendline('cat /proc/mdstat')
+        c.expect(op_mode_prompt)
+
+        log.info('Re-format new RAID member')
+        c.sendline('format disk sda like sdb')
+        c.sendline('yes')
+        c.expect(op_mode_prompt)
+
+        log.info('Add member to RAID1 (md0)')
+        c.sendline('add raid md0 member sda1')
+        c.expect(op_mode_prompt)
+
+        log.info('Now we need to wait for re-sync to complete')
+
+        start_time = time.time()
+        timeout = 60
+        while True:
+            if (start_time + timeout) < time.time():
+                break
+            c.sendline('cat /proc/mdstat')
+            c.expect(op_mode_prompt)
+            time.sleep(20)
+
+        # Reboot system with new primary RAID1 disk
+        shutdownVM(c, log, f'Shutdown VM and start from recovered RAID member "{args.disk}"')
+
+        log.info('Booting RAID-1 system')
+        cmd = get_qemu_cmd('TESTVM', kvm, args.uefi, args.disk, diskname_raid)
+        log.debug(f'Executing command: {cmd}')
+        c = pexpect.spawn(cmd, logfile=stl)
+
+        loginVM(c, log)
+
+        c.sendline('cat /proc/mdstat')
+        c.expect(op_mode_prompt)
+
+    elif not args.configtest:
+        # run default smoketest suite
+        if args.no_interfaces:
+            # remove interface tests as they consume a lot of time
+            c.sendline('sudo rm -f /usr/libexec/vyos/tests/smoke/cli/test_interfaces_*')
+            c.expect(op_mode_prompt)
 
-    # run default smoketest suite
-    if not args.raid and not args.configtest:
         log.info('Executing VyOS smoketests')
         c.sendline('/usr/bin/vyos-smoketest')
         i = c.expect(['\n +Invalid command:', '\n +Set failed',
-                      'No such file or directory', r'\n\S+@\S+[$#]'], timeout=7200)
+                      'No such file or directory', r'\n\S+@\S+[$#]'], timeout=test_timeout)
 
         if i == 0:
             raise Exception('Invalid command detected')
@@ -399,10 +488,14 @@ try:
             raise Exception("Smoketest-failed, please look into debug output")
 
     # else, run configtest suite
-    elif not args.raid:
-        log.info('Generating a WireGuard default keypair')
-        c.sendline('generate wireguard default-keypair')
-        c.expect(r'vyos@vyos:~\$')
+    else:
+        log.info('Adding a legacy WireGuard default keypair for migrations')
+        c.sendline('sudo mkdir -p /config/auth/wireguard/default')
+        c.expect(op_mode_prompt)
+        c.sendline('echo "aGx+fvW916Ej7QRnBbW3QMoldhNv1u95/WHz45zDmF0=" | sudo tee /config/auth/wireguard/default/private.key')
+        c.expect(op_mode_prompt)
+        c.sendline('echo "x39C77eavJNpvYbNzPSG3n1D68rHYei6q3AEBEyL1z8=" | sudo tee /config/auth/wireguard/default/public.key')
+        c.expect(op_mode_prompt)
 
         log.info('Generating some OpenVPN keys')
         subject = '/C=DE/ST=BY/O=VyOS/localityName=Cloud/commonName=vyos/' \
@@ -416,28 +509,28 @@ try:
 
         c.sendline(f'openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 '\
                    f'-keyout {ssl_key} -out {ssl_cert} -subj {subject}')
-        c.expect(r'vyos@vyos:~\$', timeout=600)
+        c.expect(op_mode_prompt, timeout=600)
         c.sendline(f'openssl req -new -x509 -key {ssl_key} -out {ca_cert} -subj {subject}')
-        c.expect(r'vyos@vyos:~\$', timeout=600)
+        c.expect(op_mode_prompt, timeout=600)
         c.sendline(f'openssl dhparam -out {dh_pem} 2048')
-        c.expect(r'vyos@vyos:~\$', timeout=600)
-        c.sendline(f'generate openvpn key {s2s_key}')
-        c.expect(r'vyos@vyos:~\$')
-        c.sendline(f'generate openvpn key {auth_key}')
-        c.expect(r'vyos@vyos:~\$')
+        c.expect(op_mode_prompt, timeout=600)
+        c.sendline(f'openvpn --genkey secret {s2s_key}')
+        c.expect(op_mode_prompt)
+        c.sendline(f'openvpn --genkey secret {auth_key}')
+        c.expect(op_mode_prompt)
 
         script_file = '/config/scripts/vyos-foo-update.script'
         c.sendline(f'echo "#!/bin/sh" > {script_file}; chmod 775 {script_file}')
-        c.expect(r'vyos@vyos:~\$')
+        c.expect(op_mode_prompt)
 
         for file in [ca_cert, ssl_cert, ssl_key, dh_pem, s2s_key, auth_key]:
             c.sendline(f'sudo chown openvpn:openvpn {file}')
-            c.expect(r'vyos@vyos:~\$')
+            c.expect(op_mode_prompt)
 
         log.info('Executing load config tests')
         c.sendline('/usr/bin/vyos-configtest')
         i = c.expect(['\n +Invalid command:', 'No such file or directory',
-                     r'\n\S+@\S+[$#]'], timeout=3600)
+                     r'\n\S+@\S+[$#]'], timeout=test_timeout)
 
         if i==0:
             raise Exception('Invalid command detected')