cc_grub_dpkg: determine idevs in more robust manner with grub-probe (#358)

Replace the hardcoded list of devices with a more robust way of determining
the device which grub is installed to.

We use grub-probe to fetch the underlying disk the /boot directory is
located on, and attempt to match the disk with its /dev/disk/by-id value.
If no such /dev/disk/by-id/ value exists, we fallback to the plain disk
name.

The changes are robust to unstable kernel device names and ordering, and use
/dev/disk/by-id values to populate grub-pc/install_devices where possible.

LP: #1877491

3 files changed, 247 insertions, 25 deletions

diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py
index a323edfa..7888464e 100644
--- a/cloudinit/config/cc_grub_dpkg.py
+++ b/cloudinit/config/cc_grub_dpkg.py
@@ -1,8 +1,9 @@
-# Copyright (C) 2009-2010 Canonical Ltd.
+# Copyright (C) 2009-2010, 2020 Canonical Ltd.
 # Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
 #
 # Author: Scott Moser <scott.moser@canonical.com>
 # Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Matthew Ruffell <matthew.ruffell@canonical.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
@@ -15,15 +16,15 @@ Configure which device is used as the target for grub installation. This module
 should work correctly by default without any user configuration. It can be
 enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config
 dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no
-installation device is specified this module will look for the first existing
-device in:
+installation device is specified this module will execute grub-probe to
+determine which disk the /boot directory is associated with.
 
-    - ``/dev/sda``
-    - ``/dev/vda``
-    - ``/dev/xvda``
-    - ``/dev/sda1``
-    - ``/dev/vda1``
-    - ``/dev/xvda1``
+The value which is placed into the debconf database is in the format which the
+grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value,
+but we do fallback to the plain disk name if a by-id name is not present.
+
+If this module is executed inside a container, then the debconf database is
+seeded with empty values, and install_devices_empty is set to true.
 
 **Internal name:** ``cc_grub_dpkg``
 
@@ -43,10 +44,66 @@ device in:
 import os
 
 from cloudinit import util
+from cloudinit.util import ProcessExecutionError
 
 distros = ['ubuntu', 'debian']
 
 
+def fetch_idevs(log):
+    """
+    Fetches the /dev/disk/by-id device grub is installed to.
+    Falls back to plain disk name if no by-id entry is present.
+    """
+    disk = ""
+    devices = []
+
+    try:
+        # get the root disk where the /boot directory resides.
+        disk = util.subp(['grub-probe', '-t', 'disk', '/boot'],
+                         capture=True)[0].strip()
+    except ProcessExecutionError as e:
+        # grub-common may not be installed, especially on containers
+        # FileNotFoundError is a nested exception of ProcessExecutionError
+        if isinstance(e.reason, FileNotFoundError):
+            log.debug("'grub-probe' not found in $PATH")
+        # disks from the container host are present in /proc and /sys
+        # which is where grub-probe determines where /boot is.
+        # it then checks for existence in /dev, which fails as host disks
+        # are not exposed to the container.
+        elif "failed to get canonical path" in e.stderr:
+            log.debug("grub-probe 'failed to get canonical path'")
+        else:
+            # something bad has happened, continue to log the error
+            raise
+    except Exception:
+        util.logexc(log, "grub-probe failed to execute for grub-dpkg")
+
+    if not disk or not os.path.exists(disk):
+        # If we failed to detect a disk, we can return early
+        return ''
+
+    try:
+        # check if disk exists and use udevadm to fetch symlinks
+        devices = util.subp(
+            ['udevadm', 'info', '--root', '--query=symlink', disk],
+            capture=True
+        )[0].strip().split()
+    except Exception:
+        util.logexc(
+            log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk
+        )
+
+    log.debug('considering these device symlinks: %s', ','.join(devices))
+    # filter symlinks for /dev/disk/by-id entries
+    devices = [dev for dev in devices if 'disk/by-id' in dev]
+    log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices))
+    # select first device if there is one, else fall back to plain name
+    idevs = sorted(devices)[0] if devices else disk
+    log.debug('selected %s', idevs)
+
+    return idevs
+
+
 def handle(name, cfg, _cloud, log, _args):
 
     mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {}))
@@ -62,22 +119,10 @@ def handle(name, cfg, _cloud, log, _args):
     idevs_empty = util.get_cfg_option_str(
         mycfg, "grub-pc/install_devices_empty", None)
 
-    if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or
-       (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))):
-        if idevs is None:
-            idevs = ""
-        if idevs_empty is None:
-            idevs_empty = "true"
-    else:
-        if idevs_empty is None:
-            idevs_empty = "false"
-        if idevs is None:
-            idevs = "/dev/sda"
-            for dev in ("/dev/sda", "/dev/vda", "/dev/xvda",
-                        "/dev/sda1", "/dev/vda1", "/dev/xvda1"):
-                if os.path.exists(dev):
-                    idevs = dev
-                    break
+    if idevs is None:
+        idevs = fetch_idevs(log)
+    if idevs_empty is None:
+        idevs_empty = "false" if idevs else "true"
 
     # now idevs and idevs_empty are set to determined values
     # or, those set by user
diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py
new file mode 100644
index 00000000..01efa330
--- /dev/null
+++ b/cloudinit/config/tests/test_grub_dpkg.py
@@ -0,0 +1,176 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import pytest
+
+from unittest import mock
+from logging import Logger
+from cloudinit.util import ProcessExecutionError
+from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle
+
+
+class TestFetchIdevs:
+    """Tests cc_grub_dpkg.fetch_idevs()"""
+
+    # Note: udevadm info returns devices in a large single line string
+    @pytest.mark.parametrize(
+        "grub_output,path_exists,expected_log_call,udevadm_output"
+        ",expected_idevs",
+        [
+            # Inside a container, grub not installed
+            (
+                ProcessExecutionError(reason=FileNotFoundError()),
+                False,
+                mock.call("'grub-probe' not found in $PATH"),
+                '',
+                '',
+            ),
+            # Inside a container, grub installed
+            (
+                ProcessExecutionError(stderr="failed to get canonical path"),
+                False,
+                mock.call("grub-probe 'failed to get canonical path'"),
+                '',
+                '',
+            ),
+            # KVM Instance
+            (
+                ['/dev/vda'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-path/pci-0000:00:00.0 ',
+                    '/dev/disk/by-path/virtio-pci-0000:00:00.0 '
+                ),
+                '/dev/vda',
+            ),
+            # Xen Instance
+            (
+                ['/dev/xvda'],
+                True,
+                None,
+                '',
+                '/dev/xvda',
+            ),
+            # NVMe Hardware Instance
+            (
+                ['/dev/nvme1n1'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-id/nvme-Company_hash000 ',
+                    '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ',
+                    '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 '
+                ),
+                '/dev/disk/by-id/nvme-Company_hash000',
+            ),
+            # SCSI Hardware Instance
+            (
+                ['/dev/sda'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-id/company-user-1 ',
+                    '/dev/disk/by-id/scsi-0Company_user-1 ',
+                    '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 '
+                ),
+                '/dev/disk/by-id/company-user-1',
+            ),
+        ],
+    )
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output,
+                         path_exists, expected_log_call, udevadm_output,
+                         expected_idevs):
+        """Tests outputs from grub-probe and udevadm info against grub-dpkg"""
+        m_subp.side_effect = [
+            grub_output,
+            ["".join(udevadm_output)]
+        ]
+        m_exists.return_value = path_exists
+        log = mock.Mock(spec=Logger)
+        idevs = fetch_idevs(log)
+        assert expected_idevs == idevs
+        if expected_log_call is not None:
+            assert expected_log_call in log.debug.call_args_list
+
+
+class TestHandle:
+    """Tests cc_grub_dpkg.handle()"""
+
+    @pytest.mark.parametrize(
+        "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output",
+        [
+            (
+                # No configuration
+                None,
+                None,
+                '/dev/disk/by-id/nvme-Company_hash000',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/disk/by-id/nvme-Company_hash000','false'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty unset
+                '/dev/sda',
+                None,
+                '/dev/sda',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/sda','false'"
+                ),
+            ),
+            (
+                # idevs unset, idevs_empty set
+                None,
+                'true',
+                '/dev/xvda',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/xvda','true'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty set
+                '/dev/vda',
+                'false',
+                '/dev/disk/by-id/company-user-1',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/vda','false'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty set
+                # Respect what the user defines, even if its logically wrong
+                '/dev/nvme0n1',
+                'true',
+                '',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/nvme0n1','true'"
+                ),
+            )
+        ],
+    )
+    @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs,
+                    cfg_idevs, cfg_idevs_empty, fetch_idevs_output,
+                    expected_log_output):
+        """Test setting of correct debconf database entries"""
+        m_get_cfg_str.side_effect = [
+            cfg_idevs,
+            cfg_idevs_empty
+        ]
+        m_fetch_idevs.return_value = fetch_idevs_output
+        log = mock.Mock(spec=Logger)
+        handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock())
+        log.debug.assert_called_with("".join(expected_log_output))
+
+
+# vi: ts=4 expandtab
diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
index 2c924f85..c3113705 100644
--- a/tools/.github-cla-signers
+++ b/tools/.github-cla-signers
@@ -2,6 +2,7 @@ beezly
 bipinbachhao
 dhensby
 lucasmoura
+matthewruffell
 nishigori
 tomponline
 TheRealFalcon