Merge pull request #28 from zdc/T2117-equuleus-20.4

T2117: Cloud-init updated to 20.4

13 files changed, 125 insertions, 58 deletions

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 2537608f..1e118472 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -23,6 +23,7 @@ from cloudinit import net
 from cloudinit.net import eni
 from cloudinit.net import network_state
 from cloudinit.net import renderers
+from cloudinit import persistence
 from cloudinit import ssh_util
 from cloudinit import type_utils
 from cloudinit import subp
@@ -62,7 +63,7 @@ PREFERRED_NTP_CLIENTS = ['chrony', 'systemd-timesyncd', 'ntp', 'ntpdate']
 LDH_ASCII_CHARS = string.ascii_letters + string.digits + "-"
 
 
-class Distro(metaclass=abc.ABCMeta):
+class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
 
     usr_lib_exec = "/usr/lib"
     hosts_fn = "/etc/hosts"
@@ -73,6 +74,11 @@ class Distro(metaclass=abc.ABCMeta):
     renderer_configs = {}
     _preferred_ntp_clients = None
     networking_cls = LinuxNetworking
+    # This is used by self.shutdown_command(), and can be overridden in
+    # subclasses
+    shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'}
+
+    _ci_pkl_version = 1
 
     def __init__(self, name, cfg, paths):
         self._paths = paths
@@ -80,6 +86,18 @@ class Distro(metaclass=abc.ABCMeta):
         self.name = name
         self.networking = self.networking_cls()
 
+    def _unpickle(self, ci_pkl_version: int) -> None:
+        """Perform deserialization fixes for Distro."""
+        if "networking" not in self.__dict__ or not self.networking.__dict__:
+            # This is either a Distro pickle with no networking attribute OR
+            # this is a Distro pickle with a networking attribute but from
+            # before ``Networking`` had any state (meaning that
+            # Networking.__setstate__ will not be called).  In either case, we
+            # want to ensure that `self.networking` is freshly-instantiated:
+            # either because it isn't present at all, or because it will be
+            # missing expected instance state otherwise.
+            self.networking = self.networking_cls()
+
     @abc.abstractmethod
     def install_packages(self, pkglist):
         raise NotImplementedError()
@@ -250,8 +268,9 @@ class Distro(metaclass=abc.ABCMeta):
         distros = []
         for family in family_list:
             if family not in OSFAMILIES:
-                raise ValueError("No distibutions found for osfamily %s"
-                                 % (family))
+                raise ValueError(
+                    "No distributions found for osfamily {}".format(family)
+                )
             distros.extend(OSFAMILIES[family])
         return distros
 
@@ -749,6 +768,22 @@ class Distro(metaclass=abc.ABCMeta):
                 subp.subp(['usermod', '-a', '-G', name, member])
                 LOG.info("Added user '%s' to group '%s'", member, name)
 
+    def shutdown_command(self, *, mode, delay, message):
+        # called from cc_power_state_change.load_power_state
+        command = ["shutdown", self.shutdown_options_map[mode]]
+        try:
+            if delay != "now":
+                delay = "+%d" % int(delay)
+        except ValueError as e:
+            raise TypeError(
+                "power_state[delay] must be 'now' or '+m' (minutes)."
+                " found '%s'." % (delay,)
+            ) from e
+        args = command + [delay]
+        if message:
+            args.append(message)
+        return args
+
 
 def _apply_hostname_transformations_to_url(url: str, transformations: list):
     """
diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py
index e42443fc..ca5bfe80 100644
--- a/cloudinit/distros/alpine.py
+++ b/cloudinit/distros/alpine.py
@@ -8,7 +8,6 @@
 
 from cloudinit import distros
 from cloudinit import helpers
-from cloudinit import log as logging
 from cloudinit import subp
 from cloudinit import util
 
@@ -16,8 +15,6 @@ from cloudinit.distros.parsers.hostname import HostnameConf
 
 from cloudinit.settings import PER_INSTANCE
 
-LOG = logging.getLogger(__name__)
-
 NETWORK_FILE_HEADER = """\
 # This file is generated from information provided by the datasource. Changes
 # to it will not persist across an instance reboot. To disable cloud-init's
@@ -162,4 +159,30 @@ class Distro(distros.Distro):
 
         return self._preferred_ntp_clients
 
+    def shutdown_command(self, mode='poweroff', delay='now', message=None):
+        # called from cc_power_state_change.load_power_state
+        # Alpine has halt/poweroff/reboot, with the following specifics:
+        # - we use them rather than the generic "shutdown"
+        # - delay is given with "-d [integer]"
+        # - the integer is in seconds, cannot be "now", and takes no "+"
+        # - no message is supported (argument ignored, here)
+
+        command = [mode, "-d"]
+
+        # Convert delay from minutes to seconds, as Alpine's
+        # halt/poweroff/reboot commands take seconds rather than minutes.
+        if delay == "now":
+            # Alpine's commands do not understand "now".
+            command += ['0']
+        else:
+            try:
+                command.append(str(int(delay) * 60))
+            except ValueError as e:
+                raise TypeError(
+                    "power_state[delay] must be 'now' or '+m' (minutes)."
+                    " found '%s'." % (delay,)
+                ) from e
+
+        return command
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/amazon.py b/cloudinit/distros/amazon.py
index ff9a549f..5fcec952 100644
--- a/cloudinit/distros/amazon.py
+++ b/cloudinit/distros/amazon.py
@@ -12,10 +12,6 @@
 
 from cloudinit.distros import rhel
 
-from cloudinit import log as logging
-
-LOG = logging.getLogger(__name__)
-
 
 class Distro(rhel.Distro):
 
diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py
index 2ed7a7d5..f717a667 100644
--- a/cloudinit/distros/bsd.py
+++ b/cloudinit/distros/bsd.py
@@ -17,6 +17,10 @@ class BSD(distros.Distro):
     hostname_conf_fn = '/etc/rc.conf'
     rc_conf_fn = "/etc/rc.conf"
 
+    # This differs from the parent Distro class, which has -P for
+    # poweroff.
+    shutdown_options_map = {'halt': '-H', 'poweroff': '-p', 'reboot': '-r'}
+
     # Set in BSD distro subclasses
     group_add_cmd_prefix = []
     pkg_cmd_install_prefix = []
diff --git a/cloudinit/distros/centos.py b/cloudinit/distros/centos.py
index 4b803d2e..edb3165d 100644
--- a/cloudinit/distros/centos.py
+++ b/cloudinit/distros/centos.py
@@ -1,9 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 from cloudinit.distros import rhel
-from cloudinit import log as logging
-
-LOG = logging.getLogger(__name__)
 
 
 class Distro(rhel.Distro):
diff --git a/cloudinit/distros/fedora.py b/cloudinit/distros/fedora.py
index a9490d0e..0fe1fbca 100644
--- a/cloudinit/distros/fedora.py
+++ b/cloudinit/distros/fedora.py
@@ -10,10 +10,6 @@
 
 from cloudinit.distros import rhel
 
-from cloudinit import log as logging
-
-LOG = logging.getLogger(__name__)
-
 
 class Distro(rhel.Distro):
     pass
diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py
index 2bee1c89..e9b82602 100644
--- a/cloudinit/distros/gentoo.py
+++ b/cloudinit/distros/gentoo.py
@@ -160,10 +160,12 @@ class Distro(distros.Distro):
             pass
         if not conf:
             conf = HostnameConf('')
-        conf.set_hostname(your_hostname)
-        gentoo_hostname_config = 'hostname="%s"' % conf
-        gentoo_hostname_config = gentoo_hostname_config.replace('\n', '')
-        util.write_file(out_fn, gentoo_hostname_config, 0o644)
+
+        # Many distro's format is the hostname by itself, and that is the
+        # way HostnameConf works but gentoo expects it to be in
+        #     hostname="the-actual-hostname"
+        conf.set_hostname('hostname="%s"' % your_hostname)
+        util.write_file(out_fn, str(conf), 0o644)
 
     def _read_system_hostname(self):
         sys_hostname = self._read_hostname(self.hostname_conf_fn)
diff --git a/cloudinit/distros/networking.py b/cloudinit/distros/networking.py
index 10ed249d..c291196a 100644
--- a/cloudinit/distros/networking.py
+++ b/cloudinit/distros/networking.py
@@ -2,6 +2,7 @@ import abc
 import logging
 import os
 
+from cloudinit import subp
 from cloudinit import net, util
 
 
@@ -22,6 +23,9 @@ class Networking(metaclass=abc.ABCMeta):
     Hierarchy" in HACKING.rst for full details.
     """
 
+    def __init__(self):
+        self.blacklist_drivers = None
+
     def _get_current_rename_info(self) -> dict:
         return net._get_current_rename_info()
 
@@ -68,7 +72,8 @@ class Networking(metaclass=abc.ABCMeta):
         return net.get_interfaces()
 
     def get_interfaces_by_mac(self) -> dict:
-        return net.get_interfaces_by_mac()
+        return net.get_interfaces_by_mac(
+            blacklist_drivers=self.blacklist_drivers)
 
     def get_master(self, devname: DeviceName):
         return net.get_master(devname)
@@ -171,6 +176,10 @@ class Networking(metaclass=abc.ABCMeta):
         if strict:
             raise RuntimeError(msg)
 
+    @abc.abstractmethod
+    def try_set_link_up(self, devname: DeviceName) -> bool:
+        """Try setting the link to up explicitly and return if it is up."""
+
 
 class BSDNetworking(Networking):
     """Implementation of networking functionality shared across BSDs."""
@@ -181,6 +190,9 @@ class BSDNetworking(Networking):
     def settle(self, *, exists=None) -> None:
         """BSD has no equivalent to `udevadm settle`; noop."""
 
+    def try_set_link_up(self, devname: DeviceName) -> bool:
+        raise NotImplementedError()
+
 
 class LinuxNetworking(Networking):
     """Implementation of networking functionality common to Linux distros."""
@@ -210,3 +222,10 @@ class LinuxNetworking(Networking):
         if exists is not None:
             exists = net.sys_dev_path(exists)
         util.udevadm_settle(exists=exists)
+
+    def try_set_link_up(self, devname: DeviceName) -> bool:
+        """Try setting the link to up explicitly and return if it is up.
+           Not guaranteed to bring the interface up. The caller is expected to
+           add wait times before retrying."""
+        subp.subp(['ip', 'link', 'set', devname, 'up'])
+        return self.is_up(devname)
diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index b8e557b8..7ca0ef99 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -13,15 +13,12 @@ from cloudinit import distros
 from cloudinit.distros.parsers.hostname import HostnameConf
 
 from cloudinit import helpers
-from cloudinit import log as logging
 from cloudinit import subp
 from cloudinit import util
 
 from cloudinit.distros import rhel_util as rhutil
 from cloudinit.settings import PER_INSTANCE
 
-LOG = logging.getLogger(__name__)
-
 
 class Distro(distros.Distro):
     clock_conf_fn = '/etc/sysconfig/clock'
diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py
index 387a851f..d71394b4 100644
--- a/cloudinit/distros/rhel_util.py
+++ b/cloudinit/distros/rhel_util.py
@@ -8,7 +8,6 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
-from cloudinit.distros.parsers.resolv_conf import ResolvConf
 from cloudinit.distros.parsers.sys_conf import SysConf
 
 from cloudinit import log as logging
@@ -50,29 +49,4 @@ def read_sysconfig_file(fn):
         contents = []
     return (exists, SysConf(contents))
 
-
-# Helper function to update RHEL/SUSE /etc/resolv.conf
-def update_resolve_conf_file(fn, dns_servers, search_servers):
-    try:
-        r_conf = ResolvConf(util.load_file(fn))
-        r_conf.parse()
-    except IOError:
-        util.logexc(LOG, "Failed at parsing %s reverting to an empty "
-                    "instance", fn)
-        r_conf = ResolvConf('')
-        r_conf.parse()
-    if dns_servers:
-        for s in dns_servers:
-            try:
-                r_conf.add_nameserver(s)
-            except ValueError:
-                util.logexc(LOG, "Failed at adding nameserver %s", s)
-    if search_servers:
-        for s in search_servers:
-            try:
-                r_conf.add_search_domain(s)
-            except ValueError:
-                util.logexc(LOG, "Failed at adding search domain %s", s)
-    util.write_file(fn, str(r_conf), 0o644)
-
 # vi: ts=4 expandtab
diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py
index 6e336cbf..f3bfb9c2 100644
--- a/cloudinit/distros/sles.py
+++ b/cloudinit/distros/sles.py
@@ -6,10 +6,6 @@
 
 from cloudinit.distros import opensuse
 
-from cloudinit import log as logging
-
-LOG = logging.getLogger(__name__)
-
 
 class Distro(opensuse.Distro):
     pass
diff --git a/cloudinit/distros/tests/test_networking.py b/cloudinit/distros/tests/test_networking.py
index b9a63842..ec508f4d 100644
--- a/cloudinit/distros/tests/test_networking.py
+++ b/cloudinit/distros/tests/test_networking.py
@@ -30,6 +30,9 @@ def generic_networking_cls():
         def settle(self, *args, **kwargs):
             raise NotImplementedError
 
+        def try_set_link_up(self, *args, **kwargs):
+            raise NotImplementedError
+
     error = AssertionError("Unexpectedly used /sys in generic networking code")
     with mock.patch(
         "cloudinit.net.get_sys_class_path", side_effect=error,
@@ -74,6 +77,34 @@ class TestLinuxNetworkingIsPhysical:
         assert LinuxNetworking().is_physical(devname)
 
 
+class TestBSDNetworkingTrySetLinkUp:
+    def test_raises_notimplementederror(self):
+        with pytest.raises(NotImplementedError):
+            BSDNetworking().try_set_link_up("eth0")
+
+
+@mock.patch("cloudinit.net.is_up")
+@mock.patch("cloudinit.distros.networking.subp.subp")
+class TestLinuxNetworkingTrySetLinkUp:
+    def test_calls_subp_return_true(self, m_subp, m_is_up):
+        devname = "eth0"
+        m_is_up.return_value = True
+        is_success = LinuxNetworking().try_set_link_up(devname)
+
+        assert (mock.call(['ip', 'link', 'set', devname, 'up']) ==
+                m_subp.call_args_list[-1])
+        assert is_success
+
+    def test_calls_subp_return_false(self, m_subp, m_is_up):
+        devname = "eth0"
+        m_is_up.return_value = False
+        is_success = LinuxNetworking().try_set_link_up(devname)
+
+        assert (mock.call(['ip', 'link', 'set', devname, 'up']) ==
+                m_subp.call_args_list[-1])
+        assert not is_success
+
+
 class TestBSDNetworkingSettle:
     def test_settle_doesnt_error(self):
         # This also implicitly tests that it doesn't use subp.subp
diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py
index b4c4b0c3..2a1f93d9 100644
--- a/cloudinit/distros/ubuntu.py
+++ b/cloudinit/distros/ubuntu.py
@@ -11,13 +11,10 @@
 
 from cloudinit.distros import debian
 from cloudinit.distros import PREFERRED_NTP_CLIENTS
-from cloudinit import log as logging
 from cloudinit import util
 
 import copy
 
-LOG = logging.getLogger(__name__)
-
 
 class Distro(debian.Distro):