T2117: Cloud-init updated to 20.3

Merged with 20.3 tag from the upstream Cloud-init repository

3 files changed, 160 insertions, 130 deletions

diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index c3fa1e04..74cf4b94 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -62,7 +62,7 @@ class TestParseDHCPLeasesFile(CiTestCase):
             {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
              'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}]
         write_file(lease_file, content)
-        self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+        self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file))
 
 
 class TestDHCPRFC3442(CiTestCase):
@@ -88,7 +88,7 @@ class TestDHCPRFC3442(CiTestCase):
              'renew': '4 2017/07/27 18:02:30',
              'expire': '5 2017/07/28 07:08:15'}]
         write_file(lease_file, content)
-        self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+        self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file))
 
     def test_parse_lease_finds_classless_static_routes(self):
         """
@@ -114,7 +114,7 @@ class TestDHCPRFC3442(CiTestCase):
              'renew': '4 2017/07/27 18:02:30',
              'expire': '5 2017/07/28 07:08:15'}]
         write_file(lease_file, content)
-        self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+        self.assertCountEqual(expected, parse_dhcp_lease_file(lease_file))
 
     @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
@@ -211,7 +211,7 @@ class TestDHCPParseStaticRoutes(CiTestCase):
             "class_b": "16,172,16,10",
             "class_a": "8,10,10",
             "gateway": "0,0",
-            "netlen":  "33,0",
+            "netlen": "33,0",
         }
         for rfc3442 in bad_rfc3442.values():
             self.assertEqual([], parse_static_routes(rfc3442))
@@ -266,7 +266,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
             'Skip dhcp_discovery: nic idontexist not found in get_devicelist.',
             self.logs.getvalue())
 
-    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.subp.which')
     @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
     def test_absent_dhclient_command(self, m_fallback, m_which):
         """When dhclient doesn't exist in the OS, log the issue and no-op."""
@@ -279,7 +279,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
 
     @mock.patch('cloudinit.temp_utils.os.getuid')
     @mock.patch('cloudinit.net.dhcp.dhcp_discovery')
-    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.subp.which')
     @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
     def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid):
         """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery."""
@@ -302,13 +302,14 @@ class TestDHCPDiscoveryClean(CiTestCase):
 
     @mock.patch('time.sleep', mock.MagicMock())
     @mock.patch('cloudinit.net.dhcp.os.kill')
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp,
                                                              m_kill):
         """dhcp_discovery logs a warning when pidfile contains invalid content.
 
         Lease processing still occurs and no proc kill is attempted.
         """
+        m_subp.return_value = ('', '')
         tmpdir = self.tmp_dir()
         dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
         script_content = '#!/bin/bash\necho fake-dhclient'
@@ -324,7 +325,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
         """)
         write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content)
 
-        self.assertItemsEqual(
+        self.assertCountEqual(
             [{'interface': 'eth9', 'fixed-address': '192.168.2.74',
               'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}],
             dhcp_discovery(dhclient_script, 'eth9', tmpdir))
@@ -337,13 +338,14 @@ class TestDHCPDiscoveryClean(CiTestCase):
     @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid')
     @mock.patch('cloudinit.net.dhcp.os.kill')
     @mock.patch('cloudinit.net.dhcp.util.wait_for_files')
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self,
                                                                   m_subp,
                                                                   m_wait,
                                                                   m_kill,
                                                                   m_getppid):
         """dhcp_discovery waits for the presence of pidfile and dhcp.leases."""
+        m_subp.return_value = ('', '')
         tmpdir = self.tmp_dir()
         dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
         script_content = '#!/bin/bash\necho fake-dhclient'
@@ -364,12 +366,13 @@ class TestDHCPDiscoveryClean(CiTestCase):
 
     @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid')
     @mock.patch('cloudinit.net.dhcp.os.kill')
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill, m_getppid):
         """dhcp_discovery brings up the interface and runs dhclient.
 
         It also returns the parsed dhcp.leases file generated in the sandbox.
         """
+        m_subp.return_value = ('', '')
         tmpdir = self.tmp_dir()
         dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
         script_content = '#!/bin/bash\necho fake-dhclient'
@@ -389,7 +392,7 @@ class TestDHCPDiscoveryClean(CiTestCase):
         write_file(pid_file, "%d\n" % my_pid)
         m_getppid.return_value = 1  # Indicate that dhclient has daemonized
 
-        self.assertItemsEqual(
+        self.assertCountEqual(
             [{'interface': 'eth9', 'fixed-address': '192.168.2.74',
               'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}],
             dhcp_discovery(dhclient_script, 'eth9', tmpdir))
@@ -406,6 +409,87 @@ class TestDHCPDiscoveryClean(CiTestCase):
                  'eth9', '-sf', '/bin/true'], capture=True)])
         m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)])
 
+    @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid')
+    @mock.patch('cloudinit.net.dhcp.os.kill')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
+    def test_dhcp_discovery_outside_sandbox(self, m_subp, m_kill, m_getppid):
+        """dhcp_discovery brings up the interface and runs dhclient.
+
+        It also returns the parsed dhcp.leases file generated in the sandbox.
+        """
+        m_subp.return_value = ('', '')
+        tmpdir = self.tmp_dir()
+        dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
+        script_content = '#!/bin/bash\necho fake-dhclient'
+        write_file(dhclient_script, script_content, mode=0o755)
+        lease_content = dedent("""
+            lease {
+              interface "eth9";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+            }
+        """)
+        lease_file = os.path.join(tmpdir, 'dhcp.leases')
+        write_file(lease_file, lease_content)
+        pid_file = os.path.join(tmpdir, 'dhclient.pid')
+        my_pid = 1
+        write_file(pid_file, "%d\n" % my_pid)
+        m_getppid.return_value = 1  # Indicate that dhclient has daemonized
+
+        with mock.patch('os.access', return_value=False):
+            self.assertCountEqual(
+                [{'interface': 'eth9', 'fixed-address': '192.168.2.74',
+                  'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}],
+                dhcp_discovery(dhclient_script, 'eth9', tmpdir))
+        # dhclient script got copied
+        with open(os.path.join(tmpdir, 'dhclient.orig')) as stream:
+            self.assertEqual(script_content, stream.read())
+        # Interface was brought up before dhclient called from sandbox
+        m_subp.assert_has_calls([
+            mock.call(
+                ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True),
+            mock.call(
+                [os.path.join(tmpdir, 'dhclient.orig'), '-1', '-v', '-lf',
+                 lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'),
+                 'eth9', '-sf', '/bin/true'], capture=True)])
+        m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)])
+
+    @mock.patch('cloudinit.net.dhcp.util.get_proc_ppid')
+    @mock.patch('cloudinit.net.dhcp.os.kill')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
+    def test_dhcp_output_error_stream(self, m_subp, m_kill, m_getppid):
+        """"dhcp_log_func is called with the output and error streams of
+        dhclinet when the callable is passed."""
+        dhclient_err = 'FAKE DHCLIENT ERROR'
+        dhclient_out = 'FAKE DHCLIENT OUT'
+        m_subp.return_value = (dhclient_out, dhclient_err)
+        tmpdir = self.tmp_dir()
+        dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
+        script_content = '#!/bin/bash\necho fake-dhclient'
+        write_file(dhclient_script, script_content, mode=0o755)
+        lease_content = dedent("""
+                lease {
+                  interface "eth9";
+                  fixed-address 192.168.2.74;
+                  option subnet-mask 255.255.255.0;
+                  option routers 192.168.2.1;
+                }
+            """)
+        lease_file = os.path.join(tmpdir, 'dhcp.leases')
+        write_file(lease_file, lease_content)
+        pid_file = os.path.join(tmpdir, 'dhclient.pid')
+        my_pid = 1
+        write_file(pid_file, "%d\n" % my_pid)
+        m_getppid.return_value = 1  # Indicate that dhclient has daemonized
+
+        def dhcp_log_func(out, err):
+            self.assertEqual(out, dhclient_out)
+            self.assertEqual(err, dhclient_err)
+
+        dhcp_discovery(
+            dhclient_script, 'eth9', tmpdir, dhcp_log_func=dhcp_log_func)
+
 
 class TestSystemdParseLeases(CiTestCase):
 
@@ -529,7 +613,7 @@ class TestEphemeralDhcpNoNetworkSetup(HttprettyTestCase):
         # Ensure that no teardown happens:
         m_dhcp.assert_not_called()
 
-    @mock.patch('cloudinit.net.dhcp.util.subp')
+    @mock.patch('cloudinit.net.dhcp.subp.subp')
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     def test_ephemeral_dhcp_setup_network_if_url_connectivity(
             self, m_dhcp, m_subp):
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 5081a337..311ab6f8 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -2,16 +2,20 @@
 
 import copy
 import errno
-import httpretty
+import ipaddress
 import os
-import requests
 import textwrap
 from unittest import mock
 
+import httpretty
+import pytest
+import requests
+
 import cloudinit.net as net
-from cloudinit.util import ensure_file, write_file, ProcessExecutionError
-from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase
 from cloudinit import safeyaml as yaml
+from cloudinit.tests.helpers import CiTestCase, HttprettyTestCase
+from cloudinit.subp import ProcessExecutionError
+from cloudinit.util import ensure_file, write_file
 
 
 class TestSysDevPath(CiTestCase):
@@ -139,12 +143,6 @@ class TestReadSysNet(CiTestCase):
             write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), state)
             self.assertFalse(net.is_up('eth0'))
 
-    def test_is_wireless(self):
-        """is_wireless is True when /sys/net/devname/wireless exists."""
-        self.assertFalse(net.is_wireless('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless'))
-        self.assertTrue(net.is_wireless('eth0'))
-
     def test_is_bridge(self):
         """is_bridge is True when /sys/net/devname/bridge exists."""
         self.assertFalse(net.is_bridge('eth0'))
@@ -200,32 +198,6 @@ class TestReadSysNet(CiTestCase):
         write_file(os.path.join(self.sysdir, 'eth0', 'uevent'), content)
         self.assertTrue(net.is_vlan('eth0'))
 
-    def test_is_connected_when_physically_connected(self):
-        """is_connected is True when /sys/net/devname/iflink reports 2."""
-        self.assertFalse(net.is_connected('eth0'))
-        write_file(os.path.join(self.sysdir, 'eth0', 'iflink'), "2")
-        self.assertTrue(net.is_connected('eth0'))
-
-    def test_is_connected_when_wireless_and_carrier_active(self):
-        """is_connected is True if wireless /sys/net/devname/carrier is 1."""
-        self.assertFalse(net.is_connected('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'wireless'))
-        self.assertFalse(net.is_connected('eth0'))
-        write_file(os.path.join(self.sysdir, 'eth0', 'carrier'), "1")
-        self.assertTrue(net.is_connected('eth0'))
-
-    def test_is_physical(self):
-        """is_physical is True when /sys/net/devname/device exists."""
-        self.assertFalse(net.is_physical('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
-        self.assertTrue(net.is_physical('eth0'))
-
-    def test_is_present(self):
-        """is_present is True when /sys/net/devname exists."""
-        self.assertFalse(net.is_present('eth0'))
-        ensure_file(os.path.join(self.sysdir, 'eth0', 'device'))
-        self.assertTrue(net.is_present('eth0'))
-
 
 class TestGenerateFallbackConfig(CiTestCase):
 
@@ -341,8 +313,6 @@ class TestGenerateFallbackConfig(CiTestCase):
 
 class TestNetFindFallBackNic(CiTestCase):
 
-    with_logs = True
-
     def setUp(self):
         super(TestNetFindFallBackNic, self).setUp()
         sys_mock = mock.patch('cloudinit.net.get_sys_class_path')
@@ -396,7 +366,7 @@ class TestGetDeviceList(CiTestCase):
         """get_devicelist returns a directory listing for SYS_CLASS_NET."""
         write_file(os.path.join(self.sysdir, 'eth0', 'operstate'), 'up')
         write_file(os.path.join(self.sysdir, 'eth1', 'operstate'), 'up')
-        self.assertItemsEqual(['eth0', 'eth1'], net.get_devicelist())
+        self.assertCountEqual(['eth0', 'eth1'], net.get_devicelist())
 
 
 class TestGetInterfaceMAC(CiTestCase):
@@ -540,7 +510,7 @@ class TestInterfaceHasOwnMAC(CiTestCase):
             net.interface_has_own_mac('eth1', strict=True)
 
 
-@mock.patch('cloudinit.net.util.subp')
+@mock.patch('cloudinit.net.subp.subp')
 class TestEphemeralIPV4Network(CiTestCase):
 
     with_logs = True
@@ -993,86 +963,8 @@ class TestExtractPhysdevs(CiTestCase):
             net.extract_physdevs({'version': 3, 'awesome_config': []})
 
 
-class TestWaitForPhysdevs(CiTestCase):
-
-    with_logs = True
-
-    def setUp(self):
-        super(TestWaitForPhysdevs, self).setUp()
-        self.add_patch('cloudinit.net.get_interfaces_by_mac',
-                       'm_get_iface_mac')
-        self.add_patch('cloudinit.util.udevadm_settle', 'm_udev_settle')
-
-    def test_wait_for_physdevs_skips_settle_if_all_present(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.side_effect = iter([
-            {'aa:bb:cc:dd:ee:ff': 'eth0',
-             '00:11:22:33:44:55': 'ens3'},
-        ])
-        net.wait_for_physdevs(netcfg)
-        self.assertEqual(0, self.m_udev_settle.call_count)
-
-    def test_wait_for_physdevs_calls_udev_settle_on_missing(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.side_effect = iter([
-            {'aa:bb:cc:dd:ee:ff': 'eth0'},   # first call ens3 is missing
-            {'aa:bb:cc:dd:ee:ff': 'eth0',
-             '00:11:22:33:44:55': 'ens3'},   # second call has both
-        ])
-        net.wait_for_physdevs(netcfg)
-        self.m_udev_settle.assert_called_with(exists=net.sys_dev_path('ens3'))
-
-    def test_wait_for_physdevs_raise_runtime_error_if_missing_and_strict(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.return_value = {}
-        with self.assertRaises(RuntimeError):
-            net.wait_for_physdevs(netcfg)
-
-        self.assertEqual(5 * len(physdevs), self.m_udev_settle.call_count)
-
-    def test_wait_for_physdevs_no_raise_if_not_strict(self):
-        physdevs = [
-            ['aa:bb:cc:dd:ee:ff', 'eth0', 'virtio', '0x1000'],
-            ['00:11:22:33:44:55', 'ens3', 'e1000', '0x1643'],
-        ]
-        netcfg = {
-            'version': 2,
-            'ethernets': {args[1]: _mk_v2_phys(*args)
-                          for args in physdevs},
-        }
-        self.m_get_iface_mac.return_value = {}
-        net.wait_for_physdevs(netcfg, strict=False)
-        self.assertEqual(5 * len(physdevs), self.m_udev_settle.call_count)
-
-
 class TestNetFailOver(CiTestCase):
 
-    with_logs = True
-
     def setUp(self):
         super(TestNetFailOver, self).setUp()
         self.add_patch('cloudinit.net.util', 'm_util')
@@ -1297,4 +1189,48 @@ class TestNetFailOver(CiTestCase):
         m_standby.return_value = False
         self.assertFalse(net.is_netfailover(devname, driver))
 
+
+class TestIsIpAddress:
+    """Tests for net.is_ip_address.
+
+    Instead of testing with values we rely on the ipaddress stdlib module to
+    handle all values correctly, so simply test that is_ip_address defers to
+    the ipaddress module correctly.
+    """
+
+    @pytest.mark.parametrize('ip_address_side_effect,expected_return', (
+        (ValueError, False),
+        (lambda _: ipaddress.IPv4Address('192.168.0.1'), True),
+        (lambda _: ipaddress.IPv6Address('2001:db8::'), True),
+    ))
+    def test_is_ip_address(self, ip_address_side_effect, expected_return):
+        with mock.patch('cloudinit.net.ipaddress.ip_address',
+                        side_effect=ip_address_side_effect) as m_ip_address:
+            ret = net.is_ip_address(mock.sentinel.ip_address_in)
+        assert expected_return == ret
+        expected_call = mock.call(mock.sentinel.ip_address_in)
+        assert [expected_call] == m_ip_address.call_args_list
+
+
+class TestIsIpv4Address:
+    """Tests for net.is_ipv4_address.
+
+    Instead of testing with values we rely on the ipaddress stdlib module to
+    handle all values correctly, so simply test that is_ipv4_address defers to
+    the ipaddress module correctly.
+    """
+
+    @pytest.mark.parametrize('ipv4address_mock,expected_return', (
+        (mock.Mock(side_effect=ValueError), False),
+        (mock.Mock(return_value=ipaddress.IPv4Address('192.168.0.1')), True),
+    ))
+    def test_is_ip_address(self, ipv4address_mock, expected_return):
+        with mock.patch('cloudinit.net.ipaddress.IPv4Address',
+                        ipv4address_mock) as m_ipv4address:
+            ret = net.is_ipv4_address(mock.sentinel.ip_address_in)
+        assert expected_return == ret
+        expected_call = mock.call(mock.sentinel.ip_address_in)
+        assert [expected_call] == m_ipv4address.call_args_list
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/net/tests/test_network_state.py b/cloudinit/net/tests/test_network_state.py
index 55880852..07d726e2 100644
--- a/cloudinit/net/tests/test_network_state.py
+++ b/cloudinit/net/tests/test_network_state.py
@@ -45,4 +45,14 @@ class TestNetworkStateParseConfig(CiTestCase):
         self.assertNotEqual(None, result)
 
 
+class TestNetworkStateParseConfigV2(CiTestCase):
+
+    def test_version_2_ignores_renderer_key(self):
+        ncfg = {'version': 2, 'renderer': 'networkd', 'ethernets': {}}
+        nsi = network_state.NetworkStateInterpreter(version=ncfg['version'],
+                                                    config=ncfg)
+        nsi.parse_config(skip_broken=False)
+        self.assertEqual(ncfg, nsi.as_dict()['config'])
+
+
 # vi: ts=4 expandtab