Azure VM Preprovisioning support.

This change will enable azure vms to report provisioning has completed
twice, first to tell the fabric it has completed then a second time to
enable customer settings. The datasource for the second provisioning is
the Instance Metadata Service (IMDS),and the VM will poll indefinitely for
the new ovf-env.xml from IMDS.

This branch introduces EphemeralDHCPv4 which encapsulates common logic
used by both DataSourceEc2 an DataSourceAzure for temporary DHCP
interactions without side-effects.

LP: #1734991

3 files changed, 164 insertions, 7 deletions

diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 6341e1e8..254e9876 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -5,7 +5,7 @@ from cloudinit.util import b64e, decode_binary, load_file, write_file
 from cloudinit.sources import DataSourceAzure as dsaz
 from cloudinit.util import find_freebsd_part
 from cloudinit.util import get_path_dev_freebsd
-
+from cloudinit.version import version_string as vs
 from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
                                      ExitStack, PY26, SkipTest)
 
@@ -16,7 +16,8 @@ import xml.etree.ElementTree as ET
 import yaml
 
 
-def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
+def construct_valid_ovf_env(data=None, pubkeys=None,
+                            userdata=None, platform_settings=None):
     if data is None:
         data = {'HostName': 'FOOHOST'}
     if pubkeys is None:
@@ -66,10 +67,12 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
   xmlns:i="http://www.w3.org/2001/XMLSchema-instance">
  <KmsServerHostname>kms.core.windows.net</KmsServerHostname>
  <ProvisionGuestAgent>false</ProvisionGuestAgent>
- <GuestAgentPackageName i:nil="true" />
- </PlatformSettings></wa:PlatformSettingsSection>
-</Environment>
-    """
+ <GuestAgentPackageName i:nil="true" />"""
+    if platform_settings:
+        for k, v in platform_settings.items():
+            content += "<%s>%s</%s>\n" % (k, v, k)
+    content += """</PlatformSettings></wa:PlatformSettingsSection>
+</Environment>"""
 
     return content
 
@@ -1107,4 +1110,146 @@ class TestAzureNetExists(CiTestCase):
         self.assertTrue(hasattr(dsaz, "DataSourceAzureNet"))
 
 
+@mock.patch('cloudinit.sources.DataSourceAzure.util.subp')
+@mock.patch.object(dsaz, 'get_hostname')
+@mock.patch.object(dsaz, 'set_hostname')
+class TestAzureDataSourcePreprovisioning(CiTestCase):
+
+    def setUp(self):
+        super(TestAzureDataSourcePreprovisioning, self).setUp()
+        tmp = self.tmp_dir()
+        self.waagent_d = self.tmp_path('/var/lib/waagent', tmp)
+        self.paths = helpers.Paths({'cloud_dir': tmp})
+        dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
+
+    def test_read_azure_ovf_with_true_flag(self, *args):
+        """The read_azure_ovf method should set the PreprovisionedVM
+           cfg flag if the proper setting is present."""
+        content = construct_valid_ovf_env(
+            platform_settings={"PreprovisionedVm": "True"})
+        ret = dsaz.read_azure_ovf(content)
+        cfg = ret[2]
+        self.assertTrue(cfg['PreprovisionedVm'])
+
+    def test_read_azure_ovf_with_false_flag(self, *args):
+        """The read_azure_ovf method should set the PreprovisionedVM
+           cfg flag to false if the proper setting is false."""
+        content = construct_valid_ovf_env(
+            platform_settings={"PreprovisionedVm": "False"})
+        ret = dsaz.read_azure_ovf(content)
+        cfg = ret[2]
+        self.assertFalse(cfg['PreprovisionedVm'])
+
+    def test_read_azure_ovf_without_flag(self, *args):
+        """The read_azure_ovf method should not set the
+           PreprovisionedVM cfg flag."""
+        content = construct_valid_ovf_env()
+        ret = dsaz.read_azure_ovf(content)
+        cfg = ret[2]
+        self.assertFalse(cfg['PreprovisionedVm'])
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
+    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('requests.Session.request')
+    def test_poll_imds_returns_ovf_env(self, fake_resp, m_dhcp, m_net,
+                                       m_is_bsd, *args):
+        """The _poll_imds method should return the ovf_env.xml."""
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0'}]
+        url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02'
+        host = "169.254.169.254"
+        full_url = url.format(host)
+        fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf")
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertTrue(len(dsa._poll_imds()) > 0)
+        self.assertEqual(fake_resp.call_args_list,
+                         [mock.call(allow_redirects=True,
+                                    headers={'Metadata': 'true',
+                                             'User-Agent':
+                                             'Cloud-Init/%s' % vs()
+                                             }, method='GET', timeout=60.0,
+                                    url=full_url),
+                          mock.call(allow_redirects=True,
+                                    headers={'Metadata': 'true',
+                                             'User-Agent':
+                                             'Cloud-Init/%s' % vs()
+                                             }, method='GET', url=full_url)])
+        self.assertEqual(m_dhcp.call_count, 1)
+        m_net.assert_any_call(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertEqual(m_net.call_count, 1)
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
+    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('requests.Session.request')
+    def test__reprovision_calls__poll_imds(self, fake_resp, m_dhcp, m_net,
+                                           m_is_bsd, *args):
+        """The _reprovision method should call poll IMDS."""
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'unknown-245': '624c3620'}]
+        url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02'
+        host = "169.254.169.254"
+        full_url = url.format(host)
+        hostname = "myhost"
+        username = "myuser"
+        odata = {'HostName': hostname, 'UserName': username}
+        content = construct_valid_ovf_env(data=odata)
+        fake_resp.return_value = mock.MagicMock(status_code=200, text=content)
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        md, ud, cfg, d = dsa._reprovision()
+        self.assertEqual(md['local-hostname'], hostname)
+        self.assertEqual(cfg['system_info']['default_user']['name'], username)
+        self.assertEqual(fake_resp.call_args_list,
+                         [mock.call(allow_redirects=True,
+                                    headers={'Metadata': 'true',
+                                             'User-Agent':
+                                             'Cloud-Init/%s' % vs()},
+                                    method='GET', timeout=60.0, url=full_url),
+                          mock.call(allow_redirects=True,
+                                    headers={'Metadata': 'true',
+                                             'User-Agent':
+                                             'Cloud-Init/%s' % vs()},
+                                    method='GET', url=full_url)])
+        self.assertEqual(m_dhcp.call_count, 1)
+        m_net.assert_any_call(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertEqual(m_net.call_count, 1)
+
+    @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file')
+    @mock.patch('os.path.isfile')
+    def test__should_reprovision_with_true_cfg(self, isfile, write_f, *args):
+        """The _should_reprovision method should return true with config
+           flag present."""
+        isfile.return_value = False
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertTrue(dsa._should_reprovision(
+            (None, None, {'PreprovisionedVm': True}, None)))
+
+    @mock.patch('os.path.isfile')
+    def test__should_reprovision_with_file_existing(self, isfile, *args):
+        """The _should_reprovision method should return True if the sentinal
+           exists."""
+        isfile.return_value = True
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertTrue(dsa._should_reprovision(
+            (None, None, {'preprovisionedvm': False}, None)))
+
+    @mock.patch('os.path.isfile')
+    def test__should_reprovision_returns_false(self, isfile, *args):
+        """The _should_reprovision method should return False
+           if config and sentinal are not present."""
+        isfile.return_value = False
+        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
+        self.assertFalse(dsa._should_reprovision((None, None, {}, None)))
+
+
 # vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index f0dc8338..0f7267bb 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -425,7 +425,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             self.logs.getvalue())
 
     @httpretty.activate
-    @mock.patch('cloudinit.net.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
     @mock.patch('cloudinit.net.find_fallback_nic')
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
     @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index ddea13d7..ac33e8ef 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -2948,4 +2948,16 @@ class TestRenameInterfaces(CiTestCase):
         mock_subp.assert_has_calls(expected)
 
 
+class TestNetworkState(CiTestCase):
+
+    def test_bcast_addr(self):
+        """Test mask_and_ipv4_to_bcast_addr proper execution."""
+        bcast_addr = network_state.mask_and_ipv4_to_bcast_addr
+        self.assertEqual("192.168.1.255",
+                         bcast_addr("255.255.255.0", "192.168.1.1"))
+        self.assertEqual("128.42.7.255",
+                         bcast_addr("255.255.248.0", "128.42.5.4"))
+        self.assertEqual("10.1.21.255",
+                         bcast_addr("255.255.255.0", "10.1.21.4"))
+
 # vi: ts=4 expandtab