ec2: Allow Ec2 to run in init-local using dhclient in a sandbox.

This branch is a prerequisite for IPv6 support in AWS by allowing Ec2
datasource to query the metadata source version 2016-09-02 about whether
or not it needs to configure IPv6 on interfaces. If version 2016-09-02
is not present, fallback to the min_metadata_version of 2009-04-04. The
DataSourceEc2Local not run on FreeBSD because dhclient in doesn't
support the -sf flag allowing us to run dhclient without filesystem
side-effects.

To query AWS' metadata address @ 169.254.169.254, the instance must have
a dhcp-allocated address configured. Configuring IPv4 link-local
addresses result in timeouts from the metadata service. We introduced a
DataSourceEc2Local subclass which will perform a sandboxed dhclient
discovery which obtains an authorized IP address on eth0 and crawl
metadata about full instance network configuration.

Since ec2 IPv6 metadata is not sufficient in itself to tell us all the
ipv6 knownledge we need, it only be used as a boolean to tell us which
nics need IPv6. Cloud-init will then configure desired interfaces to
DHCPv6 versus DHCPv4.

Performance side note: Shifting the dhcp work into init-local for Ec2
actually gets us 1 second faster deployments by skipping init-network
phase of alternate datasource checks because Ec2Local is configured in
an ealier boot stage. In 3 test runs prior to this change: cloud-init
runs were 5.5 seconds, with the change we now average 4.6 seconds.

This efficiency could be even further improved if we avoiding dhcp
discovery in order to talk to the metadata service from an AWS
authorized dhcp address if there were some way to advertize the dhcp
configuration via DMI/SMBIOS or system environment variables.

Inspecting time costs of the dhclient setup/teardown in 3 live runs the
time cost for the dhcp setup round trip on AWS is:
test 1: 76 milliseconds
         dhcp discovery + metadata: 0.347 seconds
         metadata alone: 0.271 seconds
test 2: 88 milliseconds
         dhcp discovery + metadata: 0.388 seconds
         metadata alone: 0.300 seconds
test 3: 75 milliseconds
         dhcp discovery + metadata: 0.366 seconds
         metadata alone: 0.291 seconds

LP: #1709772

4 files changed, 120 insertions, 30 deletions

diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 08c5c469..bf1dc5df 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -278,7 +278,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
         return root
 
 
-class HttprettyTestCase(TestCase):
+class HttprettyTestCase(CiTestCase):
     # necessary as http_proxy gets in the way of httpretty
     # https://github.com/gabrielfalcao/HTTPretty/issues/122
     def setUp(self):
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 990bff2c..996560e4 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
-        self.api_ver = self.ds.api_ver
 
     @property
     def default_metadata(self):
@@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
 
     @property
     def metadata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'meta-data') + '/'
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'meta-data') + '/'
 
     @property
     def userdata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'user-data')
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'user-data')
 
     def regist_default_server(self):
         register_mock_metaserver(self.metadata_url, self.default_metadata)
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 413e87ac..4802f105 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -35,6 +35,7 @@ DEFAULT_LOCAL = [
     OpenNebula.DataSourceOpenNebula,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
+    Ec2.DataSourceEc2Local,
 ]
 
 DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 12230ae2..33d02619 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -8,35 +8,67 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
 
 
-# collected from api version 2009-04-04/ with
+# collected from api version 2016-09-02/ with
 # python3 -c 'import json
 # from cloudinit.ec2_utils import get_instance_metadata as gm
-# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))'
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
 DEFAULT_METADATA = {
-    "ami-id": "ami-80861296",
+    "ami-id": "ami-8b92b4ee",
     "ami-launch-index": "0",
     "ami-manifest-path": "(unknown)",
     "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
-    "hostname": "ip-10-0-0-149",
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
     "instance-action": "none",
-    "instance-id": "i-0052913950685138c",
-    "instance-type": "t2.micro",
-    "local-hostname": "ip-10-0-0-149",
-    "local-ipv4": "10.0.0.149",
-    "placement": {"availability-zone": "us-east-1b"},
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": ("ip-172-3-3-15.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-13-59-77-202.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+                }
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
     "profile": "default-hvm",
-    "public-hostname": "",
-    "public-ipv4": "107.23.188.247",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
     "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
-    "reservation-id": "r-00a2c173fb5782a08",
-    "security-groups": "wide-open"
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"}
 }
 
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
 
-    public-keys in the ec2 metadata is inconsistently formatted compared
+    public-keys in the ec2 metadata is inconsistently formated compared
     to other entries.
     Given keys_data of {name1: pubkey1, name2: pubkey2}
 
@@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data):
 
 
 class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+
     valid_platform_data = {
         'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
         'uuid_source': 'dmi',
@@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
     def setUp(self):
         super(TestEc2, self).setUp()
-        self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0]
-        self.api_ver = '2009-04-04'
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
 
     @property
     def metadata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', ''])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'meta-data', ''])
 
     @property
     def userdata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'user-data'])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'user-data'])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
@@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
-        ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
@@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase):
         return ds
 
     @httpretty.activate
-    def test_valid_platform_with_strict_true(self):
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
 
     @httpretty.activate
     def test_valid_platform_with_strict_false(self):
@@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_true(self):
@@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(False, ret)
+        self.assertFalse(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_false(self):
@@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue())
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with()
+        m_net.assert_called_once_with(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
 # vi: ts=4 expandtab