Reorganize unit test locations under tests/unittests (#1126)

This attempts to standardize unit test file location under test/unittests/
such that any source file located at cloudinit/path/to/file.py may have a
corresponding unit test file at test/unittests/path/to/test_file.py.

Noteworthy Comments:
====================
Four different duplicate test files existed:
test_{gpg,util,cc_mounts,cc_resolv_conf}.py
Each of these duplicate file pairs has been merged together. This is a
break in git history for these files.

The test suite appears to have a dependency on test order. Changing test
order causes some tests to fail. This should be rectified, but for now
some tests have been modified in
tests/unittests/config/test_set_passwords.py.

A helper class name starts with "Test" which causes pytest to try
executing it as a test case, which then throws warnings "due to Class
having __init__()".  Silence by changing the name of the class.

# helpers.py is imported in many test files, import paths change
cloudinit/tests/helpers.py -> tests/unittests/helpers.py

# Move directories:
cloudinit/distros/tests -> tests/unittests/distros
cloudinit/cmd/devel/tests -> tests/unittests/cmd/devel
cloudinit/cmd/tests -> tests/unittests/cmd/
cloudinit/sources/helpers/tests -> tests/unittests/sources/helpers
cloudinit/sources/tests -> tests/unittests/sources
cloudinit/net/tests -> tests/unittests/net
cloudinit/config/tests -> tests/unittests/config
cloudinit/analyze/tests/ -> tests/unittests/analyze/

# Standardize tests already in tests/unittests/
test_datasource -> sources
test_distros -> distros
test_vmware -> sources/vmware
test_handler -> config        # this contains cloudconfig module tests
test_runs -> runs

1 files changed, 978 insertions, 0 deletions

diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py
new file mode 100644
index 00000000..19c2bbcd
--- /dev/null
+++ b/tests/unittests/sources/test_ec2.py
@@ -0,0 +1,978 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import copy
+import httpretty
+import json
+import requests
+from unittest import mock
+
+from cloudinit import helpers
+from cloudinit.sources import DataSourceEc2 as ec2
+from tests.unittests import helpers as test_helpers
+
+
+DYNAMIC_METADATA = {
+    "instance-identity": {
+        "document": json.dumps({
+            "devpayProductCodes": None,
+            "marketplaceProductCodes": ["1abc2defghijklm3nopqrs4tu"],
+            "availabilityZone": "us-west-2b",
+            "privateIp": "10.158.112.84",
+            "version": "2017-09-30",
+            "instanceId": "my-identity-id",
+            "billingProducts": None,
+            "instanceType": "t2.micro",
+            "accountId": "123456789012",
+            "imageId": "ami-5fb8c835",
+            "pendingTime": "2016-11-19T16:32:11Z",
+            "architecture": "x86_64",
+            "kernelId": None,
+            "ramdiskId": None,
+            "region": "us-west-2"
+        })
+    }
+}
+
+
+# collected from api version 2016-09-02/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
+# Note that the MAC addresses have been modified to sort in the opposite order
+# to the device-number attribute, to test LP: #1876312
+DEFAULT_METADATA = {
+    "ami-id": "ami-8b92b4ee",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
+    "instance-action": "none",
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": ("ip-172-3-3-15.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-13-59-77-202.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+                },
+                "06:17:04:d7:26:08": {
+                    "device-number": "1",   # Only IPv4 local config
+                    "interface-id": "eni-e44ef49f",
+                    "ipv4-associations": {"": "172.3.3.16"},
+                    "ipv6s": "",  # No IPv6 config
+                    "local-hostname": ("ip-172-3-3-16.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.16",
+                    "mac": "06:17:04:d7:26:08",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-172-3-3-16.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "",  # No public ipv4 config
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": ""
+                }
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
+    "profile": "default-hvm",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
+    "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"},
+}
+
+# collected from api version 2018-09-24/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
+
+NIC1_MD_IPV4_IPV6_MULTI_IP = {
+    "device-number": "0",
+    "interface-id": "eni-0d6335689899ce9cc",
+    "ipv4-associations": {
+        "18.218.219.181": "172.31.44.13"
+    },
+    "ipv6s": [
+        "2600:1f16:292:100:c187:593c:4349:136",
+        "2600:1f16:292:100:f153:12a3:c37c:11f9",
+        "2600:1f16:292:100:f152:2222:3333:4444"
+    ],
+    "local-hostname": ("ip-172-31-44-13.us-east-2."
+                       "compute.internal"),
+    "local-ipv4s": [
+        "172.31.44.13",
+        "172.31.45.70"
+    ],
+    "mac": "0a:07:84:3d:6e:38",
+    "owner-id": "329910648901",
+    "public-hostname": ("ec2-18-218-219-181.us-east-2."
+                        "compute.amazonaws.com"),
+    "public-ipv4s": "18.218.219.181",
+    "security-group-ids": "sg-0c387755222ba8d2e",
+    "security-groups": "launch-wizard-4",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "subnet_ipv6_cidr_blocks": "2600:1f16:292:100::/64",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+    "vpc_ipv6_cidr_blocks": "2600:1f16:292:100::/56"
+}
+
+NIC2_MD = {
+    "device-number": "1",
+    "interface-id": "eni-043cdce36ded5e79f",
+    "local-hostname": "ip-172-31-47-221.us-east-2.compute.internal",
+    "local-ipv4s": "172.31.47.221",
+    "mac": "0a:75:69:92:e2:16",
+    "owner-id": "329910648901",
+    "security-group-ids": "sg-0d68fef37d8cc9b77",
+    "security-groups": "launch-wizard-17",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16"
+}
+
+SECONDARY_IP_METADATA_2018_09_24 = {
+    "ami-id": "ami-0986c2ac728528ac2",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {
+        "ami": "/dev/sda1",
+        "root": "/dev/sda1"
+    },
+    "events": {
+        "maintenance": {
+            "history": "[]",
+            "scheduled": "[]"
+        }
+    },
+    "hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "identity-credentials": {
+        "ec2": {
+            "info": {
+                "AccountId": "329910648901",
+                "Code": "Success",
+                "LastUpdated": "2019-07-06T14:22:56Z"
+            }
+        }
+    },
+    "instance-action": "none",
+    "instance-id": "i-069e01e8cc43732f8",
+    "instance-type": "t2.micro",
+    "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "local-ipv4": "172.31.44.13",
+    "mac": "0a:07:84:3d:6e:38",
+    "metrics": {
+        "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+    },
+    "network": {
+        "interfaces": {
+            "macs": {
+                "0a:07:84:3d:6e:38": NIC1_MD_IPV4_IPV6_MULTI_IP,
+            }
+        }
+    },
+    "placement": {
+        "availability-zone": "us-east-2c"
+    },
+    "profile": "default-hvm",
+    "public-hostname": (
+        "ec2-18-218-219-181.us-east-2.compute.amazonaws.com"),
+    "public-ipv4": "18.218.219.181",
+    "public-keys": {
+        "yourkeyname,e": [
+            "ssh-rsa AAAAW...DZ yourkeyname"
+        ]
+    },
+    "reservation-id": "r-09b4917135cdd33be",
+    "security-groups": "launch-wizard-4",
+    "services": {
+        "domain": "amazonaws.com",
+        "partition": "aws"
+    }
+}
+
+M_PATH_NET = 'cloudinit.sources.DataSourceEc2.net.'
+
+
+def _register_ssh_keys(rfunc, base_url, keys_data):
+    """handle ssh key inconsistencies.
+
+    public-keys in the ec2 metadata is inconsistently formated compared
+    to other entries.
+    Given keys_data of {name1: pubkey1, name2: pubkey2}
+
+    This registers the following urls:
+       base_url                 0={name1}\n1={name2} # (for each name)
+       base_url/                0={name1}\n1={name2} # (for each name)
+       base_url/0               openssh-key
+       base_url/0/              openssh-key
+       base_url/0/openssh-key   {pubkey1}
+       base_url/0/openssh-key/  {pubkey1}
+       ...
+    """
+
+    base_url = base_url.rstrip("/")
+    odd_index = '\n'.join(
+        ["{0}={1}".format(n, name)
+         for n, name in enumerate(sorted(keys_data))])
+
+    rfunc(base_url, odd_index)
+    rfunc(base_url + "/", odd_index)
+
+    for n, name in enumerate(sorted(keys_data)):
+        val = keys_data[name]
+        if isinstance(val, list):
+            val = '\n'.join(val)
+        burl = base_url + "/%s" % n
+        rfunc(burl, "openssh-key")
+        rfunc(burl + "/", "openssh-key")
+        rfunc(burl + "/%s/openssh-key" % name, val)
+        rfunc(burl + "/%s/openssh-key/" % name, val)
+
+
+def register_mock_metaserver(base_url, data):
+    """Register with httpretty a ec2 metadata like service serving 'data'.
+
+    If given a dictionary, it will populate urls under base_url for
+    that dictionary.  For example, input of
+       {"instance-id": "i-abc", "mac": "00:16:3e:00:00:00"}
+    populates
+       base_url  with 'instance-id\nmac'
+       base_url/ with 'instance-id\nmac'
+       base_url/instance-id with i-abc
+       base_url/mac with 00:16:3e:00:00:00
+    In the index, references to lists or dictionaries have a trailing /.
+    """
+    def register_helper(register, base_url, body):
+        if not isinstance(base_url, str):
+            register(base_url, body)
+            return
+        base_url = base_url.rstrip("/")
+        if isinstance(body, str):
+            register(base_url, body)
+        elif isinstance(body, list):
+            register(base_url, '\n'.join(body) + '\n')
+            register(base_url + '/', '\n'.join(body) + '\n')
+        elif isinstance(body, dict):
+            vals = []
+            for k, v in body.items():
+                if k == 'public-keys':
+                    _register_ssh_keys(
+                        register, base_url + '/public-keys/', v)
+                    continue
+                suffix = k.rstrip("/")
+                if not isinstance(v, (str, list)):
+                    suffix += "/"
+                vals.append(suffix)
+                url = base_url + '/' + suffix
+                register_helper(register, url, v)
+            register(base_url, '\n'.join(vals) + '\n')
+            register(base_url + '/', '\n'.join(vals) + '\n')
+        elif body is None:
+            register(base_url, 'not found', status=404)
+
+    def myreg(*argc, **kwargs):
+        url = argc[0]
+        method = httpretty.PUT if ec2.API_TOKEN_ROUTE in url else httpretty.GET
+        return httpretty.register_uri(method, *argc, **kwargs)
+
+    register_helper(myreg, base_url, data)
+
+
+class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+    maxDiff = None
+
+    valid_platform_data = {
+        'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
+        'uuid_source': 'dmi',
+        'serial': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
+    }
+
+    def setUp(self):
+        super(TestEc2, self).setUp()
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
+        self.tmp = self.tmp_dir()
+
+    def data_url(self, version, data_item='meta-data'):
+        """Return a metadata url based on the version provided."""
+        return '/'.join([self.metadata_addr, version, data_item])
+
+    def _patch_add_cleanup(self, mpath, *args, **kwargs):
+        p = mock.patch(mpath, *args, **kwargs)
+        p.start()
+        self.addCleanup(p.stop)
+
+    def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
+        self.uris = []
+        distro = {}
+        paths = helpers.Paths({'run_dir': self.tmp})
+        if sys_cfg is None:
+            sys_cfg = {}
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        if not md_version:
+            md_version = ds.min_metadata_version
+        if platform_data is not None:
+            self._patch_add_cleanup(
+                "cloudinit.sources.DataSourceEc2._collect_platform_data",
+                return_value=platform_data)
+
+        if md:
+            all_versions = (
+                [ds.min_metadata_version] + ds.extended_metadata_versions)
+            token_url = self.data_url('latest', data_item='api/token')
+            register_mock_metaserver(token_url, 'API-TOKEN')
+            for version in all_versions:
+                metadata_url = self.data_url(version) + '/'
+                if version == md_version:
+                    # Register all metadata for desired version
+                    register_mock_metaserver(
+                        metadata_url, md.get('md', DEFAULT_METADATA))
+                    userdata_url = self.data_url(
+                        version, data_item='user-data')
+                    register_mock_metaserver(userdata_url, md.get('ud', ''))
+                    identity_url = self.data_url(
+                        version, data_item='dynamic/instance-identity')
+                    register_mock_metaserver(
+                        identity_url, md.get('id', DYNAMIC_METADATA))
+                else:
+                    instance_id_url = metadata_url + 'instance-id'
+                    if version == ds.min_metadata_version:
+                        # Add min_metadata_version service availability check
+                        register_mock_metaserver(
+                            instance_id_url, DEFAULT_METADATA['instance-id'])
+                    else:
+                        # Register 404s for all unrequested extended versions
+                        register_mock_metaserver(instance_id_url, None)
+        return ds
+
+    def test_network_config_property_returns_version_2_network_data(self):
+        """network_config property returns network version 2 for metadata"""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': DEFAULT_METADATA})
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = 'eth9'
+            ds.get_data()
+
+        mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': '06:17:04:d7:26:09'}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+                    m_find_fallback.return_value = 'eth9'
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_set_dhcp4(self):
+        """network_config property configures dhcp4 on nics with local-ipv4s.
+
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': DEFAULT_METADATA})
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = 'eth9'
+            ds.get_data()
+
+        mac1 = '06:17:04:d7:26:08'  # IPv4 only in DEFAULT_METADATA
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1.lower()}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+                    m_find_fallback.return_value = 'eth9'
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_secondary_private_ips(self):
+        """network_config property configures any secondary ipv4 addresses.
+
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': SECONDARY_IP_METADATA_2018_09_24})
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = 'eth9'
+            ds.get_data()
+
+        mac1 = '0a:07:84:3d:6e:38'  # 1 secondary IPv4 and 2 secondary IPv6
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1}, 'set-name': 'eth9',
+            'addresses': ['172.31.45.70/20',
+                          '2600:1f16:292:100:f152:2222:3333:4444/128',
+                          '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            'dhcp4': True, 'dhcp6': True}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+                    m_find_fallback.return_value = 'eth9'
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_is_cached_in_datasource(self):
+        """network_config property is cached in DataSourceEc2."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': DEFAULT_METADATA})
+        ds._network_config = {'cached': 'data'}
+        self.assertEqual({'cached': 'data'}, ds.network_config)
+
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp):
+        """Refresh the network_config Ec2 cache if network key is absent.
+
+        This catches an upgrade issue where obj.pkl contained stale metadata
+        which lacked newly required network key.
+        """
+        old_metadata = copy.deepcopy(DEFAULT_METADATA)
+        old_metadata.pop('network')
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': old_metadata})
+        self.assertTrue(ds.get_data())
+        # Provide new revision of metadata that contains network data
+        register_mock_metaserver(
+            'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA)
+        mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
+        get_interface_mac_path = M_PATH_NET + 'get_interfaces_by_mac'
+        ds.fallback_nic = 'eth9'
+        with mock.patch(get_interface_mac_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+            nc = ds.network_config  # Will re-crawl network metadata
+            self.assertIsNotNone(nc)
+        self.assertIn(
+            'Refreshing stale metadata from prior to upgrade',
+            self.logs.getvalue())
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        self.assertEqual(expected, ds.network_config)
+
+    def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
+        """get_instance-id gets DataSourceEc2Local.identity if not present.
+
+        This handles an upgrade case where the old pickled datasource didn't
+        set up self.identity, but 'systemctl cloud-init init' runs
+        get_instance_id which traces on missing self.identity. lp:1748354.
+        """
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        # Mock 404s on all versions except latest
+        all_versions = (
+            [ds.min_metadata_version] + ds.extended_metadata_versions)
+        for ver in all_versions[:-1]:
+            register_mock_metaserver(
+                'http://169.254.169.254/{0}/meta-data/instance-id'.format(ver),
+                None)
+        ds.metadata_address = 'http://169.254.169.254'
+        register_mock_metaserver(
+            '{0}/{1}/meta-data/'.format(ds.metadata_address, all_versions[-1]),
+            DEFAULT_METADATA)
+        # Register dynamic/instance-identity document which we now read.
+        register_mock_metaserver(
+            '{0}/{1}/dynamic/'.format(ds.metadata_address, all_versions[-1]),
+            DYNAMIC_METADATA)
+        ds._cloud_name = ec2.CloudNames.AWS
+        # Setup cached metadata on the Datasource
+        ds.metadata = DEFAULT_METADATA
+        self.assertEqual('my-identity-id', ds.get_instance_id())
+
+    def test_classic_instance_true(self):
+        """If no vpc-id in metadata, is_classic_instance must return true."""
+        md_copy = copy.deepcopy(DEFAULT_METADATA)
+        ifaces_md = md_copy.get('network', {}).get('interfaces', {})
+        for _mac, mac_data in ifaces_md.get('macs', {}).items():
+            if 'vpc-id' in mac_data:
+                del mac_data['vpc-id']
+
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': md_copy})
+        self.assertTrue(ds.get_data())
+        self.assertTrue(ds.is_classic_instance())
+
+    def test_classic_instance_false(self):
+        """If vpc-id in metadata, is_classic_instance must return false."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        self.assertTrue(ds.get_data())
+        self.assertFalse(ds.is_classic_instance())
+
+    def test_aws_inaccessible_imds_service_fails_with_retries(self):
+        """Inaccessibility of http://169.254.169.254 are retried."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=None)
+
+        conn_error = requests.exceptions.ConnectionError(
+            '[Errno 113] no route to host'
+        )
+
+        mock_success = mock.MagicMock(contents=b'fakesuccess')
+        mock_success.ok.return_value = True
+
+        with mock.patch('cloudinit.url_helper.readurl') as m_readurl:
+            m_readurl.side_effect = (conn_error, conn_error, mock_success)
+            with mock.patch('cloudinit.url_helper.time.sleep'):
+                self.assertTrue(ds.wait_for_metadata_service())
+
+        # Just one /latest/api/token request
+        self.assertEqual(3, len(m_readurl.call_args_list))
+        for readurl_call in m_readurl.call_args_list:
+            self.assertIn('latest/api/token', readurl_call[0][0])
+
+    def test_aws_token_403_fails_without_retries(self):
+        """Verify that 403s fetching AWS tokens are not retried."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=None)
+        token_url = self.data_url('latest', data_item='api/token')
+        httpretty.register_uri(httpretty.PUT, token_url, body={}, status=403)
+        self.assertFalse(ds.get_data())
+        # Just one /latest/api/token request
+        logs = self.logs.getvalue()
+        failed_put_log = '"PUT /latest/api/token HTTP/1.1" 403 0'
+        expected_logs = [
+            'WARNING: Ec2 IMDS endpoint returned a 403 error. HTTP endpoint is'
+            ' disabled. Aborting.',
+            "WARNING: IMDS's HTTP endpoint is probably disabled",
+            failed_put_log
+        ]
+        for log in expected_logs:
+            self.assertIn(log, logs)
+        self.assertEqual(
+            1,
+            len([line for line in logs.splitlines() if failed_put_log in line])
+        )
+
+    def test_aws_token_redacted(self):
+        """Verify that aws tokens are redacted when logged."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        self.assertTrue(ds.get_data())
+        all_logs = self.logs.getvalue().splitlines()
+        REDACT_TTL = "'X-aws-ec2-metadata-token-ttl-seconds': 'REDACTED'"
+        REDACT_TOK = "'X-aws-ec2-metadata-token': 'REDACTED'"
+        logs_with_redacted_ttl = [log for log in all_logs if REDACT_TTL in log]
+        logs_with_redacted = [log for log in all_logs if REDACT_TOK in log]
+        logs_with_token = [log for log in all_logs if 'API-TOKEN' in log]
+        self.assertEqual(1, len(logs_with_redacted_ttl))
+        self.assertEqual(81, len(logs_with_redacted))
+        self.assertEqual(0, len(logs_with_token))
+
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_valid_platform_with_strict_true(self, m_dhcp):
+        """Valid platform data should return true with strict_id true."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': DEFAULT_METADATA})
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
+        self.assertEqual('aws', ds.cloud_name)
+        self.assertEqual('ec2', ds.platform_type)
+        self.assertEqual('metadata (%s)' % ds.metadata_address, ds.subplatform)
+
+    def test_valid_platform_with_strict_false(self):
+        """Valid platform data should return true with strict_id false."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        ret = ds.get_data()
+        self.assertTrue(ret)
+
+    def test_unknown_platform_with_strict_true(self):
+        """Unknown platform data with strict_id true should return False."""
+        uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a'
+        ds = self._setup_ds(
+            platform_data={'uuid': uuid, 'uuid_source': 'dmi', 'serial': ''},
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': DEFAULT_METADATA})
+        ret = ds.get_data()
+        self.assertFalse(ret)
+
+    def test_unknown_platform_with_strict_false(self):
+        """Unknown platform data with strict_id false should return True."""
+        uuid = 'ab439480-72bf-11d3-91fc-b8aded755F9a'
+        ds = self._setup_ds(
+            platform_data={'uuid': uuid, 'uuid_source': 'dmi', 'serial': ''},
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        ret = ds.get_data()
+        self.assertTrue(ret)
+
+    def test_ec2_local_returns_false_on_non_aws(self):
+        """DataSourceEc2Local returns False when platform is not AWS."""
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        platform_attrs = [
+            attr for attr in ec2.CloudNames.__dict__.keys()
+            if not attr.startswith('__')]
+        for attr_name in platform_attrs:
+            platform_name = getattr(ec2.CloudNames, attr_name)
+            if platform_name != 'aws':
+                ds._cloud_name = platform_name
+                ret = ds.get_data()
+                self.assertEqual('ec2', ds.platform_type)
+                self.assertFalse(ret)
+                message = (
+                    "Local Ec2 mode only supported on ('aws',),"
+                    ' not {0}'.format(platform_name))
+                self.assertIn(message, self.logs.getvalue())
+
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.find_fallback_nic')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp,
+                                                m_fallback_nic, m_net):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+
+        m_fallback_nic.return_value = 'eth9'
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md={'md': DEFAULT_METADATA})
+
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with('eth9', None)
+        m_net.assert_called_once_with(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1',
+            static_routes=None)
+        self.assertIn('Crawl of metadata service took', self.logs.getvalue())
+
+
+class TestGetSecondaryAddresses(test_helpers.CiTestCase):
+
+    mac = '06:17:04:d7:26:ff'
+    with_logs = True
+
+    def test_md_with_no_secondary_addresses(self):
+        """Empty list is returned when nic metadata contains no secondary ip"""
+        self.assertEqual([], ec2.get_secondary_addresses(NIC2_MD, self.mac))
+
+    def test_md_with_secondary_v4_and_v6_addresses(self):
+        """All secondary addresses are returned from nic metadata"""
+        self.assertEqual(
+            ['172.31.45.70/20', '2600:1f16:292:100:f152:2222:3333:4444/128',
+             '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            ec2.get_secondary_addresses(NIC1_MD_IPV4_IPV6_MULTI_IP, self.mac))
+
+    def test_invalid_ipv4_ipv6_cidr_metadata_logged_with_defaults(self):
+        """Any invalid subnet-ipv(4|6)-cidr-block values use defaults"""
+        invalid_cidr_md = copy.deepcopy(NIC1_MD_IPV4_IPV6_MULTI_IP)
+        invalid_cidr_md['subnet-ipv4-cidr-block'] = "something-unexpected"
+        invalid_cidr_md['subnet-ipv6-cidr-block'] = "not/sure/what/this/is"
+        self.assertEqual(
+            ['172.31.45.70/24', '2600:1f16:292:100:f152:2222:3333:4444/128',
+             '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            ec2.get_secondary_addresses(invalid_cidr_md, self.mac))
+        expected_logs = [
+            "WARNING: Could not parse subnet-ipv4-cidr-block"
+            " something-unexpected for mac 06:17:04:d7:26:ff."
+            " ipv4 network config prefix defaults to /24",
+            "WARNING: Could not parse subnet-ipv6-cidr-block"
+            " not/sure/what/this/is for mac 06:17:04:d7:26:ff."
+            " ipv6 network config prefix defaults to /128"
+        ]
+        logs = self.logs.getvalue()
+        for log in expected_logs:
+            self.assertIn(log, logs)
+
+
+class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
+
+    def setUp(self):
+        super(TestConvertEc2MetadataNetworkConfig, self).setUp()
+        self.mac1 = '06:17:04:d7:26:09'
+        interface_dict = copy.deepcopy(
+            DEFAULT_METADATA['network']['interfaces']['macs'][self.mac1])
+        # These tests are written assuming the base interface doesn't have IPv6
+        interface_dict.pop('ipv6s')
+        self.network_metadata = {
+            'interfaces': {'macs': {self.mac1: interface_dict}}}
+
+    def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
+        """Any mac absent from metadata is skipped by network config."""
+        macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
+
+        # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                self.network_metadata, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_only_dhcp6(self):
+        """Config dhcp6 when ipv6s is in metadata for a mac."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_ipv6['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_local_only_dhcp4(self):
+        """Config dhcp4 when there are no public addresses in public-ipv4s."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_ipv6['interfaces']['macs'][self.mac1])
+        nic1_metadata['local-ipv4s'] = '172.3.3.15'
+        nic1_metadata.pop('public-ipv4s')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_absent_dhcp4(self):
+        """Config dhcp4 on fallback_nic when there are no ipv4 addresses."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_ipv6['interfaces']['macs'][self.mac1])
+        nic1_metadata['public-ipv4s'] = ''
+
+        # When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config.
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics, fallback_nic='eth9'))
+
+    def test_convert_ec2_metadata_network_config_handles_local_v4_and_v6(self):
+        """When ipv6s and local-ipv4s are non-empty, enable dhcp6 and dhcp4."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_both['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')
+        nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_multiple_nics(self):
+        """DHCP route-metric increases on secondary NICs for IPv4 and IPv6."""
+        mac2 = '06:17:04:d7:26:08'
+        macs_to_nics = {self.mac1: 'eth9', mac2: 'eth10'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        # Add 2nd nic info
+        network_metadata_both['interfaces']['macs'][mac2] = NIC2_MD
+        nic1_metadata = (
+            network_metadata_both['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')  # No public-ipv4 IPs in cfg
+        nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
+        expected = {'version': 2, 'ethernets': {
+            'eth9': {
+                'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+                'dhcp4': True, 'dhcp4-overrides': {'route-metric': 100},
+                'dhcp6': True, 'dhcp6-overrides': {'route-metric': 100}},
+            'eth10': {
+                'match': {'macaddress': mac2}, 'set-name': 'eth10',
+                'dhcp4': True, 'dhcp4-overrides': {'route-metric': 200},
+                'dhcp6': False}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_dhcp4_and_dhcp6(self):
+        """Config both dhcp4 and dhcp6 when both vpc-ipv6 and ipv4 exists."""
+        macs_to_nics = {self.mac1: 'eth9'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        nic1_metadata = (
+            network_metadata_both['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
+
+    def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
+        """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1},
+            'set-name': 'eth9', 'dhcp4': True, 'dhcp6': False}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'}
+            self.assertEqual(
+                expected,
+                ec2.convert_ec2_metadata_network_config(self.network_metadata))
+
+
+class TesIdentifyPlatform(test_helpers.CiTestCase):
+
+    def collmock(self, **kwargs):
+        """return non-special _collect_platform_data updated with changes."""
+        unspecial = {
+            'asset_tag': '3857-0037-2746-7462-1818-3997-77',
+            'serial': 'H23-C4J3JV-R6',
+            'uuid': '81c7e555-6471-4833-9551-1ab366c4cfd2',
+            'uuid_source': 'dmi',
+            'vendor': 'tothecloud',
+        }
+        unspecial.update(**kwargs)
+        return unspecial
+
+    @mock.patch('cloudinit.sources.DataSourceEc2._collect_platform_data')
+    def test_identify_zstack(self, m_collect):
+        """zstack should be identified if chassis-asset-tag ends in .zstack.io
+        """
+        m_collect.return_value = self.collmock(asset_tag='123456.zstack.io')
+        self.assertEqual(ec2.CloudNames.ZSTACK, ec2.identify_platform())
+
+    @mock.patch('cloudinit.sources.DataSourceEc2._collect_platform_data')
+    def test_identify_zstack_full_domain_only(self, m_collect):
+        """zstack asset-tag matching should match only on full domain boundary.
+        """
+        m_collect.return_value = self.collmock(asset_tag='123456.buzzstack.io')
+        self.assertEqual(ec2.CloudNames.UNKNOWN, ec2.identify_platform())
+
+    @mock.patch('cloudinit.sources.DataSourceEc2._collect_platform_data')
+    def test_identify_e24cloud(self, m_collect):
+        """e24cloud identified if vendor is e24cloud"""
+        m_collect.return_value = self.collmock(vendor='e24cloud')
+        self.assertEqual(ec2.CloudNames.E24CLOUD, ec2.identify_platform())
+
+    @mock.patch('cloudinit.sources.DataSourceEc2._collect_platform_data')
+    def test_identify_e24cloud_negative(self, m_collect):
+        """e24cloud identified if vendor is e24cloud"""
+        m_collect.return_value = self.collmock(vendor='e24cloudyday')
+        self.assertEqual(ec2.CloudNames.UNKNOWN, ec2.identify_platform())
+
+# vi: ts=4 expandtab