T2117: Cloud-init updated to 22.1

Merged with 22.1 tag from the upstream Cloud-init repository.

Our modules were slightly modified for compatibility with the new
version.

1 files changed, 1125 insertions, 0 deletions

diff --git a/tests/unittests/sources/test_ec2.py b/tests/unittests/sources/test_ec2.py
new file mode 100644
index 00000000..b376660d
--- /dev/null
+++ b/tests/unittests/sources/test_ec2.py
@@ -0,0 +1,1125 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import copy
+import json
+from unittest import mock
+
+import httpretty
+import requests
+
+from cloudinit import helpers
+from cloudinit.sources import DataSourceEc2 as ec2
+from tests.unittests import helpers as test_helpers
+
+DYNAMIC_METADATA = {
+    "instance-identity": {
+        "document": json.dumps(
+            {
+                "devpayProductCodes": None,
+                "marketplaceProductCodes": ["1abc2defghijklm3nopqrs4tu"],
+                "availabilityZone": "us-west-2b",
+                "privateIp": "10.158.112.84",
+                "version": "2017-09-30",
+                "instanceId": "my-identity-id",
+                "billingProducts": None,
+                "instanceType": "t2.micro",
+                "accountId": "123456789012",
+                "imageId": "ami-5fb8c835",
+                "pendingTime": "2016-11-19T16:32:11Z",
+                "architecture": "x86_64",
+                "kernelId": None,
+                "ramdiskId": None,
+                "region": "us-west-2",
+            }
+        )
+    }
+}
+
+
+# collected from api version 2016-09-02/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
+# Note that the MAC addresses have been modified to sort in the opposite order
+# to the device-number attribute, to test LP: #1876312
+DEFAULT_METADATA = {
+    "ami-id": "ami-8b92b4ee",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
+    "instance-action": "none",
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": '<?xml version="1.0" encoding="UTF-8"?>'},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": (
+                        "ip-172-3-3-15.us-east-2.compute.internal"
+                    ),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": (
+                        "ec2-13-59-77-202.us-east-2.compute.amazonaws.com"
+                    ),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56",
+                },
+                "06:17:04:d7:26:08": {
+                    "device-number": "1",  # Only IPv4 local config
+                    "interface-id": "eni-e44ef49f",
+                    "ipv4-associations": {"": "172.3.3.16"},
+                    "ipv6s": "",  # No IPv6 config
+                    "local-hostname": (
+                        "ip-172-3-3-16.us-east-2.compute.internal"
+                    ),
+                    "local-ipv4s": "172.3.3.16",
+                    "mac": "06:17:04:d7:26:08",
+                    "owner-id": "950047163771",
+                    "public-hostname": (
+                        "ec2-172-3-3-16.us-east-2.compute.amazonaws.com"
+                    ),
+                    "public-ipv4s": "",  # No public ipv4 config
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "",
+                },
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
+    "profile": "default-hvm",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
+    "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"},
+}
+
+# collected from api version 2018-09-24/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
+
+NIC1_MD_IPV4_IPV6_MULTI_IP = {
+    "device-number": "0",
+    "interface-id": "eni-0d6335689899ce9cc",
+    "ipv4-associations": {"18.218.219.181": "172.31.44.13"},
+    "ipv6s": [
+        "2600:1f16:292:100:c187:593c:4349:136",
+        "2600:1f16:292:100:f153:12a3:c37c:11f9",
+        "2600:1f16:292:100:f152:2222:3333:4444",
+    ],
+    "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "local-ipv4s": ["172.31.44.13", "172.31.45.70"],
+    "mac": "0a:07:84:3d:6e:38",
+    "owner-id": "329910648901",
+    "public-hostname": "ec2-18-218-219-181.us-east-2.compute.amazonaws.com",
+    "public-ipv4s": "18.218.219.181",
+    "security-group-ids": "sg-0c387755222ba8d2e",
+    "security-groups": "launch-wizard-4",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "subnet_ipv6_cidr_blocks": "2600:1f16:292:100::/64",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+    "vpc_ipv6_cidr_blocks": "2600:1f16:292:100::/56",
+}
+
+NIC2_MD = {
+    "device-number": "1",
+    "interface-id": "eni-043cdce36ded5e79f",
+    "local-hostname": "ip-172-31-47-221.us-east-2.compute.internal",
+    "local-ipv4s": "172.31.47.221",
+    "mac": "0a:75:69:92:e2:16",
+    "owner-id": "329910648901",
+    "security-group-ids": "sg-0d68fef37d8cc9b77",
+    "security-groups": "launch-wizard-17",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+}
+
+SECONDARY_IP_METADATA_2018_09_24 = {
+    "ami-id": "ami-0986c2ac728528ac2",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
+    "events": {"maintenance": {"history": "[]", "scheduled": "[]"}},
+    "hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "identity-credentials": {
+        "ec2": {
+            "info": {
+                "AccountId": "329910648901",
+                "Code": "Success",
+                "LastUpdated": "2019-07-06T14:22:56Z",
+            }
+        }
+    },
+    "instance-action": "none",
+    "instance-id": "i-069e01e8cc43732f8",
+    "instance-type": "t2.micro",
+    "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "local-ipv4": "172.31.44.13",
+    "mac": "0a:07:84:3d:6e:38",
+    "metrics": {"vhostmd": '<?xml version="1.0" encoding="UTF-8"?>'},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "0a:07:84:3d:6e:38": NIC1_MD_IPV4_IPV6_MULTI_IP,
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2c"},
+    "profile": "default-hvm",
+    "public-hostname": "ec2-18-218-219-181.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "18.218.219.181",
+    "public-keys": {"yourkeyname,e": ["ssh-rsa AAAAW...DZ yourkeyname"]},
+    "reservation-id": "r-09b4917135cdd33be",
+    "security-groups": "launch-wizard-4",
+    "services": {"domain": "amazonaws.com", "partition": "aws"},
+}
+
+M_PATH_NET = "cloudinit.sources.DataSourceEc2.net."
+
+
+def _register_ssh_keys(rfunc, base_url, keys_data):
+    """handle ssh key inconsistencies.
+
+    public-keys in the ec2 metadata is inconsistently formated compared
+    to other entries.
+    Given keys_data of {name1: pubkey1, name2: pubkey2}
+
+    This registers the following urls:
+       base_url                 0={name1}\n1={name2} # (for each name)
+       base_url/                0={name1}\n1={name2} # (for each name)
+       base_url/0               openssh-key
+       base_url/0/              openssh-key
+       base_url/0/openssh-key   {pubkey1}
+       base_url/0/openssh-key/  {pubkey1}
+       ...
+    """
+
+    base_url = base_url.rstrip("/")
+    odd_index = "\n".join(
+        ["{0}={1}".format(n, name) for n, name in enumerate(sorted(keys_data))]
+    )
+
+    rfunc(base_url, odd_index)
+    rfunc(base_url + "/", odd_index)
+
+    for n, name in enumerate(sorted(keys_data)):
+        val = keys_data[name]
+        if isinstance(val, list):
+            val = "\n".join(val)
+        burl = base_url + "/%s" % n
+        rfunc(burl, "openssh-key")
+        rfunc(burl + "/", "openssh-key")
+        rfunc(burl + "/%s/openssh-key" % name, val)
+        rfunc(burl + "/%s/openssh-key/" % name, val)
+
+
+def register_mock_metaserver(base_url, data):
+    """Register with httpretty a ec2 metadata like service serving 'data'.
+
+    If given a dictionary, it will populate urls under base_url for
+    that dictionary.  For example, input of
+       {"instance-id": "i-abc", "mac": "00:16:3e:00:00:00"}
+    populates
+       base_url  with 'instance-id\nmac'
+       base_url/ with 'instance-id\nmac'
+       base_url/instance-id with i-abc
+       base_url/mac with 00:16:3e:00:00:00
+    In the index, references to lists or dictionaries have a trailing /.
+    """
+
+    def register_helper(register, base_url, body):
+        if not isinstance(base_url, str):
+            register(base_url, body)
+            return
+        base_url = base_url.rstrip("/")
+        if isinstance(body, str):
+            register(base_url, body)
+        elif isinstance(body, list):
+            register(base_url, "\n".join(body) + "\n")
+            register(base_url + "/", "\n".join(body) + "\n")
+        elif isinstance(body, dict):
+            vals = []
+            for k, v in body.items():
+                if k == "public-keys":
+                    _register_ssh_keys(register, base_url + "/public-keys/", v)
+                    continue
+                suffix = k.rstrip("/")
+                if not isinstance(v, (str, list)):
+                    suffix += "/"
+                vals.append(suffix)
+                url = base_url + "/" + suffix
+                register_helper(register, url, v)
+            register(base_url, "\n".join(vals) + "\n")
+            register(base_url + "/", "\n".join(vals) + "\n")
+        elif body is None:
+            register(base_url, "not found", status=404)
+
+    def myreg(*argc, **kwargs):
+        url = argc[0]
+        method = httpretty.PUT if ec2.API_TOKEN_ROUTE in url else httpretty.GET
+        return httpretty.register_uri(method, *argc, **kwargs)
+
+    register_helper(myreg, base_url, data)
+
+
+class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+    maxDiff = None
+
+    valid_platform_data = {
+        "uuid": "ec212f79-87d1-2f1d-588f-d86dc0fd5412",
+        "uuid_source": "dmi",
+        "serial": "ec212f79-87d1-2f1d-588f-d86dc0fd5412",
+    }
+
+    def setUp(self):
+        super(TestEc2, self).setUp()
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
+        self.tmp = self.tmp_dir()
+
+    def data_url(self, version, data_item="meta-data"):
+        """Return a metadata url based on the version provided."""
+        return "/".join([self.metadata_addr, version, data_item])
+
+    def _patch_add_cleanup(self, mpath, *args, **kwargs):
+        p = mock.patch(mpath, *args, **kwargs)
+        p.start()
+        self.addCleanup(p.stop)
+
+    def _setup_ds(self, sys_cfg, platform_data, md, md_version=None):
+        self.uris = []
+        distro = {}
+        paths = helpers.Paths({"run_dir": self.tmp})
+        if sys_cfg is None:
+            sys_cfg = {}
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        if not md_version:
+            md_version = ds.min_metadata_version
+        if platform_data is not None:
+            self._patch_add_cleanup(
+                "cloudinit.sources.DataSourceEc2._collect_platform_data",
+                return_value=platform_data,
+            )
+
+        if md:
+            all_versions = [
+                ds.min_metadata_version
+            ] + ds.extended_metadata_versions
+            token_url = self.data_url("latest", data_item="api/token")
+            register_mock_metaserver(token_url, "API-TOKEN")
+            for version in all_versions:
+                metadata_url = self.data_url(version) + "/"
+                if version == md_version:
+                    # Register all metadata for desired version
+                    register_mock_metaserver(
+                        metadata_url, md.get("md", DEFAULT_METADATA)
+                    )
+                    userdata_url = self.data_url(
+                        version, data_item="user-data"
+                    )
+                    register_mock_metaserver(userdata_url, md.get("ud", ""))
+                    identity_url = self.data_url(
+                        version, data_item="dynamic/instance-identity"
+                    )
+                    register_mock_metaserver(
+                        identity_url, md.get("id", DYNAMIC_METADATA)
+                    )
+                else:
+                    instance_id_url = metadata_url + "instance-id"
+                    if version == ds.min_metadata_version:
+                        # Add min_metadata_version service availability check
+                        register_mock_metaserver(
+                            instance_id_url, DEFAULT_METADATA["instance-id"]
+                        )
+                    else:
+                        # Register 404s for all unrequested extended versions
+                        register_mock_metaserver(instance_id_url, None)
+        return ds
+
+    def test_network_config_property_returns_version_2_network_data(self):
+        """network_config property returns network version 2 for metadata"""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        find_fallback_path = M_PATH_NET + "find_fallback_nic"
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = "eth9"
+            ds.get_data()
+
+        mac1 = "06:17:04:d7:26:09"  # Defined in DEFAULT_METADATA
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": "06:17:04:d7:26:09"},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": True,
+                }
+            },
+        }
+        patch_path = M_PATH_NET + "get_interfaces_by_mac"
+        get_interface_mac_path = M_PATH_NET + "get_interface_mac"
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: "eth9"}
+                    m_find_fallback.return_value = "eth9"
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_set_dhcp4(self):
+        """network_config property configures dhcp4 on nics with local-ipv4s.
+
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        find_fallback_path = M_PATH_NET + "find_fallback_nic"
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = "eth9"
+            ds.get_data()
+
+        mac1 = "06:17:04:d7:26:08"  # IPv4 only in DEFAULT_METADATA
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": mac1.lower()},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": False,
+                }
+            },
+        }
+        patch_path = M_PATH_NET + "get_interfaces_by_mac"
+        get_interface_mac_path = M_PATH_NET + "get_interface_mac"
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: "eth9"}
+                    m_find_fallback.return_value = "eth9"
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_secondary_private_ips(self):
+        """network_config property configures any secondary ipv4 addresses.
+
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": SECONDARY_IP_METADATA_2018_09_24},
+        )
+        find_fallback_path = M_PATH_NET + "find_fallback_nic"
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = "eth9"
+            ds.get_data()
+
+        mac1 = "0a:07:84:3d:6e:38"  # 1 secondary IPv4 and 2 secondary IPv6
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": mac1},
+                    "set-name": "eth9",
+                    "addresses": [
+                        "172.31.45.70/20",
+                        "2600:1f16:292:100:f152:2222:3333:4444/128",
+                        "2600:1f16:292:100:f153:12a3:c37c:11f9/128",
+                    ],
+                    "dhcp4": True,
+                    "dhcp6": True,
+                }
+            },
+        }
+        patch_path = M_PATH_NET + "get_interfaces_by_mac"
+        get_interface_mac_path = M_PATH_NET + "get_interface_mac"
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: "eth9"}
+                    m_find_fallback.return_value = "eth9"
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_is_cached_in_datasource(self):
+        """network_config property is cached in DataSourceEc2."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        ds._network_config = {"cached": "data"}
+        self.assertEqual({"cached": "data"}, ds.network_config)
+
+    @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery")
+    def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp):
+        """Refresh the network_config Ec2 cache if network key is absent.
+
+        This catches an upgrade issue where obj.pkl contained stale metadata
+        which lacked newly required network key.
+        """
+        old_metadata = copy.deepcopy(DEFAULT_METADATA)
+        old_metadata.pop("network")
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": old_metadata},
+        )
+        self.assertTrue(ds.get_data())
+        # Provide new revision of metadata that contains network data
+        register_mock_metaserver(
+            "http://169.254.169.254/2009-04-04/meta-data/", DEFAULT_METADATA
+        )
+        mac1 = "06:17:04:d7:26:09"  # Defined in DEFAULT_METADATA
+        get_interface_mac_path = M_PATH_NET + "get_interfaces_by_mac"
+        ds.fallback_nic = "eth9"
+        with mock.patch(get_interface_mac_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {mac1: "eth9"}
+            nc = ds.network_config  # Will re-crawl network metadata
+            self.assertIsNotNone(nc)
+        self.assertIn(
+            "Refreshing stale metadata from prior to upgrade",
+            self.logs.getvalue(),
+        )
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": True,
+                }
+            },
+        }
+        self.assertEqual(expected, ds.network_config)
+
+    def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
+        """get_instance-id gets DataSourceEc2Local.identity if not present.
+
+        This handles an upgrade case where the old pickled datasource didn't
+        set up self.identity, but 'systemctl cloud-init init' runs
+        get_instance_id which traces on missing self.identity. lp:1748354.
+        """
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        # Mock 404s on all versions except latest
+        all_versions = [
+            ds.min_metadata_version
+        ] + ds.extended_metadata_versions
+        for ver in all_versions[:-1]:
+            register_mock_metaserver(
+                "http://169.254.169.254/{0}/meta-data/instance-id".format(ver),
+                None,
+            )
+        ds.metadata_address = "http://169.254.169.254"
+        register_mock_metaserver(
+            "{0}/{1}/meta-data/".format(ds.metadata_address, all_versions[-1]),
+            DEFAULT_METADATA,
+        )
+        # Register dynamic/instance-identity document which we now read.
+        register_mock_metaserver(
+            "{0}/{1}/dynamic/".format(ds.metadata_address, all_versions[-1]),
+            DYNAMIC_METADATA,
+        )
+        ds._cloud_name = ec2.CloudNames.AWS
+        # Setup cached metadata on the Datasource
+        ds.metadata = DEFAULT_METADATA
+        self.assertEqual("my-identity-id", ds.get_instance_id())
+
+    def test_classic_instance_true(self):
+        """If no vpc-id in metadata, is_classic_instance must return true."""
+        md_copy = copy.deepcopy(DEFAULT_METADATA)
+        ifaces_md = md_copy.get("network", {}).get("interfaces", {})
+        for _mac, mac_data in ifaces_md.get("macs", {}).items():
+            if "vpc-id" in mac_data:
+                del mac_data["vpc-id"]
+
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": md_copy},
+        )
+        self.assertTrue(ds.get_data())
+        self.assertTrue(ds.is_classic_instance())
+
+    def test_classic_instance_false(self):
+        """If vpc-id in metadata, is_classic_instance must return false."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        self.assertTrue(ds.get_data())
+        self.assertFalse(ds.is_classic_instance())
+
+    def test_aws_inaccessible_imds_service_fails_with_retries(self):
+        """Inaccessibility of http://169.254.169.254 are retried."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md=None,
+        )
+
+        conn_error = requests.exceptions.ConnectionError(
+            "[Errno 113] no route to host"
+        )
+
+        mock_success = mock.MagicMock(contents=b"fakesuccess")
+        mock_success.ok.return_value = True
+
+        with mock.patch("cloudinit.url_helper.readurl") as m_readurl:
+            m_readurl.side_effect = (conn_error, conn_error, mock_success)
+            with mock.patch("cloudinit.url_helper.time.sleep"):
+                self.assertTrue(ds.wait_for_metadata_service())
+
+        # Just one /latest/api/token request
+        self.assertEqual(3, len(m_readurl.call_args_list))
+        for readurl_call in m_readurl.call_args_list:
+            self.assertIn("latest/api/token", readurl_call[0][0])
+
+    def test_aws_token_403_fails_without_retries(self):
+        """Verify that 403s fetching AWS tokens are not retried."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md=None,
+        )
+        token_url = self.data_url("latest", data_item="api/token")
+        httpretty.register_uri(httpretty.PUT, token_url, body={}, status=403)
+        self.assertFalse(ds.get_data())
+        # Just one /latest/api/token request
+        logs = self.logs.getvalue()
+        failed_put_log = '"PUT /latest/api/token HTTP/1.1" 403 0'
+        expected_logs = [
+            "WARNING: Ec2 IMDS endpoint returned a 403 error. HTTP endpoint is"
+            " disabled. Aborting.",
+            "WARNING: IMDS's HTTP endpoint is probably disabled",
+            failed_put_log,
+        ]
+        for log in expected_logs:
+            self.assertIn(log, logs)
+        self.assertEqual(
+            1,
+            len(
+                [line for line in logs.splitlines() if failed_put_log in line]
+            ),
+        )
+
+    def test_aws_token_redacted(self):
+        """Verify that aws tokens are redacted when logged."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        self.assertTrue(ds.get_data())
+        all_logs = self.logs.getvalue().splitlines()
+        REDACT_TTL = "'X-aws-ec2-metadata-token-ttl-seconds': 'REDACTED'"
+        REDACT_TOK = "'X-aws-ec2-metadata-token': 'REDACTED'"
+        logs_with_redacted_ttl = [log for log in all_logs if REDACT_TTL in log]
+        logs_with_redacted = [log for log in all_logs if REDACT_TOK in log]
+        logs_with_token = [log for log in all_logs if "API-TOKEN" in log]
+        self.assertEqual(1, len(logs_with_redacted_ttl))
+        self.assertEqual(81, len(logs_with_redacted))
+        self.assertEqual(0, len(logs_with_token))
+
+    @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery")
+    def test_valid_platform_with_strict_true(self, m_dhcp):
+        """Valid platform data should return true with strict_id true."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
+        self.assertEqual("aws", ds.cloud_name)
+        self.assertEqual("ec2", ds.platform_type)
+        self.assertEqual("metadata (%s)" % ds.metadata_address, ds.subplatform)
+
+    def test_valid_platform_with_strict_false(self):
+        """Valid platform data should return true with strict_id false."""
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        ret = ds.get_data()
+        self.assertTrue(ret)
+
+    def test_unknown_platform_with_strict_true(self):
+        """Unknown platform data with strict_id true should return False."""
+        uuid = "ab439480-72bf-11d3-91fc-b8aded755F9a"
+        ds = self._setup_ds(
+            platform_data={"uuid": uuid, "uuid_source": "dmi", "serial": ""},
+            sys_cfg={"datasource": {"Ec2": {"strict_id": True}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        ret = ds.get_data()
+        self.assertFalse(ret)
+
+    def test_unknown_platform_with_strict_false(self):
+        """Unknown platform data with strict_id false should return True."""
+        uuid = "ab439480-72bf-11d3-91fc-b8aded755F9a"
+        ds = self._setup_ds(
+            platform_data={"uuid": uuid, "uuid_source": "dmi", "serial": ""},
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        ret = ds.get_data()
+        self.assertTrue(ret)
+
+    def test_ec2_local_returns_false_on_non_aws(self):
+        """DataSourceEc2Local returns False when platform is not AWS."""
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        platform_attrs = [
+            attr
+            for attr in ec2.CloudNames.__dict__.keys()
+            if not attr.startswith("__")
+        ]
+        for attr_name in platform_attrs:
+            platform_name = getattr(ec2.CloudNames, attr_name)
+            if platform_name != "aws":
+                ds._cloud_name = platform_name
+                ret = ds.get_data()
+                self.assertEqual("ec2", ds.platform_type)
+                self.assertFalse(ret)
+                message = (
+                    "Local Ec2 mode only supported on ('aws',),"
+                    " not {0}".format(platform_name)
+                )
+                self.assertIn(message, self.logs.getvalue())
+
+    @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD")
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue(),
+        )
+
+    @mock.patch("cloudinit.net.dhcp.EphemeralIPv4Network")
+    @mock.patch("cloudinit.net.find_fallback_nic")
+    @mock.patch("cloudinit.net.dhcp.maybe_perform_dhcp_discovery")
+    @mock.patch("cloudinit.sources.DataSourceEc2.util.is_FreeBSD")
+    def test_ec2_local_performs_dhcp_on_non_bsd(
+        self, m_is_bsd, m_dhcp, m_fallback_nic, m_net
+    ):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+
+        m_fallback_nic.return_value = "eth9"
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [
+            {
+                "interface": "eth9",
+                "fixed-address": "192.168.2.9",
+                "routers": "192.168.2.1",
+                "subnet-mask": "255.255.255.0",
+                "broadcast-address": "192.168.2.255",
+            }
+        ]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={"datasource": {"Ec2": {"strict_id": False}}},
+            md={"md": DEFAULT_METADATA},
+        )
+
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with("eth9", None)
+        m_net.assert_called_once_with(
+            broadcast="192.168.2.255",
+            interface="eth9",
+            ip="192.168.2.9",
+            prefix_or_mask="255.255.255.0",
+            router="192.168.2.1",
+            static_routes=None,
+        )
+        self.assertIn("Crawl of metadata service took", self.logs.getvalue())
+
+
+class TestGetSecondaryAddresses(test_helpers.CiTestCase):
+
+    mac = "06:17:04:d7:26:ff"
+    with_logs = True
+
+    def test_md_with_no_secondary_addresses(self):
+        """Empty list is returned when nic metadata contains no secondary ip"""
+        self.assertEqual([], ec2.get_secondary_addresses(NIC2_MD, self.mac))
+
+    def test_md_with_secondary_v4_and_v6_addresses(self):
+        """All secondary addresses are returned from nic metadata"""
+        self.assertEqual(
+            [
+                "172.31.45.70/20",
+                "2600:1f16:292:100:f152:2222:3333:4444/128",
+                "2600:1f16:292:100:f153:12a3:c37c:11f9/128",
+            ],
+            ec2.get_secondary_addresses(NIC1_MD_IPV4_IPV6_MULTI_IP, self.mac),
+        )
+
+    def test_invalid_ipv4_ipv6_cidr_metadata_logged_with_defaults(self):
+        """Any invalid subnet-ipv(4|6)-cidr-block values use defaults"""
+        invalid_cidr_md = copy.deepcopy(NIC1_MD_IPV4_IPV6_MULTI_IP)
+        invalid_cidr_md["subnet-ipv4-cidr-block"] = "something-unexpected"
+        invalid_cidr_md["subnet-ipv6-cidr-block"] = "not/sure/what/this/is"
+        self.assertEqual(
+            [
+                "172.31.45.70/24",
+                "2600:1f16:292:100:f152:2222:3333:4444/128",
+                "2600:1f16:292:100:f153:12a3:c37c:11f9/128",
+            ],
+            ec2.get_secondary_addresses(invalid_cidr_md, self.mac),
+        )
+        expected_logs = [
+            "WARNING: Could not parse subnet-ipv4-cidr-block"
+            " something-unexpected for mac 06:17:04:d7:26:ff."
+            " ipv4 network config prefix defaults to /24",
+            "WARNING: Could not parse subnet-ipv6-cidr-block"
+            " not/sure/what/this/is for mac 06:17:04:d7:26:ff."
+            " ipv6 network config prefix defaults to /128",
+        ]
+        logs = self.logs.getvalue()
+        for log in expected_logs:
+            self.assertIn(log, logs)
+
+
+class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
+    def setUp(self):
+        super(TestConvertEc2MetadataNetworkConfig, self).setUp()
+        self.mac1 = "06:17:04:d7:26:09"
+        interface_dict = copy.deepcopy(
+            DEFAULT_METADATA["network"]["interfaces"]["macs"][self.mac1]
+        )
+        # These tests are written assuming the base interface doesn't have IPv6
+        interface_dict.pop("ipv6s")
+        self.network_metadata = {
+            "interfaces": {"macs": {self.mac1: interface_dict}}
+        }
+
+    def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
+        """Any mac absent from metadata is skipped by network config."""
+        macs_to_nics = {self.mac1: "eth9", "DE:AD:BE:EF:FF:FF": "vitualnic2"}
+
+        # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": False,
+                }
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                self.network_metadata, macs_to_nics
+            ),
+        )
+
+    def test_convert_ec2_metadata_network_config_handles_only_dhcp6(self):
+        """Config dhcp6 when ipv6s is in metadata for a mac."""
+        macs_to_nics = {self.mac1: "eth9"}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = network_metadata_ipv6["interfaces"]["macs"][self.mac1]
+        nic1_metadata["ipv6s"] = "2620:0:1009:fd00:e442:c88d:c04d:dc85/64"
+        nic1_metadata.pop("public-ipv4s")
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": True,
+                }
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics
+            ),
+        )
+
+    def test_convert_ec2_metadata_network_config_local_only_dhcp4(self):
+        """Config dhcp4 when there are no public addresses in public-ipv4s."""
+        macs_to_nics = {self.mac1: "eth9"}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = network_metadata_ipv6["interfaces"]["macs"][self.mac1]
+        nic1_metadata["local-ipv4s"] = "172.3.3.15"
+        nic1_metadata.pop("public-ipv4s")
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": False,
+                }
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics
+            ),
+        )
+
+    def test_convert_ec2_metadata_network_config_handles_absent_dhcp4(self):
+        """Config dhcp4 on fallback_nic when there are no ipv4 addresses."""
+        macs_to_nics = {self.mac1: "eth9"}
+        network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
+        nic1_metadata = network_metadata_ipv6["interfaces"]["macs"][self.mac1]
+        nic1_metadata["public-ipv4s"] = ""
+
+        # When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config.
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": False,
+                }
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_ipv6, macs_to_nics, fallback_nic="eth9"
+            ),
+        )
+
+    def test_convert_ec2_metadata_network_config_handles_local_v4_and_v6(self):
+        """When ipv6s and local-ipv4s are non-empty, enable dhcp6 and dhcp4."""
+        macs_to_nics = {self.mac1: "eth9"}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        nic1_metadata = network_metadata_both["interfaces"]["macs"][self.mac1]
+        nic1_metadata["ipv6s"] = "2620:0:1009:fd00:e442:c88d:c04d:dc85/64"
+        nic1_metadata.pop("public-ipv4s")
+        nic1_metadata["local-ipv4s"] = "10.0.0.42"  # Local ipv4 only on vpc
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": True,
+                }
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics
+            ),
+        )
+
+    def test_convert_ec2_metadata_network_config_handles_multiple_nics(self):
+        """DHCP route-metric increases on secondary NICs for IPv4 and IPv6."""
+        mac2 = "06:17:04:d7:26:08"
+        macs_to_nics = {self.mac1: "eth9", mac2: "eth10"}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        # Add 2nd nic info
+        network_metadata_both["interfaces"]["macs"][mac2] = NIC2_MD
+        nic1_metadata = network_metadata_both["interfaces"]["macs"][self.mac1]
+        nic1_metadata["ipv6s"] = "2620:0:1009:fd00:e442:c88d:c04d:dc85/64"
+        nic1_metadata.pop("public-ipv4s")  # No public-ipv4 IPs in cfg
+        nic1_metadata["local-ipv4s"] = "10.0.0.42"  # Local ipv4 only on vpc
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp4-overrides": {"route-metric": 100},
+                    "dhcp6": True,
+                    "dhcp6-overrides": {"route-metric": 100},
+                },
+                "eth10": {
+                    "match": {"macaddress": mac2},
+                    "set-name": "eth10",
+                    "dhcp4": True,
+                    "dhcp4-overrides": {"route-metric": 200},
+                    "dhcp6": False,
+                },
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics
+            ),
+        )
+
+    def test_convert_ec2_metadata_network_config_handles_dhcp4_and_dhcp6(self):
+        """Config both dhcp4 and dhcp6 when both vpc-ipv6 and ipv4 exists."""
+        macs_to_nics = {self.mac1: "eth9"}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        nic1_metadata = network_metadata_both["interfaces"]["macs"][self.mac1]
+        nic1_metadata["ipv6s"] = "2620:0:1009:fd00:e442:c88d:c04d:dc85/64"
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": True,
+                }
+            },
+        }
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics
+            ),
+        )
+
+    def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
+        """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
+        expected = {
+            "version": 2,
+            "ethernets": {
+                "eth9": {
+                    "match": {"macaddress": self.mac1},
+                    "set-name": "eth9",
+                    "dhcp4": True,
+                    "dhcp6": False,
+                }
+            },
+        }
+        patch_path = M_PATH_NET + "get_interfaces_by_mac"
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {self.mac1: "eth9"}
+            self.assertEqual(
+                expected,
+                ec2.convert_ec2_metadata_network_config(self.network_metadata),
+            )
+
+
+class TesIdentifyPlatform(test_helpers.CiTestCase):
+    def collmock(self, **kwargs):
+        """return non-special _collect_platform_data updated with changes."""
+        unspecial = {
+            "asset_tag": "3857-0037-2746-7462-1818-3997-77",
+            "serial": "H23-C4J3JV-R6",
+            "uuid": "81c7e555-6471-4833-9551-1ab366c4cfd2",
+            "uuid_source": "dmi",
+            "vendor": "tothecloud",
+        }
+        unspecial.update(**kwargs)
+        return unspecial
+
+    @mock.patch("cloudinit.sources.DataSourceEc2._collect_platform_data")
+    def test_identify_zstack(self, m_collect):
+        """zstack should be identified if chassis-asset-tag
+        ends in .zstack.io
+        """
+        m_collect.return_value = self.collmock(asset_tag="123456.zstack.io")
+        self.assertEqual(ec2.CloudNames.ZSTACK, ec2.identify_platform())
+
+    @mock.patch("cloudinit.sources.DataSourceEc2._collect_platform_data")
+    def test_identify_zstack_full_domain_only(self, m_collect):
+        """zstack asset-tag matching should match only on
+        full domain boundary.
+        """
+        m_collect.return_value = self.collmock(asset_tag="123456.buzzstack.io")
+        self.assertEqual(ec2.CloudNames.UNKNOWN, ec2.identify_platform())
+
+    @mock.patch("cloudinit.sources.DataSourceEc2._collect_platform_data")
+    def test_identify_e24cloud(self, m_collect):
+        """e24cloud identified if vendor is e24cloud"""
+        m_collect.return_value = self.collmock(vendor="e24cloud")
+        self.assertEqual(ec2.CloudNames.E24CLOUD, ec2.identify_platform())
+
+    @mock.patch("cloudinit.sources.DataSourceEc2._collect_platform_data")
+    def test_identify_e24cloud_negative(self, m_collect):
+        """e24cloud identified if vendor is e24cloud"""
+        m_collect.return_value = self.collmock(vendor="e24cloudyday")
+        self.assertEqual(ec2.CloudNames.UNKNOWN, ec2.identify_platform())
+
+
+# vi: ts=4 expandtab