ec2: render network on all NICs and add secondary IPs as static (#114)

Add support for rendering secondary static IPv4/IPv6 addresses on
any NIC attached to the machine. In order to see secondary IP
addresses in Ec2 IMDS network config, cloud-init now reads metadata
version 2018-09-24. Metadata services which do not support the Ec2
API version will not get secondary IP addresses configured.

In order to discover secondary IP address config, cloud-init now
relies on metadata API Parse local-ipv4s, ipv6s,
subnet-ipv4-cidr-block and subnet-ipv6-cidr-block metadata keys to
determine additional IPs and appropriate subnet prefix to set for a
nic.

Also add the datasource config option apply_full_imds_netork_config
which defaults to true to allow cloud-init to automatically configure
secondary IP addresses. Setting this option to false will tell
cloud-init to avoid setting up secondary IP addresses.

Also in this branch:
 - Shift Ec2 datasource to emit network config v2 instead of v1.

LP: #1866930

3 files changed, 379 insertions, 85 deletions

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 8f0d73bb..4203c3a6 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -62,7 +62,7 @@ class DataSourceEc2(sources.DataSource):
 
     # Priority ordered list of additional metadata versions which will be tried
     # for extended metadata content. IPv6 support comes in 2016-09-02
-    extended_metadata_versions = ['2016-09-02']
+    extended_metadata_versions = ['2018-09-24', '2016-09-02']
 
     # Setup read_url parameters per get_url_params.
     url_max_wait = 120
@@ -405,13 +405,16 @@ class DataSourceEc2(sources.DataSource):
                 logfunc=LOG.debug, msg='Re-crawl of metadata service',
                 func=self.get_data)
 
-        # Limit network configuration to only the primary/fallback nic
         iface = self.fallback_interface
-        macs_to_nics = {net.get_interface_mac(iface): iface}
         net_md = self.metadata.get('network')
         if isinstance(net_md, dict):
+            # SRU_BLOCKER: xenial, bionic and eoan should default
+            # apply_full_imds_network_config to False to retain original
+            # behavior on those releases.
             result = convert_ec2_metadata_network_config(
-                net_md, macs_to_nics=macs_to_nics, fallback_nic=iface)
+                net_md, fallback_nic=iface,
+                full_network_config=util.get_cfg_option_bool(
+                    self.ds_cfg, 'apply_full_imds_network_config', True))
 
             # RELEASE_BLOCKER: xenial should drop the below if statement,
             # because the issue being addressed doesn't exist pre-netplan.
@@ -719,9 +722,10 @@ def _collect_platform_data():
     return data
 
 
-def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
-                                        fallback_nic=None):
-    """Convert ec2 metadata to network config version 1 data dict.
+def convert_ec2_metadata_network_config(
+        network_md, macs_to_nics=None, fallback_nic=None,
+        full_network_config=True):
+    """Convert ec2 metadata to network config version 2 data dict.
 
     @param: network_md: 'network' portion of EC2 metadata.
        generally formed as {"interfaces": {"macs": {}} where
@@ -731,28 +735,104 @@ def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
        not provided, get_interfaces_by_mac is called to get it from the OS.
     @param: fallback_nic: Optionally provide the primary nic interface name.
        This nic will be guaranteed to minimally have a dhcp4 configuration.
+    @param: full_network_config: Boolean set True to configure all networking
+       presented by IMDS. This includes rendering secondary IPv4 and IPv6
+       addresses on all NICs and rendering network config on secondary NICs.
+       If False, only the primary nic will be configured and only with dhcp
+       (IPv4/IPv6).
 
-    @return A dict of network config version 1 based on the metadata and macs.
+    @return A dict of network config version 2 based on the metadata and macs.
     """
-    netcfg = {'version': 1, 'config': []}
+    netcfg = {'version': 2, 'ethernets': {}}
     if not macs_to_nics:
         macs_to_nics = net.get_interfaces_by_mac()
     macs_metadata = network_md['interfaces']['macs']
-    for mac, nic_name in macs_to_nics.items():
+
+    if not full_network_config:
+        for mac, nic_name in macs_to_nics.items():
+            if nic_name == fallback_nic:
+                break
+        dev_config = {'dhcp4': True,
+                      'dhcp6': False,
+                      'match': {'macaddress': mac.lower()},
+                      'set-name': nic_name}
+        nic_metadata = macs_metadata.get(mac)
+        if nic_metadata.get('ipv6s'):  # Any IPv6 addresses configured
+            dev_config['dhcp6'] = True
+        netcfg['ethernets'][nic_name] = dev_config
+        return netcfg
+    # Apply network config for all nics and any secondary IPv4/v6 addresses
+    nic_idx = 1
+    for mac, nic_name in sorted(macs_to_nics.items()):
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
             continue  # Not a physical nic represented in metadata
-        nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
-        nic_cfg['mac_address'] = mac
-        if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or
-                nic_metadata.get('local-ipv4s')):
-            nic_cfg['subnets'].append({'type': 'dhcp4'})
-        if nic_metadata.get('ipv6s'):
-            nic_cfg['subnets'].append({'type': 'dhcp6'})
-        netcfg['config'].append(nic_cfg)
+        dhcp_override = {'route-metric': nic_idx * 100}
+        nic_idx += 1
+        dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override,
+                      'dhcp6': False,
+                      'match': {'macaddress': mac.lower()},
+                      'set-name': nic_name}
+        if nic_metadata.get('ipv6s'):  # Any IPv6 addresses configured
+            dev_config['dhcp6'] = True
+            dev_config['dhcp6-overrides'] = dhcp_override
+        dev_config['addresses'] = get_secondary_addresses(nic_metadata, mac)
+        if not dev_config['addresses']:
+            dev_config.pop('addresses')  # Since we found none configured
+        netcfg['ethernets'][nic_name] = dev_config
+    # Remove route-metric dhcp overrides if only one nic configured
+    if len(netcfg['ethernets']) == 1:
+        for nic_name in netcfg['ethernets'].keys():
+            netcfg['ethernets'][nic_name].pop('dhcp4-overrides')
+            netcfg['ethernets'][nic_name].pop('dhcp6-overrides', None)
     return netcfg
 
 
+def get_secondary_addresses(nic_metadata, mac):
+    """Parse interface-specific nic metadata and return any secondary IPs
+
+    :return: List of secondary IPv4 or IPv6 addresses to configure on the
+    interface
+    """
+    ipv4s = nic_metadata.get('local-ipv4s')
+    ipv6s = nic_metadata.get('ipv6s')
+    addresses = []
+    # In version < 2018-09-24 local_ipv4s or ipv6s is a str with one IP
+    if bool(isinstance(ipv4s, list) and len(ipv4s) > 1):
+        addresses.extend(
+            _get_secondary_addresses(
+                nic_metadata, 'subnet-ipv4-cidr-block', mac, ipv4s, '24'))
+    if bool(isinstance(ipv6s, list) and len(ipv6s) > 1):
+        addresses.extend(
+            _get_secondary_addresses(
+                nic_metadata, 'subnet-ipv6-cidr-block', mac, ipv6s, '128'))
+    return sorted(addresses)
+
+
+def _get_secondary_addresses(nic_metadata, cidr_key, mac, ips, default_prefix):
+    """Return list of IP addresses as CIDRs for secondary IPs
+
+    The CIDR prefix will be default_prefix if cidr_key is absent or not
+    parseable in nic_metadata.
+    """
+    addresses = []
+    cidr = nic_metadata.get(cidr_key)
+    prefix = default_prefix
+    if not cidr or len(cidr.split('/')) != 2:
+        ip_type = 'ipv4' if 'ipv4' in cidr_key else 'ipv6'
+        LOG.warning(
+            'Could not parse %s %s for mac %s. %s network'
+            ' config prefix defaults to /%s',
+            cidr_key, cidr, mac, ip_type, prefix)
+    else:
+        prefix = cidr.split('/')[1]
+    # We know we have > 1 ips for in metadata for this IP type
+    for ip in ips[1:]:
+        addresses.append(
+            '{ip}/{prefix}'.format(ip=ip, prefix=prefix))
+    return addresses
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)),  # Run at init-local
diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst
index a90f3779..1c3a880f 100644
--- a/doc/rtd/topics/datasources/ec2.rst
+++ b/doc/rtd/topics/datasources/ec2.rst
@@ -42,6 +42,7 @@ Note that there are multiple versions of this data provided, cloud-init
 by default uses **2009-04-04** but newer versions can be supported with
 relative ease (newer versions have more data exposed, while maintaining
 backward compatibility with the previous versions).
+Version **2016-09-02** is required for secondary IP address support.
 
 To see which versions are supported from your cloud provider use the following
 URL:
@@ -80,6 +81,15 @@ The settings that may be configured are:
  * **timeout**: the timeout value provided to urlopen for each individual http
    request.  This is used both when selecting a metadata_url and when crawling
    the metadata service. (default: 50)
+ * **apply_full_imds_network_config**: Boolean (default: True) to allow
+   cloud-init to configure any secondary NICs and secondary IPs described by
+   the metadata service. All network interfaces are configured with DHCP (v4)
+   to obtain an primary IPv4 address and route. Interfaces which have a
+   non-empty 'ipv6s' list will also enable DHCPv6 to obtain a primary IPv6
+   address and route. The DHCP response (v4 and v6) return an IP that matches
+   the first element of local-ipv4s and ipv6s lists respectively. All
+   additional values (secondary addresses) in the static ip lists will be
+   added to interface.
 
 An example configuration with the default values is provided below:
 
@@ -90,6 +100,7 @@ An example configuration with the default values is provided below:
     metadata_urls: ["http://169.254.169.254:80", "http://instance-data:8773"]
     max_wait: 120
     timeout: 50
+    apply_full_imds_network_config: true
 
 Notes
 -----
@@ -102,4 +113,12 @@ Notes
    The check for the instance type is performed by is_classic_instance()
    method.
 
+ * For EC2 instances with multiple network interfaces (NICs) attached, dhcp4
+   will be enabled to obtain the primary private IPv4 address of those NICs.
+   Wherever dhcp4 or dhcp6 is enabled for a NIC, a dhcp route-metric will be
+   added with the value of ``<device-number + 1> * 100`` to ensure dhcp
+   routes on the primary NIC are preferred to any secondary NICs.
+   For example: the primary NIC will have a DHCP route-metric of 100,
+   the next NIC will be 200.
+
 .. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 78e82c7e..9126c676 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -113,6 +113,122 @@ DEFAULT_METADATA = {
     "services": {"domain": "amazonaws.com", "partition": "aws"},
 }
 
+# collected from api version 2018-09-24/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
+
+NIC1_MD_IPV4_IPV6_MULTI_IP = {
+    "device-number": "0",
+    "interface-id": "eni-0d6335689899ce9cc",
+    "ipv4-associations": {
+        "18.218.219.181": "172.31.44.13"
+    },
+    "ipv6s": [
+        "2600:1f16:292:100:c187:593c:4349:136",
+        "2600:1f16:292:100:f153:12a3:c37c:11f9",
+        "2600:1f16:292:100:f152:2222:3333:4444"
+    ],
+    "local-hostname": ("ip-172-31-44-13.us-east-2."
+                       "compute.internal"),
+    "local-ipv4s": [
+        "172.31.44.13",
+        "172.31.45.70"
+    ],
+    "mac": "0a:07:84:3d:6e:38",
+    "owner-id": "329910648901",
+    "public-hostname": ("ec2-18-218-219-181.us-east-2."
+                        "compute.amazonaws.com"),
+    "public-ipv4s": "18.218.219.181",
+    "security-group-ids": "sg-0c387755222ba8d2e",
+    "security-groups": "launch-wizard-4",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "subnet_ipv6_cidr_blocks": "2600:1f16:292:100::/64",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+    "vpc_ipv6_cidr_blocks": "2600:1f16:292:100::/56"
+}
+
+NIC2_MD = {
+    "device_number": "1",
+    "interface_id": "eni-043cdce36ded5e79f",
+    "local_hostname": "ip-172-31-47-221.us-east-2.compute.internal",
+    "local_ipv4s": "172.31.47.221",
+    "mac": "0a:75:69:92:e2:16",
+    "owner_id": "329910648901",
+    "security_group_ids": "sg-0d68fef37d8cc9b77",
+    "security_groups": "launch-wizard-17",
+    "subnet_id": "subnet-9d7ba0d1",
+    "subnet_ipv4_cidr_block": "172.31.32.0/20",
+    "vpc_id": "vpc-a07f62c8",
+    "vpc_ipv4_cidr_block": "172.31.0.0/16",
+    "vpc_ipv4_cidr_blocks": "172.31.0.0/16"
+}
+
+SECONDARY_IP_METADATA_2018_09_24 = {
+    "ami-id": "ami-0986c2ac728528ac2",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {
+        "ami": "/dev/sda1",
+        "root": "/dev/sda1"
+    },
+    "events": {
+        "maintenance": {
+            "history": "[]",
+            "scheduled": "[]"
+        }
+    },
+    "hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "identity-credentials": {
+        "ec2": {
+            "info": {
+                "AccountId": "329910648901",
+                "Code": "Success",
+                "LastUpdated": "2019-07-06T14:22:56Z"
+            }
+        }
+    },
+    "instance-action": "none",
+    "instance-id": "i-069e01e8cc43732f8",
+    "instance-type": "t2.micro",
+    "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "local-ipv4": "172.31.44.13",
+    "mac": "0a:07:84:3d:6e:38",
+    "metrics": {
+        "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+    },
+    "network": {
+        "interfaces": {
+            "macs": {
+                "0a:07:84:3d:6e:38": NIC1_MD_IPV4_IPV6_MULTI_IP,
+            }
+        }
+    },
+    "placement": {
+        "availability-zone": "us-east-2c"
+    },
+    "profile": "default-hvm",
+    "public-hostname": (
+        "ec2-18-218-219-181.us-east-2.compute.amazonaws.com"),
+    "public-ipv4": "18.218.219.181",
+    "public-keys": {
+        "yourkeyname,e": [
+            "ssh-rsa AAAAW...DZ yourkeyname"
+        ]
+    },
+    "reservation-id": "r-09b4917135cdd33be",
+    "security-groups": "launch-wizard-4",
+    "services": {
+        "domain": "amazonaws.com",
+        "partition": "aws"
+    }
+}
+
+M_PATH_NET = 'cloudinit.sources.DataSourceEc2.net.'
+
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
@@ -267,30 +383,23 @@ class TestEc2(test_helpers.HttprettyTestCase):
                         register_mock_metaserver(instance_id_url, None)
         return ds
 
-    def test_network_config_property_returns_version_1_network_data(self):
-        """network_config property returns network version 1 for metadata.
-
-        Only one device is configured even when multiple exist in metadata.
-        """
+    def test_network_config_property_returns_version_2_network_data(self):
+        """network_config property returns network version 2 for metadata"""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md={'md': DEFAULT_METADATA})
-        find_fallback_path = (
-            'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
         with mock.patch(find_fallback_path) as m_find_fallback:
             m_find_fallback.return_value = 'eth9'
             ds.get_data()
 
         mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
-             'type': 'physical'}]}
-        patch_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
-        get_interface_mac_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': '06:17:04:d7:26:09'}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
         with mock.patch(patch_path) as m_get_interfaces_by_mac:
             with mock.patch(find_fallback_path) as m_find_fallback:
                 with mock.patch(get_interface_mac_path) as m_get_mac:
@@ -299,30 +408,59 @@ class TestEc2(test_helpers.HttprettyTestCase):
                     m_get_mac.return_value = mac1
                     self.assertEqual(expected, ds.network_config)
 
-    def test_network_config_property_set_dhcp4_on_private_ipv4(self):
-        """network_config property configures dhcp4 on private ipv4 nics.
+    def test_network_config_property_set_dhcp4(self):
+        """network_config property configures dhcp4 on nics with local-ipv4s.
 
-        Only one device is configured even when multiple exist in metadata.
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
         """
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md={'md': DEFAULT_METADATA})
-        find_fallback_path = (
-            'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
         with mock.patch(find_fallback_path) as m_find_fallback:
             m_find_fallback.return_value = 'eth9'
             ds.get_data()
 
         mac1 = '06:17:04:d7:26:0A'  # IPv4 only in DEFAULT_METADATA
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:0A', 'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}],
-             'type': 'physical'}]}
-        patch_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
-        get_interface_mac_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1.lower()}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+                    m_find_fallback.return_value = 'eth9'
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_secondary_private_ips(self):
+        """network_config property configures any secondary ipv4 addresses.
+
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': SECONDARY_IP_METADATA_2018_09_24})
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = 'eth9'
+            ds.get_data()
+
+        mac1 = '0a:07:84:3d:6e:38'  # 1 secondary IPv4 and 2 secondary IPv6
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1}, 'set-name': 'eth9',
+            'addresses': ['172.31.45.70/20',
+                          '2600:1f16:292:100:f152:2222:3333:4444/128',
+                          '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            'dhcp4': True, 'dhcp6': True}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
         with mock.patch(patch_path) as m_get_interfaces_by_mac:
             with mock.patch(find_fallback_path) as m_find_fallback:
                 with mock.patch(get_interface_mac_path) as m_get_mac:
@@ -358,21 +496,18 @@ class TestEc2(test_helpers.HttprettyTestCase):
         register_mock_metaserver(
             'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA)
         mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
-        get_interface_mac_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        get_interface_mac_path = M_PATH_NET + 'get_interfaces_by_mac'
         ds.fallback_nic = 'eth9'
-        with mock.patch(get_interface_mac_path) as m_get_interface_mac:
-            m_get_interface_mac.return_value = mac1
+        with mock.patch(get_interface_mac_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
             nc = ds.network_config  # Will re-crawl network metadata
             self.assertIsNotNone(nc)
         self.assertIn(
             'Refreshing stale metadata from prior to upgrade',
             self.logs.getvalue())
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:09',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
-             'type': 'physical'}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(expected, ds.network_config)
 
     def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
@@ -491,7 +626,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         logs_with_redacted = [log for log in all_logs if REDACT_TOK in log]
         logs_with_token = [log for log in all_logs if 'API-TOKEN' in log]
         self.assertEqual(1, len(logs_with_redacted_ttl))
-        self.assertEqual(79, len(logs_with_redacted))
+        self.assertEqual(81, len(logs_with_redacted))
         self.assertEqual(0, len(logs_with_token))
 
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
@@ -612,6 +747,44 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
+class TestGetSecondaryAddresses(test_helpers.CiTestCase):
+
+    mac = '06:17:04:d7:26:ff'
+    with_logs = True
+
+    def test_md_with_no_secondary_addresses(self):
+        """Empty list is returned when nic metadata contains no secondary ip"""
+        self.assertEqual([], ec2.get_secondary_addresses(NIC2_MD, self.mac))
+
+    def test_md_with_secondary_v4_and_v6_addresses(self):
+        """All secondary addresses are returned from nic metadata"""
+        self.assertEqual(
+            ['172.31.45.70/20', '2600:1f16:292:100:f152:2222:3333:4444/128',
+             '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            ec2.get_secondary_addresses(NIC1_MD_IPV4_IPV6_MULTI_IP, self.mac))
+
+    def test_invalid_ipv4_ipv6_cidr_metadata_logged_with_defaults(self):
+        """Any invalid subnet-ipv(4|6)-cidr-block values use defaults"""
+        invalid_cidr_md = copy.deepcopy(NIC1_MD_IPV4_IPV6_MULTI_IP)
+        invalid_cidr_md['subnet-ipv4-cidr-block'] = "something-unexpected"
+        invalid_cidr_md['subnet-ipv6-cidr-block'] = "not/sure/what/this/is"
+        self.assertEqual(
+            ['172.31.45.70/24', '2600:1f16:292:100:f152:2222:3333:4444/128',
+             '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            ec2.get_secondary_addresses(invalid_cidr_md, self.mac))
+        expected_logs = [
+           "WARNING: Could not parse subnet-ipv4-cidr-block"
+           " something-unexpected for mac 06:17:04:d7:26:ff."
+           " ipv4 network config prefix defaults to /24",
+           "WARNING: Could not parse subnet-ipv6-cidr-block"
+           " not/sure/what/this/is for mac 06:17:04:d7:26:ff."
+           " ipv6 network config prefix defaults to /128"
+        ]
+        logs = self.logs.getvalue()
+        for log in expected_logs:
+            self.assertIn(log, logs)
+
+
 class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def setUp(self):
@@ -619,16 +792,16 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         self.mac1 = '06:17:04:d7:26:09'
         self.network_metadata = {
             'interfaces': {'macs': {
-                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}
+                self.mac1: {'mac': self.mac1, 'public-ipv4s': '172.31.2.16'}}}}
 
     def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
         """Any mac absent from metadata is skipped by network config."""
         macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
 
         # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -642,15 +815,15 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
             network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
                 network_metadata_ipv6, macs_to_nics))
 
-    def test_convert_ec2_metadata_network_config_handles_local_dhcp4(self):
+    def test_convert_ec2_metadata_network_config_local_only_dhcp4(self):
         """Config dhcp4 when there are no public addresses in public-ipv4s."""
         macs_to_nics = {self.mac1: 'eth9'}
         network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
@@ -658,9 +831,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
             network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['local-ipv4s'] = '172.3.3.15'
         nic1_metadata.pop('public-ipv4s')
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -675,16 +848,16 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata['public-ipv4s'] = ''
 
         # When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config.
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
                 network_metadata_ipv6, macs_to_nics, fallback_nic='eth9'))
 
     def test_convert_ec2_metadata_network_config_handles_local_v4_and_v6(self):
-        """When dhcp6 is public and dhcp4 is set to local enable both."""
+        """When ipv6s and local-ipv4s are non-empty, enable dhcp6 and dhcp4."""
         macs_to_nics = {self.mac1: 'eth9'}
         network_metadata_both = copy.deepcopy(self.network_metadata)
         nic1_metadata = (
@@ -692,10 +865,35 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
         nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_multiple_nics(self):
+        """DHCP route-metric increases on secondary NICs for IPv4 and IPv6."""
+        mac2 = '06:17:04:d7:26:0a'
+        macs_to_nics = {self.mac1: 'eth9', mac2: 'eth10'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        # Add 2nd nic info
+        network_metadata_both['interfaces']['macs'][mac2] = NIC2_MD
+        nic1_metadata = (
+            network_metadata_both['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')  # No public-ipv4 IPs in cfg
+        nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
+        expected = {'version': 2, 'ethernets': {
+            'eth9': {
+                'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+                'dhcp4': True, 'dhcp4-overrides': {'route-metric': 100},
+                'dhcp6': True, 'dhcp6-overrides': {'route-metric': 100}},
+            'eth10': {
+                'match': {'macaddress': mac2}, 'set-name': 'eth10',
+                'dhcp4': True, 'dhcp4-overrides': {'route-metric': 200},
+                'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -708,10 +906,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata = (
             network_metadata_both['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -719,12 +916,10 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
         """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}]}]}
-        patch_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1},
+            'set-name': 'eth9', 'dhcp4': True, 'dhcp6': False}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
         with mock.patch(patch_path) as m_get_interfaces_by_mac:
             m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'}
             self.assertEqual(