From 1f860e5ac7ebb5b809c72d8703a0b7cb3e84ccd0 Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Thu, 5 Mar 2020 17:38:28 -0700
Subject: ec2: Do not fallback to IMDSv1 on EC2 (#216)

The EC2 Data Source needs to handle 3 states of the Instance
Metadata Service configured for a given instance:

1. HttpTokens : optional & HttpEndpoint : enabled
   Either IMDSv2 or IMDSv1 can be used.
2. HttpTokens : required & HttpEndpoint : enabled
   Calls to IMDS without a valid token (IMDSv1 or IMDSv2 with expired token)
   will return a 401 error.
3. HttpEndpoint : disabled
   The IMDS http endpoint will return a 403 error.

Previous work to support IMDSv2 in cloud-init handled case 1 and case 2.

This commit handles case 3 by bypassing the retry block when IMDS returns HTTP
status code >= 400 on official AWS cloud platform.

It shaves 2 minutes when rebooting an instance that has its IMDS http token endpoint
disabled but creates some inconsistencies. An instance that doesn't set
"manual_cache_clean" to "True" will have its /var/lib/cloud/instance symlink
removed altogether after it has failed to find a datasource.
---
 cloudinit/sources/DataSourceEc2.py | 75 +++++++++++++++++++++++++++++---------
 1 file changed, 58 insertions(+), 17 deletions(-)

(limited to 'cloudinit/sources/DataSourceEc2.py')

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 0f2bfef4..8f0d73bb 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -29,7 +29,6 @@ STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
 STRICT_ID_DEFAULT = "warn"
 
 API_TOKEN_ROUTE = 'latest/api/token'
-API_TOKEN_DISABLED = '_ec2_disable_api_token'
 AWS_TOKEN_TTL_SECONDS = '21600'
 AWS_TOKEN_PUT_HEADER = 'X-aws-ec2-metadata-token'
 AWS_TOKEN_REQ_HEADER = AWS_TOKEN_PUT_HEADER + '-ttl-seconds'
@@ -193,6 +192,12 @@ class DataSourceEc2(sources.DataSource):
             return self.metadata['instance-id']
 
     def _maybe_fetch_api_token(self, mdurls, timeout=None, max_wait=None):
+        """ Get an API token for EC2 Instance Metadata Service.
+
+        On EC2. IMDS will always answer an API token, unless
+        the instance owner has disabled the IMDS HTTP endpoint or
+        the network topology conflicts with the configured hop-limit.
+        """
         if self.cloud_name != CloudNames.AWS:
             return
 
@@ -205,18 +210,33 @@ class DataSourceEc2(sources.DataSource):
             urls.append(cur)
             url2base[cur] = url
 
-        # use the self._status_cb to check for Read errors, which means
-        # we can't reach the API token URL, so we should disable IMDSv2
+        # use the self._imds_exception_cb to check for Read errors
         LOG.debug('Fetching Ec2 IMDSv2 API Token')
-        url, response = uhelp.wait_for_url(
-            urls=urls, max_wait=1, timeout=1, status_cb=self._status_cb,
-            headers_cb=self._get_headers, request_method=request_method,
-            headers_redact=AWS_TOKEN_REDACT)
+
+        response = None
+        url = None
+        url_params = self.get_url_params()
+        try:
+            url, response = uhelp.wait_for_url(
+                urls=urls, max_wait=url_params.max_wait_seconds,
+                timeout=url_params.timeout_seconds, status_cb=LOG.warning,
+                headers_cb=self._get_headers,
+                exception_cb=self._imds_exception_cb,
+                request_method=request_method,
+                headers_redact=AWS_TOKEN_REDACT)
+        except uhelp.UrlError:
+            # We use the raised exception to interupt the retry loop.
+            # Nothing else to do here.
+            pass
 
         if url and response:
             self._api_token = response
             return url2base[url]
 
+        # If we get here, then wait_for_url timed out, waiting for IMDS
+        # or the IMDS HTTP endpoint is disabled
+        return None
+
     def wait_for_metadata_service(self):
         mcfg = self.ds_cfg
 
@@ -240,9 +260,11 @@ class DataSourceEc2(sources.DataSource):
 
         # try the api token path first
         metadata_address = self._maybe_fetch_api_token(mdurls)
-        if not metadata_address:
-            if self._api_token == API_TOKEN_DISABLED:
-                LOG.warning('Retrying with IMDSv1')
+        # When running on EC2, we always access IMDS with an API token.
+        # If we could not get an API token, then we assume the IMDS
+        # endpoint was disabled and we move on without a data source.
+        # Fallback to IMDSv1 if not running on EC2
+        if not metadata_address and self.cloud_name != CloudNames.AWS:
             # if we can't get a token, use instance-id path
             urls = []
             url2base = {}
@@ -267,6 +289,8 @@ class DataSourceEc2(sources.DataSource):
         if metadata_address:
             self.metadata_address = metadata_address
             LOG.debug("Using metadata source: '%s'", self.metadata_address)
+        elif self.cloud_name == CloudNames.AWS:
+            LOG.warning("IMDS's HTTP endpoint is probably disabled")
         else:
             LOG.critical("Giving up on md from %s after %s seconds",
                          urls, int(time.time() - start_time))
@@ -496,11 +520,29 @@ class DataSourceEc2(sources.DataSource):
             self._api_token = None
         return True  # always retry
 
-    def _status_cb(self, msg, exc=None):
-        LOG.warning(msg)
-        if 'Read timed out' in msg:
-            LOG.warning('Cannot use Ec2 IMDSv2 API tokens, using IMDSv1')
-            self._api_token = API_TOKEN_DISABLED
+    def _imds_exception_cb(self, msg, exception=None):
+        """Fail quickly on proper AWS if IMDSv2 rejects API token request
+
+        Guidance from Amazon is that if IMDSv2 had disabled token requests
+        by returning a 403, or cloud-init malformed requests resulting in
+        other 40X errors, we want the datasource detection to fail quickly
+        without retries as those symptoms will likely not be resolved by
+        retries.
+
+        Exceptions such as requests.ConnectionError due to IMDS being
+        temporarily unroutable or unavailable will still retry due to the
+        callsite wait_for_url.
+        """
+        if isinstance(exception, uhelp.UrlError):
+            # requests.ConnectionError will have exception.code == None
+            if exception.code and exception.code >= 400:
+                if exception.code == 403:
+                    LOG.warning('Ec2 IMDS endpoint returned a 403 error. '
+                                'HTTP endpoint is disabled. Aborting.')
+                else:
+                    LOG.warning('Fatal error while requesting '
+                                'Ec2 IMDSv2 API tokens')
+                raise exception
 
     def _get_headers(self, url=''):
         """Return a dict of headers for accessing a url.
@@ -508,8 +550,7 @@ class DataSourceEc2(sources.DataSource):
         If _api_token is unset on AWS, attempt to refresh the token via a PUT
         and then return the updated token header.
         """
-        if self.cloud_name != CloudNames.AWS or (self._api_token ==
-                                                 API_TOKEN_DISABLED):
+        if self.cloud_name != CloudNames.AWS:
             return {}
         # Request a 6 hour token if URL is API_TOKEN_ROUTE
         request_token_header = {AWS_TOKEN_REQ_HEADER: AWS_TOKEN_TTL_SECONDS}
-- 
cgit v1.2.3


From 6600c642af3817fe5e0170cb7b4eeac4be3c60eb Mon Sep 17 00:00:00 2001
From: Chad Smith <chad.smith@canonical.com>
Date: Wed, 18 Mar 2020 13:33:37 -0600
Subject: ec2: render network on all NICs and add secondary IPs as static
 (#114)

Add support for rendering secondary static IPv4/IPv6 addresses on
any NIC attached to the machine. In order to see secondary IP
addresses in Ec2 IMDS network config, cloud-init now reads metadata
version 2018-09-24. Metadata services which do not support the Ec2
API version will not get secondary IP addresses configured.

In order to discover secondary IP address config, cloud-init now
relies on metadata API Parse local-ipv4s, ipv6s,
subnet-ipv4-cidr-block and subnet-ipv6-cidr-block metadata keys to
determine additional IPs and appropriate subnet prefix to set for a
nic.

Also add the datasource config option apply_full_imds_netork_config
which defaults to true to allow cloud-init to automatically configure
secondary IP addresses. Setting this option to false will tell
cloud-init to avoid setting up secondary IP addresses.

Also in this branch:
 - Shift Ec2 datasource to emit network config v2 instead of v1.

LP: #1866930
---
 cloudinit/sources/DataSourceEc2.py          | 116 ++++++++--
 doc/rtd/topics/datasources/ec2.rst          |  19 ++
 tests/unittests/test_datasource/test_ec2.py | 329 ++++++++++++++++++++++------
 3 files changed, 379 insertions(+), 85 deletions(-)

(limited to 'cloudinit/sources/DataSourceEc2.py')

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 8f0d73bb..4203c3a6 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -62,7 +62,7 @@ class DataSourceEc2(sources.DataSource):
 
     # Priority ordered list of additional metadata versions which will be tried
     # for extended metadata content. IPv6 support comes in 2016-09-02
-    extended_metadata_versions = ['2016-09-02']
+    extended_metadata_versions = ['2018-09-24', '2016-09-02']
 
     # Setup read_url parameters per get_url_params.
     url_max_wait = 120
@@ -405,13 +405,16 @@ class DataSourceEc2(sources.DataSource):
                 logfunc=LOG.debug, msg='Re-crawl of metadata service',
                 func=self.get_data)
 
-        # Limit network configuration to only the primary/fallback nic
         iface = self.fallback_interface
-        macs_to_nics = {net.get_interface_mac(iface): iface}
         net_md = self.metadata.get('network')
         if isinstance(net_md, dict):
+            # SRU_BLOCKER: xenial, bionic and eoan should default
+            # apply_full_imds_network_config to False to retain original
+            # behavior on those releases.
             result = convert_ec2_metadata_network_config(
-                net_md, macs_to_nics=macs_to_nics, fallback_nic=iface)
+                net_md, fallback_nic=iface,
+                full_network_config=util.get_cfg_option_bool(
+                    self.ds_cfg, 'apply_full_imds_network_config', True))
 
             # RELEASE_BLOCKER: xenial should drop the below if statement,
             # because the issue being addressed doesn't exist pre-netplan.
@@ -719,9 +722,10 @@ def _collect_platform_data():
     return data
 
 
-def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
-                                        fallback_nic=None):
-    """Convert ec2 metadata to network config version 1 data dict.
+def convert_ec2_metadata_network_config(
+        network_md, macs_to_nics=None, fallback_nic=None,
+        full_network_config=True):
+    """Convert ec2 metadata to network config version 2 data dict.
 
     @param: network_md: 'network' portion of EC2 metadata.
        generally formed as {"interfaces": {"macs": {}} where
@@ -731,28 +735,104 @@ def convert_ec2_metadata_network_config(network_md, macs_to_nics=None,
        not provided, get_interfaces_by_mac is called to get it from the OS.
     @param: fallback_nic: Optionally provide the primary nic interface name.
        This nic will be guaranteed to minimally have a dhcp4 configuration.
+    @param: full_network_config: Boolean set True to configure all networking
+       presented by IMDS. This includes rendering secondary IPv4 and IPv6
+       addresses on all NICs and rendering network config on secondary NICs.
+       If False, only the primary nic will be configured and only with dhcp
+       (IPv4/IPv6).
 
-    @return A dict of network config version 1 based on the metadata and macs.
+    @return A dict of network config version 2 based on the metadata and macs.
     """
-    netcfg = {'version': 1, 'config': []}
+    netcfg = {'version': 2, 'ethernets': {}}
     if not macs_to_nics:
         macs_to_nics = net.get_interfaces_by_mac()
     macs_metadata = network_md['interfaces']['macs']
-    for mac, nic_name in macs_to_nics.items():
+
+    if not full_network_config:
+        for mac, nic_name in macs_to_nics.items():
+            if nic_name == fallback_nic:
+                break
+        dev_config = {'dhcp4': True,
+                      'dhcp6': False,
+                      'match': {'macaddress': mac.lower()},
+                      'set-name': nic_name}
+        nic_metadata = macs_metadata.get(mac)
+        if nic_metadata.get('ipv6s'):  # Any IPv6 addresses configured
+            dev_config['dhcp6'] = True
+        netcfg['ethernets'][nic_name] = dev_config
+        return netcfg
+    # Apply network config for all nics and any secondary IPv4/v6 addresses
+    nic_idx = 1
+    for mac, nic_name in sorted(macs_to_nics.items()):
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
             continue  # Not a physical nic represented in metadata
-        nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []}
-        nic_cfg['mac_address'] = mac
-        if (nic_name == fallback_nic or nic_metadata.get('public-ipv4s') or
-                nic_metadata.get('local-ipv4s')):
-            nic_cfg['subnets'].append({'type': 'dhcp4'})
-        if nic_metadata.get('ipv6s'):
-            nic_cfg['subnets'].append({'type': 'dhcp6'})
-        netcfg['config'].append(nic_cfg)
+        dhcp_override = {'route-metric': nic_idx * 100}
+        nic_idx += 1
+        dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override,
+                      'dhcp6': False,
+                      'match': {'macaddress': mac.lower()},
+                      'set-name': nic_name}
+        if nic_metadata.get('ipv6s'):  # Any IPv6 addresses configured
+            dev_config['dhcp6'] = True
+            dev_config['dhcp6-overrides'] = dhcp_override
+        dev_config['addresses'] = get_secondary_addresses(nic_metadata, mac)
+        if not dev_config['addresses']:
+            dev_config.pop('addresses')  # Since we found none configured
+        netcfg['ethernets'][nic_name] = dev_config
+    # Remove route-metric dhcp overrides if only one nic configured
+    if len(netcfg['ethernets']) == 1:
+        for nic_name in netcfg['ethernets'].keys():
+            netcfg['ethernets'][nic_name].pop('dhcp4-overrides')
+            netcfg['ethernets'][nic_name].pop('dhcp6-overrides', None)
     return netcfg
 
 
+def get_secondary_addresses(nic_metadata, mac):
+    """Parse interface-specific nic metadata and return any secondary IPs
+
+    :return: List of secondary IPv4 or IPv6 addresses to configure on the
+    interface
+    """
+    ipv4s = nic_metadata.get('local-ipv4s')
+    ipv6s = nic_metadata.get('ipv6s')
+    addresses = []
+    # In version < 2018-09-24 local_ipv4s or ipv6s is a str with one IP
+    if bool(isinstance(ipv4s, list) and len(ipv4s) > 1):
+        addresses.extend(
+            _get_secondary_addresses(
+                nic_metadata, 'subnet-ipv4-cidr-block', mac, ipv4s, '24'))
+    if bool(isinstance(ipv6s, list) and len(ipv6s) > 1):
+        addresses.extend(
+            _get_secondary_addresses(
+                nic_metadata, 'subnet-ipv6-cidr-block', mac, ipv6s, '128'))
+    return sorted(addresses)
+
+
+def _get_secondary_addresses(nic_metadata, cidr_key, mac, ips, default_prefix):
+    """Return list of IP addresses as CIDRs for secondary IPs
+
+    The CIDR prefix will be default_prefix if cidr_key is absent or not
+    parseable in nic_metadata.
+    """
+    addresses = []
+    cidr = nic_metadata.get(cidr_key)
+    prefix = default_prefix
+    if not cidr or len(cidr.split('/')) != 2:
+        ip_type = 'ipv4' if 'ipv4' in cidr_key else 'ipv6'
+        LOG.warning(
+            'Could not parse %s %s for mac %s. %s network'
+            ' config prefix defaults to /%s',
+            cidr_key, cidr, mac, ip_type, prefix)
+    else:
+        prefix = cidr.split('/')[1]
+    # We know we have > 1 ips for in metadata for this IP type
+    for ip in ips[1:]:
+        addresses.append(
+            '{ip}/{prefix}'.format(ip=ip, prefix=prefix))
+    return addresses
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)),  # Run at init-local
diff --git a/doc/rtd/topics/datasources/ec2.rst b/doc/rtd/topics/datasources/ec2.rst
index a90f3779..1c3a880f 100644
--- a/doc/rtd/topics/datasources/ec2.rst
+++ b/doc/rtd/topics/datasources/ec2.rst
@@ -42,6 +42,7 @@ Note that there are multiple versions of this data provided, cloud-init
 by default uses **2009-04-04** but newer versions can be supported with
 relative ease (newer versions have more data exposed, while maintaining
 backward compatibility with the previous versions).
+Version **2016-09-02** is required for secondary IP address support.
 
 To see which versions are supported from your cloud provider use the following
 URL:
@@ -80,6 +81,15 @@ The settings that may be configured are:
  * **timeout**: the timeout value provided to urlopen for each individual http
    request.  This is used both when selecting a metadata_url and when crawling
    the metadata service. (default: 50)
+ * **apply_full_imds_network_config**: Boolean (default: True) to allow
+   cloud-init to configure any secondary NICs and secondary IPs described by
+   the metadata service. All network interfaces are configured with DHCP (v4)
+   to obtain an primary IPv4 address and route. Interfaces which have a
+   non-empty 'ipv6s' list will also enable DHCPv6 to obtain a primary IPv6
+   address and route. The DHCP response (v4 and v6) return an IP that matches
+   the first element of local-ipv4s and ipv6s lists respectively. All
+   additional values (secondary addresses) in the static ip lists will be
+   added to interface.
 
 An example configuration with the default values is provided below:
 
@@ -90,6 +100,7 @@ An example configuration with the default values is provided below:
     metadata_urls: ["http://169.254.169.254:80", "http://instance-data:8773"]
     max_wait: 120
     timeout: 50
+    apply_full_imds_network_config: true
 
 Notes
 -----
@@ -102,4 +113,12 @@ Notes
    The check for the instance type is performed by is_classic_instance()
    method.
 
+ * For EC2 instances with multiple network interfaces (NICs) attached, dhcp4
+   will be enabled to obtain the primary private IPv4 address of those NICs.
+   Wherever dhcp4 or dhcp6 is enabled for a NIC, a dhcp route-metric will be
+   added with the value of ``<device-number + 1> * 100`` to ensure dhcp
+   routes on the primary NIC are preferred to any secondary NICs.
+   For example: the primary NIC will have a DHCP route-metric of 100,
+   the next NIC will be 200.
+
 .. vi: textwidth=78
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 78e82c7e..9126c676 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -113,6 +113,122 @@ DEFAULT_METADATA = {
     "services": {"domain": "amazonaws.com", "partition": "aws"},
 }
 
+# collected from api version 2018-09-24/ with
+# python3 -c 'import json
+# from cloudinit.ec2_utils import get_instance_metadata as gm
+# print(json.dumps(gm("2018-09-24"), indent=1, sort_keys=True))'
+
+NIC1_MD_IPV4_IPV6_MULTI_IP = {
+    "device-number": "0",
+    "interface-id": "eni-0d6335689899ce9cc",
+    "ipv4-associations": {
+        "18.218.219.181": "172.31.44.13"
+    },
+    "ipv6s": [
+        "2600:1f16:292:100:c187:593c:4349:136",
+        "2600:1f16:292:100:f153:12a3:c37c:11f9",
+        "2600:1f16:292:100:f152:2222:3333:4444"
+    ],
+    "local-hostname": ("ip-172-31-44-13.us-east-2."
+                       "compute.internal"),
+    "local-ipv4s": [
+        "172.31.44.13",
+        "172.31.45.70"
+    ],
+    "mac": "0a:07:84:3d:6e:38",
+    "owner-id": "329910648901",
+    "public-hostname": ("ec2-18-218-219-181.us-east-2."
+                        "compute.amazonaws.com"),
+    "public-ipv4s": "18.218.219.181",
+    "security-group-ids": "sg-0c387755222ba8d2e",
+    "security-groups": "launch-wizard-4",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "subnet_ipv6_cidr_blocks": "2600:1f16:292:100::/64",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+    "vpc_ipv6_cidr_blocks": "2600:1f16:292:100::/56"
+}
+
+NIC2_MD = {
+    "device_number": "1",
+    "interface_id": "eni-043cdce36ded5e79f",
+    "local_hostname": "ip-172-31-47-221.us-east-2.compute.internal",
+    "local_ipv4s": "172.31.47.221",
+    "mac": "0a:75:69:92:e2:16",
+    "owner_id": "329910648901",
+    "security_group_ids": "sg-0d68fef37d8cc9b77",
+    "security_groups": "launch-wizard-17",
+    "subnet_id": "subnet-9d7ba0d1",
+    "subnet_ipv4_cidr_block": "172.31.32.0/20",
+    "vpc_id": "vpc-a07f62c8",
+    "vpc_ipv4_cidr_block": "172.31.0.0/16",
+    "vpc_ipv4_cidr_blocks": "172.31.0.0/16"
+}
+
+SECONDARY_IP_METADATA_2018_09_24 = {
+    "ami-id": "ami-0986c2ac728528ac2",
+    "ami-launch-index": "0",
+    "ami-manifest-path": "(unknown)",
+    "block-device-mapping": {
+        "ami": "/dev/sda1",
+        "root": "/dev/sda1"
+    },
+    "events": {
+        "maintenance": {
+            "history": "[]",
+            "scheduled": "[]"
+        }
+    },
+    "hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "identity-credentials": {
+        "ec2": {
+            "info": {
+                "AccountId": "329910648901",
+                "Code": "Success",
+                "LastUpdated": "2019-07-06T14:22:56Z"
+            }
+        }
+    },
+    "instance-action": "none",
+    "instance-id": "i-069e01e8cc43732f8",
+    "instance-type": "t2.micro",
+    "local-hostname": "ip-172-31-44-13.us-east-2.compute.internal",
+    "local-ipv4": "172.31.44.13",
+    "mac": "0a:07:84:3d:6e:38",
+    "metrics": {
+        "vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+    },
+    "network": {
+        "interfaces": {
+            "macs": {
+                "0a:07:84:3d:6e:38": NIC1_MD_IPV4_IPV6_MULTI_IP,
+            }
+        }
+    },
+    "placement": {
+        "availability-zone": "us-east-2c"
+    },
+    "profile": "default-hvm",
+    "public-hostname": (
+        "ec2-18-218-219-181.us-east-2.compute.amazonaws.com"),
+    "public-ipv4": "18.218.219.181",
+    "public-keys": {
+        "yourkeyname,e": [
+            "ssh-rsa AAAAW...DZ yourkeyname"
+        ]
+    },
+    "reservation-id": "r-09b4917135cdd33be",
+    "security-groups": "launch-wizard-4",
+    "services": {
+        "domain": "amazonaws.com",
+        "partition": "aws"
+    }
+}
+
+M_PATH_NET = 'cloudinit.sources.DataSourceEc2.net.'
+
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
@@ -267,30 +383,23 @@ class TestEc2(test_helpers.HttprettyTestCase):
                         register_mock_metaserver(instance_id_url, None)
         return ds
 
-    def test_network_config_property_returns_version_1_network_data(self):
-        """network_config property returns network version 1 for metadata.
-
-        Only one device is configured even when multiple exist in metadata.
-        """
+    def test_network_config_property_returns_version_2_network_data(self):
+        """network_config property returns network version 2 for metadata"""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md={'md': DEFAULT_METADATA})
-        find_fallback_path = (
-            'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
         with mock.patch(find_fallback_path) as m_find_fallback:
             m_find_fallback.return_value = 'eth9'
             ds.get_data()
 
         mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
-             'type': 'physical'}]}
-        patch_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
-        get_interface_mac_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': '06:17:04:d7:26:09'}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
         with mock.patch(patch_path) as m_get_interfaces_by_mac:
             with mock.patch(find_fallback_path) as m_find_fallback:
                 with mock.patch(get_interface_mac_path) as m_get_mac:
@@ -299,30 +408,59 @@ class TestEc2(test_helpers.HttprettyTestCase):
                     m_get_mac.return_value = mac1
                     self.assertEqual(expected, ds.network_config)
 
-    def test_network_config_property_set_dhcp4_on_private_ipv4(self):
-        """network_config property configures dhcp4 on private ipv4 nics.
+    def test_network_config_property_set_dhcp4(self):
+        """network_config property configures dhcp4 on nics with local-ipv4s.
 
-        Only one device is configured even when multiple exist in metadata.
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
         """
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md={'md': DEFAULT_METADATA})
-        find_fallback_path = (
-            'cloudinit.sources.DataSourceEc2.net.find_fallback_nic')
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
         with mock.patch(find_fallback_path) as m_find_fallback:
             m_find_fallback.return_value = 'eth9'
             ds.get_data()
 
         mac1 = '06:17:04:d7:26:0A'  # IPv4 only in DEFAULT_METADATA
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:0A', 'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}],
-             'type': 'physical'}]}
-        patch_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
-        get_interface_mac_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1.lower()}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
+        with mock.patch(patch_path) as m_get_interfaces_by_mac:
+            with mock.patch(find_fallback_path) as m_find_fallback:
+                with mock.patch(get_interface_mac_path) as m_get_mac:
+                    m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
+                    m_find_fallback.return_value = 'eth9'
+                    m_get_mac.return_value = mac1
+                    self.assertEqual(expected, ds.network_config)
+
+    def test_network_config_property_secondary_private_ips(self):
+        """network_config property configures any secondary ipv4 addresses.
+
+        Only one device is configured based on get_interfaces_by_mac even when
+        multiple MACs exist in metadata.
+        """
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
+            md={'md': SECONDARY_IP_METADATA_2018_09_24})
+        find_fallback_path = M_PATH_NET + 'find_fallback_nic'
+        with mock.patch(find_fallback_path) as m_find_fallback:
+            m_find_fallback.return_value = 'eth9'
+            ds.get_data()
+
+        mac1 = '0a:07:84:3d:6e:38'  # 1 secondary IPv4 and 2 secondary IPv6
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1}, 'set-name': 'eth9',
+            'addresses': ['172.31.45.70/20',
+                          '2600:1f16:292:100:f152:2222:3333:4444/128',
+                          '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            'dhcp4': True, 'dhcp6': True}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
+        get_interface_mac_path = M_PATH_NET + 'get_interface_mac'
         with mock.patch(patch_path) as m_get_interfaces_by_mac:
             with mock.patch(find_fallback_path) as m_find_fallback:
                 with mock.patch(get_interface_mac_path) as m_get_mac:
@@ -358,21 +496,18 @@ class TestEc2(test_helpers.HttprettyTestCase):
         register_mock_metaserver(
             'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA)
         mac1 = '06:17:04:d7:26:09'  # Defined in DEFAULT_METADATA
-        get_interface_mac_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interface_mac')
+        get_interface_mac_path = M_PATH_NET + 'get_interfaces_by_mac'
         ds.fallback_nic = 'eth9'
-        with mock.patch(get_interface_mac_path) as m_get_interface_mac:
-            m_get_interface_mac.return_value = mac1
+        with mock.patch(get_interface_mac_path) as m_get_interfaces_by_mac:
+            m_get_interfaces_by_mac.return_value = {mac1: 'eth9'}
             nc = ds.network_config  # Will re-crawl network metadata
             self.assertIsNotNone(nc)
         self.assertIn(
             'Refreshing stale metadata from prior to upgrade',
             self.logs.getvalue())
-        expected = {'version': 1, 'config': [
-            {'mac_address': '06:17:04:d7:26:09',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}],
-             'type': 'physical'}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(expected, ds.network_config)
 
     def test_ec2_get_instance_id_refreshes_identity_on_upgrade(self):
@@ -491,7 +626,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         logs_with_redacted = [log for log in all_logs if REDACT_TOK in log]
         logs_with_token = [log for log in all_logs if 'API-TOKEN' in log]
         self.assertEqual(1, len(logs_with_redacted_ttl))
-        self.assertEqual(79, len(logs_with_redacted))
+        self.assertEqual(81, len(logs_with_redacted))
         self.assertEqual(0, len(logs_with_token))
 
     @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
@@ -612,6 +747,44 @@ class TestEc2(test_helpers.HttprettyTestCase):
         self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
+class TestGetSecondaryAddresses(test_helpers.CiTestCase):
+
+    mac = '06:17:04:d7:26:ff'
+    with_logs = True
+
+    def test_md_with_no_secondary_addresses(self):
+        """Empty list is returned when nic metadata contains no secondary ip"""
+        self.assertEqual([], ec2.get_secondary_addresses(NIC2_MD, self.mac))
+
+    def test_md_with_secondary_v4_and_v6_addresses(self):
+        """All secondary addresses are returned from nic metadata"""
+        self.assertEqual(
+            ['172.31.45.70/20', '2600:1f16:292:100:f152:2222:3333:4444/128',
+             '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            ec2.get_secondary_addresses(NIC1_MD_IPV4_IPV6_MULTI_IP, self.mac))
+
+    def test_invalid_ipv4_ipv6_cidr_metadata_logged_with_defaults(self):
+        """Any invalid subnet-ipv(4|6)-cidr-block values use defaults"""
+        invalid_cidr_md = copy.deepcopy(NIC1_MD_IPV4_IPV6_MULTI_IP)
+        invalid_cidr_md['subnet-ipv4-cidr-block'] = "something-unexpected"
+        invalid_cidr_md['subnet-ipv6-cidr-block'] = "not/sure/what/this/is"
+        self.assertEqual(
+            ['172.31.45.70/24', '2600:1f16:292:100:f152:2222:3333:4444/128',
+             '2600:1f16:292:100:f153:12a3:c37c:11f9/128'],
+            ec2.get_secondary_addresses(invalid_cidr_md, self.mac))
+        expected_logs = [
+           "WARNING: Could not parse subnet-ipv4-cidr-block"
+           " something-unexpected for mac 06:17:04:d7:26:ff."
+           " ipv4 network config prefix defaults to /24",
+           "WARNING: Could not parse subnet-ipv6-cidr-block"
+           " not/sure/what/this/is for mac 06:17:04:d7:26:ff."
+           " ipv6 network config prefix defaults to /128"
+        ]
+        logs = self.logs.getvalue()
+        for log in expected_logs:
+            self.assertIn(log, logs)
+
+
 class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def setUp(self):
@@ -619,16 +792,16 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         self.mac1 = '06:17:04:d7:26:09'
         self.network_metadata = {
             'interfaces': {'macs': {
-                self.mac1: {'public-ipv4s': '172.31.2.16'}}}}
+                self.mac1: {'mac': self.mac1, 'public-ipv4s': '172.31.2.16'}}}}
 
     def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
         """Any mac absent from metadata is skipped by network config."""
         macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'}
 
         # DE:AD:BE:EF:FF:FF represented by OS but not in metadata
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -642,15 +815,15 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
             network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
                 network_metadata_ipv6, macs_to_nics))
 
-    def test_convert_ec2_metadata_network_config_handles_local_dhcp4(self):
+    def test_convert_ec2_metadata_network_config_local_only_dhcp4(self):
         """Config dhcp4 when there are no public addresses in public-ipv4s."""
         macs_to_nics = {self.mac1: 'eth9'}
         network_metadata_ipv6 = copy.deepcopy(self.network_metadata)
@@ -658,9 +831,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
             network_metadata_ipv6['interfaces']['macs'][self.mac1])
         nic1_metadata['local-ipv4s'] = '172.3.3.15'
         nic1_metadata.pop('public-ipv4s')
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -675,16 +848,16 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata['public-ipv4s'] = ''
 
         # When no ipv4 or ipv6 content but fallback_nic set, set dhcp4 config.
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
                 network_metadata_ipv6, macs_to_nics, fallback_nic='eth9'))
 
     def test_convert_ec2_metadata_network_config_handles_local_v4_and_v6(self):
-        """When dhcp6 is public and dhcp4 is set to local enable both."""
+        """When ipv6s and local-ipv4s are non-empty, enable dhcp6 and dhcp4."""
         macs_to_nics = {self.mac1: 'eth9'}
         network_metadata_both = copy.deepcopy(self.network_metadata)
         nic1_metadata = (
@@ -692,10 +865,35 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
         nic1_metadata.pop('public-ipv4s')
         nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
+        self.assertEqual(
+            expected,
+            ec2.convert_ec2_metadata_network_config(
+                network_metadata_both, macs_to_nics))
+
+    def test_convert_ec2_metadata_network_config_handles_multiple_nics(self):
+        """DHCP route-metric increases on secondary NICs for IPv4 and IPv6."""
+        mac2 = '06:17:04:d7:26:0a'
+        macs_to_nics = {self.mac1: 'eth9', mac2: 'eth10'}
+        network_metadata_both = copy.deepcopy(self.network_metadata)
+        # Add 2nd nic info
+        network_metadata_both['interfaces']['macs'][mac2] = NIC2_MD
+        nic1_metadata = (
+            network_metadata_both['interfaces']['macs'][self.mac1])
+        nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
+        nic1_metadata.pop('public-ipv4s')  # No public-ipv4 IPs in cfg
+        nic1_metadata['local-ipv4s'] = '10.0.0.42'  # Local ipv4 only on vpc
+        expected = {'version': 2, 'ethernets': {
+            'eth9': {
+                'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+                'dhcp4': True, 'dhcp4-overrides': {'route-metric': 100},
+                'dhcp6': True, 'dhcp6-overrides': {'route-metric': 100}},
+            'eth10': {
+                'match': {'macaddress': mac2}, 'set-name': 'eth10',
+                'dhcp4': True, 'dhcp4-overrides': {'route-metric': 200},
+                'dhcp6': False}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -708,10 +906,9 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
         nic1_metadata = (
             network_metadata_both['interfaces']['macs'][self.mac1])
         nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64'
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]}
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1}, 'set-name': 'eth9',
+            'dhcp4': True, 'dhcp6': True}}}
         self.assertEqual(
             expected,
             ec2.convert_ec2_metadata_network_config(
@@ -719,12 +916,10 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self):
         """Convert Ec2 Metadata calls get_interfaces_by_mac by default."""
-        expected = {'version': 1, 'config': [
-            {'mac_address': self.mac1, 'type': 'physical',
-             'name': 'eth9',
-             'subnets': [{'type': 'dhcp4'}]}]}
-        patch_path = (
-            'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac')
+        expected = {'version': 2, 'ethernets': {'eth9': {
+            'match': {'macaddress': self.mac1},
+            'set-name': 'eth9', 'dhcp4': True, 'dhcp6': False}}}
+        patch_path = M_PATH_NET + 'get_interfaces_by_mac'
         with mock.patch(patch_path) as m_get_interfaces_by_mac:
             m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'}
             self.assertEqual(
-- 
cgit v1.2.3


From 70dbccbbb27f7cc3f2decd692d41403f0d745c62 Mon Sep 17 00:00:00 2001
From: Daniel Watkins <oddbloke@ubuntu.com>
Date: Fri, 1 May 2020 16:56:56 -0400
Subject: DataSourceEc2: use metadata's NIC ordering to determine route-metrics
 (#342)

We want to set route-metrics such that NICs are configured with the priority that they are given in the network metadata that we receive from the IMDS.  (This switches away from using MAC ordering.)

This also required the following test changes:

* reverse the sort order of the MACs in test data (so that they would trigger the bug being fixed)
* fix up the key names in `NIC2_MD` (which were under_scored instead of dash-separated)
* use a full interface dict (rather than a minimal one) for `TestConvertEc2MetadataNetworkConfig`

LP: #1876312
---
 cloudinit/sources/DataSourceEc2.py          |  5 ++--
 tests/unittests/test_datasource/test_ec2.py | 41 ++++++++++++++++-------------
 2 files changed, 26 insertions(+), 20 deletions(-)

(limited to 'cloudinit/sources/DataSourceEc2.py')

diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 4203c3a6..355b4e2f 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -762,13 +762,14 @@ def convert_ec2_metadata_network_config(
         netcfg['ethernets'][nic_name] = dev_config
         return netcfg
     # Apply network config for all nics and any secondary IPv4/v6 addresses
-    nic_idx = 1
     for mac, nic_name in sorted(macs_to_nics.items()):
         nic_metadata = macs_metadata.get(mac)
         if not nic_metadata:
             continue  # Not a physical nic represented in metadata
+        # device-number is zero-indexed, we want it 1-indexed for the
+        # multiplication on the following line
+        nic_idx = int(nic_metadata['device-number']) + 1
         dhcp_override = {'route-metric': nic_idx * 100}
-        nic_idx += 1
         dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override,
                       'dhcp6': False,
                       'match': {'macaddress': mac.lower()},
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 9126c676..d413d1cd 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -38,6 +38,8 @@ DYNAMIC_METADATA = {
 # python3 -c 'import json
 # from cloudinit.ec2_utils import get_instance_metadata as gm
 # print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
+# Note that the MAC addresses have been modified to sort in the opposite order
+# to the device-number attribute, to test LP: #1876312
 DEFAULT_METADATA = {
     "ami-id": "ami-8b92b4ee",
     "ami-launch-index": "0",
@@ -77,7 +79,7 @@ DEFAULT_METADATA = {
                     "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
                     "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
                 },
-                "06:17:04:d7:26:0A": {
+                "06:17:04:d7:26:08": {
                     "device-number": "1",   # Only IPv4 local config
                     "interface-id": "eni-e44ef49f",
                     "ipv4-associations": {"": "172.3.3.16"},
@@ -85,7 +87,7 @@ DEFAULT_METADATA = {
                     "local-hostname": ("ip-172-3-3-16.us-east-2."
                                        "compute.internal"),
                     "local-ipv4s": "172.3.3.16",
-                    "mac": "06:17:04:d7:26:0A",
+                    "mac": "06:17:04:d7:26:08",
                     "owner-id": "950047163771",
                     "public-hostname": ("ec2-172-3-3-16.us-east-2."
                                         "compute.amazonaws.com"),
@@ -152,19 +154,19 @@ NIC1_MD_IPV4_IPV6_MULTI_IP = {
 }
 
 NIC2_MD = {
-    "device_number": "1",
-    "interface_id": "eni-043cdce36ded5e79f",
-    "local_hostname": "ip-172-31-47-221.us-east-2.compute.internal",
-    "local_ipv4s": "172.31.47.221",
+    "device-number": "1",
+    "interface-id": "eni-043cdce36ded5e79f",
+    "local-hostname": "ip-172-31-47-221.us-east-2.compute.internal",
+    "local-ipv4s": "172.31.47.221",
     "mac": "0a:75:69:92:e2:16",
-    "owner_id": "329910648901",
-    "security_group_ids": "sg-0d68fef37d8cc9b77",
-    "security_groups": "launch-wizard-17",
-    "subnet_id": "subnet-9d7ba0d1",
-    "subnet_ipv4_cidr_block": "172.31.32.0/20",
-    "vpc_id": "vpc-a07f62c8",
-    "vpc_ipv4_cidr_block": "172.31.0.0/16",
-    "vpc_ipv4_cidr_blocks": "172.31.0.0/16"
+    "owner-id": "329910648901",
+    "security-group-ids": "sg-0d68fef37d8cc9b77",
+    "security-groups": "launch-wizard-17",
+    "subnet-id": "subnet-9d7ba0d1",
+    "subnet-ipv4-cidr-block": "172.31.32.0/20",
+    "vpc-id": "vpc-a07f62c8",
+    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+    "vpc-ipv4-cidr-blocks": "172.31.0.0/16"
 }
 
 SECONDARY_IP_METADATA_2018_09_24 = {
@@ -423,7 +425,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             m_find_fallback.return_value = 'eth9'
             ds.get_data()
 
-        mac1 = '06:17:04:d7:26:0A'  # IPv4 only in DEFAULT_METADATA
+        mac1 = '06:17:04:d7:26:08'  # IPv4 only in DEFAULT_METADATA
         expected = {'version': 2, 'ethernets': {'eth9': {
             'match': {'macaddress': mac1.lower()}, 'set-name': 'eth9',
             'dhcp4': True, 'dhcp6': False}}}
@@ -790,9 +792,12 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
     def setUp(self):
         super(TestConvertEc2MetadataNetworkConfig, self).setUp()
         self.mac1 = '06:17:04:d7:26:09'
+        interface_dict = copy.deepcopy(
+            DEFAULT_METADATA['network']['interfaces']['macs'][self.mac1])
+        # These tests are written assuming the base interface doesn't have IPv6
+        interface_dict.pop('ipv6s')
         self.network_metadata = {
-            'interfaces': {'macs': {
-                self.mac1: {'mac': self.mac1, 'public-ipv4s': '172.31.2.16'}}}}
+            'interfaces': {'macs': {self.mac1: interface_dict}}}
 
     def test_convert_ec2_metadata_network_config_skips_absent_macs(self):
         """Any mac absent from metadata is skipped by network config."""
@@ -875,7 +880,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase):
 
     def test_convert_ec2_metadata_network_config_handles_multiple_nics(self):
         """DHCP route-metric increases on secondary NICs for IPv4 and IPv6."""
-        mac2 = '06:17:04:d7:26:0a'
+        mac2 = '06:17:04:d7:26:08'
         macs_to_nics = {self.mac1: 'eth9', mac2: 'eth10'}
         network_metadata_both = copy.deepcopy(self.network_metadata)
         # Add 2nd nic info
-- 
cgit v1.2.3


From 07104504ab5b30efd2d1f7a8c36effe18b8e5fe0 Mon Sep 17 00:00:00 2001
From: Paride Legovini <paride.legovini@canonical.com>
Date: Tue, 25 Aug 2020 17:21:18 +0200
Subject: tox: bump the pylint version to 2.6.0 in the default run (#544)

Changes:

tox: bump the pylint version to 2.6.0 in the default run
Fix pylint 2.6.0 W0707 warnings (raise-missing-from)
---
 cloudinit/analyze/show.py                          |  2 +-
 cloudinit/cmd/devel/make_mime.py                   |  6 ++--
 cloudinit/config/cc_disk_setup.py                  | 36 ++++++++++++++--------
 cloudinit/config/cc_growpart.py                    |  8 ++---
 cloudinit/config/cc_power_state_change.py          | 12 +++++---
 cloudinit/config/cc_rsyslog.py                     |  6 ++--
 cloudinit/config/cc_set_hostname.py                |  2 +-
 cloudinit/config/cc_ubuntu_advantage.py            |  2 +-
 cloudinit/config/schema.py                         |  2 +-
 cloudinit/distros/__init__.py                      |  5 +--
 cloudinit/distros/arch.py                          |  4 +--
 cloudinit/distros/parsers/resolv_conf.py           |  7 +++--
 cloudinit/gpg.py                                   |  5 +--
 cloudinit/handlers/jinja_template.py               |  3 +-
 cloudinit/net/__init__.py                          |  7 +++--
 cloudinit/net/cmdline.py                           |  4 +--
 cloudinit/net/dhcp.py                              |  4 +--
 cloudinit/net/network_state.py                     | 19 +++++++-----
 cloudinit/sources/DataSourceAzure.py               |  2 +-
 cloudinit/sources/DataSourceEc2.py                 |  8 +++--
 cloudinit/sources/DataSourceIBMCloud.py            |  3 +-
 cloudinit/sources/DataSourceMAAS.py                |  3 +-
 cloudinit/sources/DataSourceOpenNebula.py          | 13 +++++---
 cloudinit/sources/DataSourceOpenStack.py           |  4 +--
 cloudinit/sources/DataSourceSmartOS.py             |  4 ++-
 cloudinit/sources/helpers/azure.py                 | 34 +++++++++++---------
 cloudinit/sources/helpers/netlink.py               |  2 +-
 cloudinit/sources/helpers/openstack.py             | 34 +++++++++++---------
 cloudinit/subp.py                                  |  3 +-
 cloudinit/url_helper.py                            |  6 ++--
 cloudinit/util.py                                  |  8 ++---
 tests/cloud_tests/platforms/azurecloud/instance.py |  7 +++--
 tests/cloud_tests/platforms/azurecloud/platform.py | 23 +++++++++-----
 tests/cloud_tests/platforms/ec2/instance.py        |  4 +--
 tests/cloud_tests/platforms/ec2/platform.py        | 18 ++++++-----
 tests/cloud_tests/platforms/lxd/instance.py        |  3 +-
 tests/cloud_tests/platforms/platforms.py           |  6 ++--
 tests/cloud_tests/testcases/__init__.py            |  6 ++--
 tools/mock-meta.py                                 | 14 +++++----
 tox.ini                                            |  2 +-
 40 files changed, 205 insertions(+), 136 deletions(-)

(limited to 'cloudinit/sources/DataSourceEc2.py')

diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py
index 0c825b23..01a4d3e5 100644
--- a/cloudinit/analyze/show.py
+++ b/cloudinit/analyze/show.py
@@ -267,7 +267,7 @@ def gather_timestamps_using_systemd():
                 except OSError as err:
                     raise RuntimeError('Could not determine container boot '
                                        'time from /proc/1/cmdline. ({})'
-                                       .format(err))
+                                       .format(err)) from err
                 status = CONTAINER_CODE
             else:
                 status = FAIL_CODE
diff --git a/cloudinit/cmd/devel/make_mime.py b/cloudinit/cmd/devel/make_mime.py
index 77e10540..4e6a5778 100755
--- a/cloudinit/cmd/devel/make_mime.py
+++ b/cloudinit/cmd/devel/make_mime.py
@@ -22,8 +22,10 @@ def file_content_type(text):
     try:
         filename, content_type = text.split(":", 1)
         return (open(filename, 'r'), filename, content_type.strip())
-    except ValueError:
-        raise argparse.ArgumentError(text, "Invalid value for %r" % (text))
+    except ValueError as e:
+        raise argparse.ArgumentError(
+            text, "Invalid value for %r" % (text)
+        ) from e
 
 
 def get_parser(parser=None):
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index d957cfe3..a7bdc703 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -251,7 +251,9 @@ def enumerate_disk(device, nodeps=False):
     try:
         info, _err = subp.subp(lsblk_cmd)
     except Exception as e:
-        raise Exception("Failed during disk check for %s\n%s" % (device, e))
+        raise Exception(
+            "Failed during disk check for %s\n%s" % (device, e)
+        ) from e
 
     parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
 
@@ -313,7 +315,9 @@ def check_fs(device):
     try:
         out, _err = subp.subp(blkid_cmd, rcs=[0, 2])
     except Exception as e:
-        raise Exception("Failed during disk check for %s\n%s" % (device, e))
+        raise Exception(
+            "Failed during disk check for %s\n%s" % (device, e)
+        ) from e
 
     if out:
         if len(out.splitlines()) == 1:
@@ -428,8 +432,8 @@ def get_dyn_func(*args):
         else:
             return globals()[func_name]
 
-    except KeyError:
-        raise Exception("No such function %s to call!" % func_name)
+    except KeyError as e:
+        raise Exception("No such function %s to call!" % func_name) from e
 
 
 def get_hdd_size(device):
@@ -437,7 +441,7 @@ def get_hdd_size(device):
         size_in_bytes, _ = subp.subp([BLKDEV_CMD, '--getsize64', device])
         sector_size, _ = subp.subp([BLKDEV_CMD, '--getss', device])
     except Exception as e:
-        raise Exception("Failed to get %s size\n%s" % (device, e))
+        raise Exception("Failed to get %s size\n%s" % (device, e)) from e
 
     return int(size_in_bytes) / int(sector_size)
 
@@ -455,8 +459,9 @@ def check_partition_mbr_layout(device, layout):
     try:
         out, _err = subp.subp(prt_cmd, data="%s\n" % layout)
     except Exception as e:
-        raise Exception("Error running partition command on %s\n%s" % (
-                        device, e))
+        raise Exception(
+            "Error running partition command on %s\n%s" % (device, e)
+        ) from e
 
     found_layout = []
     for line in out.splitlines():
@@ -485,8 +490,9 @@ def check_partition_gpt_layout(device, layout):
     try:
         out, _err = subp.subp(prt_cmd, update_env=LANG_C_ENV)
     except Exception as e:
-        raise Exception("Error running partition command on %s\n%s" % (
-                        device, e))
+        raise Exception(
+            "Error running partition command on %s\n%s" % (device, e)
+        ) from e
 
     out_lines = iter(out.splitlines())
     # Skip header.  Output looks like:
@@ -657,8 +663,10 @@ def purge_disk(device):
             try:
                 LOG.info("Purging filesystem on /dev/%s", d['name'])
                 subp.subp(wipefs_cmd)
-            except Exception:
-                raise Exception("Failed FS purge of /dev/%s" % d['name'])
+            except Exception as e:
+                raise Exception(
+                    "Failed FS purge of /dev/%s" % d['name']
+                ) from e
 
     purge_disk_ptable(device)
 
@@ -700,7 +708,9 @@ def exec_mkpart_mbr(device, layout):
     try:
         subp.subp(prt_cmd, data="%s\n" % layout)
     except Exception as e:
-        raise Exception("Failed to partition device %s\n%s" % (device, e))
+        raise Exception(
+            "Failed to partition device %s\n%s" % (device, e)
+        ) from e
 
     read_parttbl(device)
 
@@ -997,6 +1007,6 @@ def mkfs(fs_cfg):
     try:
         subp.subp(fs_cmd, shell=shell)
     except Exception as e:
-        raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e))
+        raise Exception("Failed to exec of '%s':\n%s" % (fs_cmd, e)) from e
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index c5d93f81..237c3d02 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -148,14 +148,14 @@ class ResizeGrowPart(object):
             if e.exit_code != 1:
                 util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)",
                             diskdev, partnum)
-                raise ResizeFailedException(e)
+                raise ResizeFailedException(e) from e
             return (before, before)
 
         try:
             subp.subp(["growpart", diskdev, partnum])
         except subp.ProcessExecutionError as e:
             util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum)
-            raise ResizeFailedException(e)
+            raise ResizeFailedException(e) from e
 
         return (before, get_size(partdev))
 
@@ -187,14 +187,14 @@ class ResizeGpart(object):
         except subp.ProcessExecutionError as e:
             if e.exit_code != 0:
                 util.logexc(LOG, "Failed: gpart recover %s", diskdev)
-                raise ResizeFailedException(e)
+                raise ResizeFailedException(e) from e
 
         before = get_size(partdev)
         try:
             subp.subp(["gpart", "resize", "-i", partnum, diskdev])
         except subp.ProcessExecutionError as e:
             util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
-            raise ResizeFailedException(e)
+            raise ResizeFailedException(e) from e
 
         # Since growing the FS requires a reboot, make sure we reboot
         # first when this module has finished.
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index ab953a0d..6fcb8a7d 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -196,10 +196,11 @@ def load_power_state(cfg, distro_name):
 
     try:
         delay = convert_delay(delay, fmt=fmt, scale=scale)
-    except ValueError:
+    except ValueError as e:
         raise TypeError(
             "power_state[delay] must be 'now' or '+m' (minutes)."
-            " found '%s'." % delay)
+            " found '%s'." % delay
+        ) from e
 
     args = command + [delay]
     if message:
@@ -207,9 +208,10 @@ def load_power_state(cfg, distro_name):
 
     try:
         timeout = float(pstate.get('timeout', 30.0))
-    except ValueError:
-        raise ValueError("failed to convert timeout '%s' to float." %
-                         pstate['timeout'])
+    except ValueError as e:
+        raise ValueError(
+            "failed to convert timeout '%s' to float." % pstate['timeout']
+        ) from e
 
     condition = pstate.get("condition", True)
     if not isinstance(condition, (str, list, bool)):
diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py
index 1354885a..2a2bc931 100644
--- a/cloudinit/config/cc_rsyslog.py
+++ b/cloudinit/config/cc_rsyslog.py
@@ -347,8 +347,10 @@ class SyslogRemotesLine(object):
         if self.port:
             try:
                 int(self.port)
-            except ValueError:
-                raise ValueError("port '%s' is not an integer" % self.port)
+            except ValueError as e:
+                raise ValueError(
+                    "port '%s' is not an integer" % self.port
+                ) from e
 
         if not self.addr:
             raise ValueError("address is required")
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index c13020d8..1d23d80d 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args):
     except Exception as e:
         msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname)
         util.logexc(log, msg)
-        raise SetHostnameError("%s: %s" % (msg, e))
+        raise SetHostnameError("%s: %s" % (msg, e)) from e
     write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn})
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py
index 35ded5db..d61dc655 100644
--- a/cloudinit/config/cc_ubuntu_advantage.py
+++ b/cloudinit/config/cc_ubuntu_advantage.py
@@ -115,7 +115,7 @@ def configure_ua(token=None, enable=None):
         msg = 'Failure attaching Ubuntu Advantage:\n{error}'.format(
             error=str(e))
         util.logexc(LOG, msg)
-        raise RuntimeError(msg)
+        raise RuntimeError(msg) from e
     enable_errors = []
     for service in enable:
         try:
diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
index 2d8c7577..8a966aee 100644
--- a/cloudinit/config/schema.py
+++ b/cloudinit/config/schema.py
@@ -210,7 +210,7 @@ def validate_cloudconfig_file(config_path, schema, annotate=False):
         error = SchemaValidationError(errors)
         if annotate:
             print(annotated_cloudconfig_file({}, content, error.schema_errors))
-        raise error
+        raise error from e
     try:
         validate_cloudconfig_schema(
             cloudconfig, schema, strict=True)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index effb4276..2537608f 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -607,10 +607,11 @@ class Distro(metaclass=abc.ABCMeta):
         lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
         try:
             cmd = next(tool for tool in lock_tools if subp.which(tool[0]))
-        except StopIteration:
+        except StopIteration as e:
             raise RuntimeError((
                 "Unable to lock user account '%s'. No tools available. "
-                "  Tried: %s.") % (name, [c[0] for c in lock_tools]))
+                "  Tried: %s.") % (name, [c[0] for c in lock_tools])
+            ) from e
         try:
             subp.subp(cmd)
         except Exception as e:
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 038aa9ac..967be168 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -61,9 +61,9 @@ class Distro(distros.Distro):
     def _write_network_config(self, netconfig):
         try:
             return self._supported_write_network_config(netconfig)
-        except RendererNotFoundError:
+        except RendererNotFoundError as e:
             # Fall back to old _write_network
-            raise NotImplementedError
+            raise NotImplementedError from e
 
     def _write_network(self, settings):
         entries = net_util.translate_network(settings)
diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py
index 299d54b5..62929d03 100644
--- a/cloudinit/distros/parsers/resolv_conf.py
+++ b/cloudinit/distros/parsers/resolv_conf.py
@@ -150,9 +150,10 @@ class ResolvConf(object):
                 tail = ''
             try:
                 (cfg_opt, cfg_values) = head.split(None, 1)
-            except (IndexError, ValueError):
-                raise IOError("Incorrectly formatted resolv.conf line %s"
-                              % (i + 1))
+            except (IndexError, ValueError) as e:
+                raise IOError(
+                    "Incorrectly formatted resolv.conf line %s" % (i + 1)
+                ) from e
             if cfg_opt not in ['nameserver', 'domain',
                                'search', 'sortlist', 'options']:
                 raise IOError("Unexpected resolv.conf option %s" % (cfg_opt))
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
index 72b5ac59..be0ca0ea 100644
--- a/cloudinit/gpg.py
+++ b/cloudinit/gpg.py
@@ -63,10 +63,11 @@ def recv_key(key, keyserver, retries=(1, 1)):
                 "Import failed with exit code %d, will try again in %ss",
                 error.exit_code, naplen)
             time.sleep(naplen)
-        except StopIteration:
+        except StopIteration as e:
             raise ValueError(
                 ("Failed to import key '%s' from keyserver '%s' "
-                 "after %d tries: %s") % (key, keyserver, trynum, error))
+                 "after %d tries: %s") % (key, keyserver, trynum, error)
+            ) from e
 
 
 def delete_key(key):
diff --git a/cloudinit/handlers/jinja_template.py b/cloudinit/handlers/jinja_template.py
index ce3accf6..aadfbf86 100644
--- a/cloudinit/handlers/jinja_template.py
+++ b/cloudinit/handlers/jinja_template.py
@@ -83,7 +83,8 @@ def render_jinja_payload_from_file(
         if e.errno == EACCES:
             raise RuntimeError(
                 'Cannot render jinja template vars. No read permission on'
-                " '%s'. Try sudo" % instance_data_file)
+                " '%s'. Try sudo" % instance_data_file
+            ) from e
 
     rendered_payload = render_jinja_payload(
         payload, payload_fn, instance_data, debug)
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 322af77b..e233149a 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -506,7 +506,9 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
     try:
         _rename_interfaces(extract_physdevs(netcfg))
     except RuntimeError as e:
-        raise RuntimeError('Failed to apply network config names: %s' % e)
+        raise RuntimeError(
+            'Failed to apply network config names: %s' % e
+        ) from e
 
 
 def interface_has_own_mac(ifname, strict=False):
@@ -965,7 +967,8 @@ class EphemeralIPv4Network(object):
             self.prefix = mask_to_net_prefix(prefix_or_mask)
         except ValueError as e:
             raise ValueError(
-                'Cannot setup network: {0}'.format(e))
+                'Cannot setup network: {0}'.format(e)
+            ) from e
 
         self.connectivity_url = connectivity_url
         self.interface = interface
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index 7ca7262b..cc8dc17b 100755
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -112,8 +112,8 @@ def _klibc_to_config_entry(content, mac_addrs=None):
     data = util.load_shell_content(content)
     try:
         name = data['DEVICE'] if 'DEVICE' in data else data['DEVICE6']
-    except KeyError:
-        raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data")
+    except KeyError as e:
+        raise ValueError("no 'DEVICE' or 'DEVICE6' entry in data") from e
 
     # ipconfig on precise does not write PROTO
     # IPv6 config gives us IPV6PROTO, not PROTO.
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
index 9230cf7a..4394c68b 100644
--- a/cloudinit/net/dhcp.py
+++ b/cloudinit/net/dhcp.py
@@ -82,8 +82,8 @@ class EphemeralDHCPv4(object):
         try:
             leases = maybe_perform_dhcp_discovery(
                 self.iface, self.dhcp_log_func)
-        except InvalidDHCPLeaseFileError:
-            raise NoDHCPLeaseError()
+        except InvalidDHCPLeaseFileError as e:
+            raise NoDHCPLeaseError() from e
         if not leases:
             raise NoDHCPLeaseError()
         self.lease = leases[-1]
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index 7bfe8be0..b2f7d31e 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -297,9 +297,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta):
             command_type = command['type']
             try:
                 handler = self.command_handlers[command_type]
-            except KeyError:
-                raise RuntimeError("No handler found for"
-                                   " command '%s'" % command_type)
+            except KeyError as e:
+                raise RuntimeError(
+                    "No handler found for  command '%s'" % command_type
+                ) from e
             try:
                 handler(self, command)
             except InvalidCommand:
@@ -316,9 +317,10 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta):
                 continue
             try:
                 handler = self.command_handlers[command_type]
-            except KeyError:
-                raise RuntimeError("No handler found for"
-                                   " command '%s'" % command_type)
+            except KeyError as e:
+                raise RuntimeError(
+                    "No handler found for command '%s'" % command_type
+                ) from e
             try:
                 handler(self, command)
                 self._v2_common(command)
@@ -914,9 +916,10 @@ def _normalize_route(route):
     if metric:
         try:
             normal_route['metric'] = int(metric)
-        except ValueError:
+        except ValueError as e:
             raise TypeError(
-                'Route config metric {} is not an integer'.format(metric))
+                'Route config metric {} is not an integer'.format(metric)
+            ) from e
     return normal_route
 
 
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 86cc7c28..f3c6452b 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -1147,7 +1147,7 @@ def read_azure_ovf(contents):
     except Exception as e:
         error_str = "Invalid ovf-env.xml: %s" % e
         report_diagnostic_event(error_str)
-        raise BrokenAzureDataSource(error_str)
+        raise BrokenAzureDataSource(error_str) from e
 
     results = find_child(dom.documentElement,
                          lambda n: n.localName == "ProvisioningSection")
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 355b4e2f..1d09c12a 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -617,9 +617,11 @@ def parse_strict_mode(cfgval):
     if sleep:
         try:
             sleep = int(sleep)
-        except ValueError:
-            raise ValueError("Invalid sleep '%s' in strict_id setting '%s': "
-                             "not an integer" % (sleep, cfgval))
+        except ValueError as e:
+            raise ValueError(
+                "Invalid sleep '%s' in strict_id setting '%s': not an integer"
+                % (sleep, cfgval)
+            ) from e
     else:
         sleep = None
 
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
index d2aa9a58..8d196185 100644
--- a/cloudinit/sources/DataSourceIBMCloud.py
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -303,7 +303,8 @@ def read_md():
     except sources.BrokenMetadata as e:
         raise RuntimeError(
             "Failed reading IBM config disk (platform=%s path=%s): %s" %
-            (platform, path, e))
+            (platform, path, e)
+        ) from e
 
     ret.update(results)
     return ret
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index c80f70c2..9156925f 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -226,7 +226,8 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None,
         except url_helper.UrlError as e:
             if e.code == 404 and not optional:
                 raise MAASSeedDirMalformed(
-                    "Missing required %s: %s" % (path, e))
+                    "Missing required %s: %s" % (path, e)
+                ) from e
             elif e.code != 404:
                 raise e
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 12b1f94f..45481938 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -399,18 +399,23 @@ def read_context_disk_dir(source_dir, distro, asuser=None):
         if asuser is not None:
             try:
                 pwd.getpwnam(asuser)
-            except KeyError:
+            except KeyError as e:
                 raise BrokenContextDiskDir(
                     "configured user '{user}' does not exist".format(
-                        user=asuser))
+                        user=asuser)
+                ) from e
         try:
             path = os.path.join(source_dir, 'context.sh')
             content = util.load_file(path)
             context = parse_shell_config(content, asuser=asuser)
         except subp.ProcessExecutionError as e:
-            raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
+            raise BrokenContextDiskDir(
+                "Error processing context.sh: %s" % (e)
+            ) from e
         except IOError as e:
-            raise NonContextDiskDir("Error reading context.sh: %s" % (e))
+            raise NonContextDiskDir(
+                "Error reading context.sh: %s" % (e)
+            ) from e
     else:
         raise NonContextDiskDir("Missing context.sh")
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index bf539091..d4b43f44 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -194,10 +194,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                         'timeout': url_params.timeout_seconds})
         except openstack.NonReadable as e:
             raise sources.InvalidMetaDataException(str(e))
-        except (openstack.BrokenMetadata, IOError):
+        except (openstack.BrokenMetadata, IOError) as e:
             msg = 'Broken metadata address {addr}'.format(
                 addr=self.metadata_address)
-            raise sources.InvalidMetaDataException(msg)
+            raise sources.InvalidMetaDataException(msg) from e
         return result
 
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 843b3a2a..f1f903bc 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -413,7 +413,9 @@ class JoyentMetadataClient(object):
                 response.append(byte)
             except OSError as exc:
                 if exc.errno == errno.EAGAIN:
-                    raise JoyentMetadataTimeoutException(msg % as_ascii())
+                    raise JoyentMetadataTimeoutException(
+                        msg % as_ascii()
+                    ) from exc
                 raise
 
     def _write(self, msg):
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 6156c75b..b968a96f 100755
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -98,8 +98,10 @@ def get_boot_telemetry():
     LOG.debug("Collecting boot telemetry")
     try:
         kernel_start = float(time.time()) - float(util.uptime())
-    except ValueError:
-        raise RuntimeError("Failed to determine kernel start timestamp")
+    except ValueError as e:
+        raise RuntimeError(
+            "Failed to determine kernel start timestamp"
+        ) from e
 
     try:
         out, _ = subp.subp(['/bin/systemctl',
@@ -116,12 +118,13 @@ def get_boot_telemetry():
 
         user_start = kernel_start + (float(tsm) / 1000000)
     except subp.ProcessExecutionError as e:
-        raise RuntimeError("Failed to get UserspaceTimestampMonotonic: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to get UserspaceTimestampMonotonic: %s" % e
+        ) from e
     except ValueError as e:
-        raise RuntimeError("Failed to parse "
-                           "UserspaceTimestampMonotonic from systemd: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to parse UserspaceTimestampMonotonic from systemd: %s" % e
+        ) from e
 
     try:
         out, _ = subp.subp(['/bin/systemctl', 'show',
@@ -137,12 +140,14 @@ def get_boot_telemetry():
 
         cloudinit_activation = kernel_start + (float(tsm) / 1000000)
     except subp.ProcessExecutionError as e:
-        raise RuntimeError("Failed to get InactiveExitTimestampMonotonic: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to get InactiveExitTimestampMonotonic: %s" % e
+        ) from e
     except ValueError as e:
-        raise RuntimeError("Failed to parse "
-                           "InactiveExitTimestampMonotonic from systemd: %s"
-                           % e)
+        raise RuntimeError(
+            "Failed to parse InactiveExitTimestampMonotonic from systemd: %s"
+            % e
+        ) from e
 
     evt = events.ReportingEvent(
         BOOT_EVENT_TYPE, 'boot-telemetry',
@@ -642,9 +647,10 @@ class WALinuxAgentShim:
             try:
                 name = os.path.basename(hook_file).replace('.json', '')
                 dhcp_options[name] = json.loads(util.load_file((hook_file)))
-            except ValueError:
+            except ValueError as e:
                 raise ValueError(
-                    '{_file} is not valid JSON data'.format(_file=hook_file))
+                    '{_file} is not valid JSON data'.format(_file=hook_file)
+                ) from e
         return dhcp_options
 
     @staticmethod
diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py
index a74a3588..c2ad587b 100644
--- a/cloudinit/sources/helpers/netlink.py
+++ b/cloudinit/sources/helpers/netlink.py
@@ -74,7 +74,7 @@ def create_bound_netlink_socket():
         netlink_socket.setblocking(0)
     except socket.error as e:
         msg = "Exception during netlink socket create: %s" % e
-        raise NetlinkCreateSocketError(msg)
+        raise NetlinkCreateSocketError(msg) from e
     LOG.debug("Created netlink socket")
     return netlink_socket
 
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 1050efb0..65e020c5 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -280,8 +280,9 @@ class BaseReader(metaclass=abc.ABCMeta):
                 try:
                     data = translator(data)
                 except Exception as e:
-                    raise BrokenMetadata("Failed to process "
-                                         "path %s: %s" % (path, e))
+                    raise BrokenMetadata(
+                        "Failed to process path %s: %s" % (path, e)
+                    ) from e
             if found:
                 results[name] = data
 
@@ -291,8 +292,9 @@ class BaseReader(metaclass=abc.ABCMeta):
             try:
                 metadata['random_seed'] = base64.b64decode(random_seed)
             except (ValueError, TypeError) as e:
-                raise BrokenMetadata("Badly formatted metadata"
-                                     " random_seed entry: %s" % e)
+                raise BrokenMetadata(
+                    "Badly formatted metadata random_seed entry: %s" % e
+                ) from e
 
         # load any files that were provided
         files = {}
@@ -304,8 +306,9 @@ class BaseReader(metaclass=abc.ABCMeta):
             try:
                 files[path] = self._read_content_path(item)
             except Exception as e:
-                raise BrokenMetadata("Failed to read provided "
-                                     "file %s: %s" % (path, e))
+                raise BrokenMetadata(
+                    "Failed to read provided file %s: %s" % (path, e)
+                ) from e
         results['files'] = files
 
         # The 'network_config' item in metadata is a content pointer
@@ -317,8 +320,9 @@ class BaseReader(metaclass=abc.ABCMeta):
                 content = self._read_content_path(net_item, decode=True)
                 results['network_config'] = content
             except IOError as e:
-                raise BrokenMetadata("Failed to read network"
-                                     " configuration: %s" % (e))
+                raise BrokenMetadata(
+                    "Failed to read network configuration: %s" % (e)
+                ) from e
 
         # To openstack, user can specify meta ('nova boot --meta=key=value')
         # and those will appear under metadata['meta'].
@@ -370,8 +374,9 @@ class ConfigDriveReader(BaseReader):
             try:
                 return util.load_json(self._path_read(path))
             except Exception as e:
-                raise BrokenMetadata("Failed to process "
-                                     "path %s: %s" % (path, e))
+                raise BrokenMetadata(
+                    "Failed to process path %s: %s" % (path, e)
+                ) from e
 
     def read_v1(self):
         """Reads a version 1 formatted location.
@@ -395,16 +400,17 @@ class ConfigDriveReader(BaseReader):
                 path = found[name]
                 try:
                     contents = self._path_read(path)
-                except IOError:
-                    raise BrokenMetadata("Failed to read: %s" % path)
+                except IOError as e:
+                    raise BrokenMetadata("Failed to read: %s" % path) from e
                 try:
                     # Disable not-callable pylint check; pylint isn't able to
                     # determine that every member of FILES_V1 has a callable in
                     # the appropriate position
                     md[key] = translator(contents)  # pylint: disable=E1102
                 except Exception as e:
-                    raise BrokenMetadata("Failed to process "
-                                         "path %s: %s" % (path, e))
+                    raise BrokenMetadata(
+                        "Failed to process path %s: %s" % (path, e)
+                    ) from e
             else:
                 md[key] = copy.deepcopy(default)
 
diff --git a/cloudinit/subp.py b/cloudinit/subp.py
index 804ef3ca..3e4efa42 100644
--- a/cloudinit/subp.py
+++ b/cloudinit/subp.py
@@ -262,7 +262,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True,
         raise ProcessExecutionError(
             cmd=args, reason=e, errno=e.errno,
             stdout="-" if decode else b"-",
-            stderr="-" if decode else b"-")
+            stderr="-" if decode else b"-"
+        ) from e
     finally:
         if devnull_fp:
             devnull_fp.close()
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index f3c0cf9c..caa88435 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -95,7 +95,7 @@ def read_file_or_url(url, **kwargs):
             code = e.errno
             if e.errno == ENOENT:
                 code = NOT_FOUND
-            raise UrlError(cause=e, code=code, headers=None, url=url)
+            raise UrlError(cause=e, code=code, headers=None, url=url) from e
         return FileResponse(file_path, contents=contents)
     else:
         return readurl(url, **kwargs)
@@ -575,8 +575,8 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
                   timestamp=None):
     try:
         import oauthlib.oauth1 as oauth1
-    except ImportError:
-        raise NotImplementedError('oauth support is not available')
+    except ImportError as e:
+        raise NotImplementedError('oauth support is not available') from e
 
     if timestamp:
         timestamp = str(timestamp)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index dd263803..cf9e349f 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -359,7 +359,7 @@ def decomp_gzip(data, quiet=True, decode=True):
         if quiet:
             return data
         else:
-            raise DecompressionError(str(e))
+            raise DecompressionError(str(e)) from e
 
 
 def extract_usergroup(ug_pair):
@@ -1363,7 +1363,7 @@ def chownbyname(fname, user=None, group=None):
         if group:
             gid = grp.getgrnam(group).gr_gid
     except KeyError as e:
-        raise OSError("Unknown user or group: %s" % (e))
+        raise OSError("Unknown user or group: %s" % (e)) from e
     chownbyid(fname, uid, gid)
 
 
@@ -2387,8 +2387,8 @@ def human2bytes(size):
 
     try:
         num = float(num)
-    except ValueError:
-        raise ValueError("'%s' is not valid input." % size_in)
+    except ValueError as e:
+        raise ValueError("'%s' is not valid input." % size_in) from e
 
     if num < 0:
         raise ValueError("'%s': cannot be negative" % size_in)
diff --git a/tests/cloud_tests/platforms/azurecloud/instance.py b/tests/cloud_tests/platforms/azurecloud/instance.py
index a136cf0d..eedbaae8 100644
--- a/tests/cloud_tests/platforms/azurecloud/instance.py
+++ b/tests/cloud_tests/platforms/azurecloud/instance.py
@@ -134,9 +134,10 @@ class AzureCloudInstance(Instance):
                                  self.vm_name, vm_params)
             LOG.debug('creating instance %s from image_id=%s', self.vm_name,
                       self.image_id)
-        except CloudError:
-            raise RuntimeError('failed creating instance:\n{}'.format(
-                traceback.format_exc()))
+        except CloudError as e:
+            raise RuntimeError(
+                'failed creating instance:\n{}'.format(traceback.format_exc())
+            ) from e
 
         if wait:
             self.instance.wait()
diff --git a/tests/cloud_tests/platforms/azurecloud/platform.py b/tests/cloud_tests/platforms/azurecloud/platform.py
index cb62a74b..a664f612 100644
--- a/tests/cloud_tests/platforms/azurecloud/platform.py
+++ b/tests/cloud_tests/platforms/azurecloud/platform.py
@@ -59,9 +59,12 @@ class AzureCloudPlatform(Platform):
             self.vnet = self._create_vnet()
             self.subnet = self._create_subnet()
             self.nic = self._create_nic()
-        except CloudError:
-            raise RuntimeError('failed creating a resource:\n{}'.format(
-                traceback.format_exc()))
+        except CloudError as e:
+            raise RuntimeError(
+                'failed creating a resource:\n{}'.format(
+                    traceback.format_exc()
+                )
+            ) from e
 
     def create_instance(self, properties, config, features,
                         image_id, user_data=None):
@@ -105,8 +108,10 @@ class AzureCloudPlatform(Platform):
             if image_id.find('__') > 0:
                 image_id = image_id.split('__')[1]
                 LOG.debug('image_id shortened to %s', image_id)
-        except KeyError:
-            raise RuntimeError('no images found for %s' % img_conf['release'])
+        except KeyError as e:
+            raise RuntimeError(
+                'no images found for %s' % img_conf['release']
+            ) from e
 
         return AzureCloudImage(self, img_conf, image_id)
 
@@ -140,9 +145,11 @@ class AzureCloudPlatform(Platform):
                 secret=azure_creds['clientSecret'],
                 tenant=azure_creds['tenantId'])
             return credentials, subscription_id
-        except KeyError:
-            raise RuntimeError('Please configure Azure service principal'
-                               ' credentials in %s' % cred_file)
+        except KeyError as e:
+            raise RuntimeError(
+                'Please configure Azure service principal'
+                ' credentials in %s' % cred_file
+            ) from e
 
     def _create_resource_group(self):
         """Create resource group"""
diff --git a/tests/cloud_tests/platforms/ec2/instance.py b/tests/cloud_tests/platforms/ec2/instance.py
index ab6037b1..d2e84047 100644
--- a/tests/cloud_tests/platforms/ec2/instance.py
+++ b/tests/cloud_tests/platforms/ec2/instance.py
@@ -49,11 +49,11 @@ class EC2Instance(Instance):
             # OutputBytes comes from platform._decode_console_output_as_bytes
             response = self.instance.console_output()
             return response['OutputBytes']
-        except KeyError:
+        except KeyError as e:
             if 'Output' in response:
                 msg = ("'OutputBytes' did not exist in console_output() but "
                        "'Output' did: %s..." % response['Output'][0:128])
-                raise util.PlatformError('console_log', msg)
+                raise util.PlatformError('console_log', msg) from e
             return ('No Console Output [%s]' % self.instance).encode()
 
     def destroy(self):
diff --git a/tests/cloud_tests/platforms/ec2/platform.py b/tests/cloud_tests/platforms/ec2/platform.py
index 7a3d0fe0..b61a2ffb 100644
--- a/tests/cloud_tests/platforms/ec2/platform.py
+++ b/tests/cloud_tests/platforms/ec2/platform.py
@@ -35,12 +35,14 @@ class EC2Platform(Platform):
             self.ec2_resource = b3session.resource('ec2')
             self.ec2_region = b3session.region_name
             self.key_name = self._upload_public_key(config)
-        except botocore.exceptions.NoRegionError:
+        except botocore.exceptions.NoRegionError as e:
             raise RuntimeError(
-                'Please configure default region in $HOME/.aws/config')
-        except botocore.exceptions.NoCredentialsError:
+                'Please configure default region in $HOME/.aws/config'
+            ) from e
+        except botocore.exceptions.NoCredentialsError as e:
             raise RuntimeError(
-                'Please configure ec2 credentials in $HOME/.aws/credentials')
+                'Please configure ec2 credentials in $HOME/.aws/credentials'
+            ) from e
 
         self.vpc = self._create_vpc()
         self.internet_gateway = self._create_internet_gateway()
@@ -125,8 +127,10 @@ class EC2Platform(Platform):
 
         try:
             image_ami = image['id']
-        except KeyError:
-            raise RuntimeError('No images found for %s!' % img_conf['release'])
+        except KeyError as e:
+            raise RuntimeError(
+                'No images found for %s!' % img_conf['release']
+            ) from e
 
         LOG.debug('found image: %s', image_ami)
         image = EC2Image(self, img_conf, image_ami)
@@ -195,7 +199,7 @@ class EC2Platform(Platform):
                 CidrBlock=self.ipv4_cidr,
                 AmazonProvidedIpv6CidrBlock=True)
         except botocore.exceptions.ClientError as e:
-            raise RuntimeError(e)
+            raise RuntimeError(e) from e
 
         vpc.wait_until_available()
         self._tag_resource(vpc)
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index b27b9848..2b973a08 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -175,7 +175,8 @@ class LXDInstance(Instance):
             raise PlatformError(
                 "console log",
                 "Console log failed [%d]: stdout=%s stderr=%s" % (
-                    e.exit_code, e.stdout, e.stderr))
+                    e.exit_code, e.stdout, e.stderr)
+            ) from e
 
     def reboot(self, wait=True):
         """Reboot instance."""
diff --git a/tests/cloud_tests/platforms/platforms.py b/tests/cloud_tests/platforms/platforms.py
index 58f65e52..ac3b6563 100644
--- a/tests/cloud_tests/platforms/platforms.py
+++ b/tests/cloud_tests/platforms/platforms.py
@@ -74,8 +74,10 @@ class Platform(object):
 
         try:
             return tmirror.json_entries[0]
-        except IndexError:
-            raise RuntimeError('no images found with filter: %s' % img_filter)
+        except IndexError as e:
+            raise RuntimeError(
+                'no images found with filter: %s' % img_filter
+            ) from e
 
 
 class FilterMirror(mirrors.BasicMirrorWriter):
diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
index e8c371ca..bb9785d3 100644
--- a/tests/cloud_tests/testcases/__init__.py
+++ b/tests/cloud_tests/testcases/__init__.py
@@ -21,8 +21,10 @@ def discover_test(test_name):
         config.name_sanitize(test_name))
     try:
         testmod = importlib.import_module(testmod_name)
-    except NameError:
-        raise ValueError('no test verifier found at: {}'.format(testmod_name))
+    except NameError as e:
+        raise ValueError(
+            'no test verifier found at: {}'.format(testmod_name)
+        ) from e
 
     found = [mod for name, mod in inspect.getmembers(testmod)
              if (inspect.isclass(mod)
diff --git a/tools/mock-meta.py b/tools/mock-meta.py
index a58e0260..9dd067b9 100755
--- a/tools/mock-meta.py
+++ b/tools/mock-meta.py
@@ -258,12 +258,14 @@ class MetaDataHandler(object):
                 try:
                     key_id = int(mybe_key)
                     key_name = key_ids[key_id]
-                except ValueError:
-                    raise WebException(hclient.BAD_REQUEST,
-                                       "%s: not an integer" % mybe_key)
-                except IndexError:
-                    raise WebException(hclient.NOT_FOUND,
-                                       "Unknown key id %r" % mybe_key)
+                except ValueError as e:
+                    raise WebException(
+                        hclient.BAD_REQUEST, "%s: not an integer" % mybe_key
+                    ) from e
+                except IndexError as e:
+                    raise WebException(
+                        hclient.NOT_FOUND, "Unknown key id %r" % mybe_key
+                    ) from e
                 # Extract the possible sub-params
                 result = traverse(nparams[1:], {
                     "openssh-key": "\n".join(avail_keys[key_name]),
diff --git a/tox.ini b/tox.ini
index f619dbf5..a92c63e0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,7 +23,7 @@ setenv =
 basepython = python3
 deps =
     # requirements
-    pylint==2.5.3
+    pylint==2.6.0
     # test-requirements because unit tests are now present in cloudinit tree
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/integration-requirements.txt
-- 
cgit v1.2.3