ec2: Allow Ec2 to run in init-local using dhclient in a sandbox.

This branch is a prerequisite for IPv6 support in AWS by allowing Ec2
datasource to query the metadata source version 2016-09-02 about whether
or not it needs to configure IPv6 on interfaces. If version 2016-09-02
is not present, fallback to the min_metadata_version of 2009-04-04. The
DataSourceEc2Local not run on FreeBSD because dhclient in doesn't
support the -sf flag allowing us to run dhclient without filesystem
side-effects.

To query AWS' metadata address @ 169.254.169.254, the instance must have
a dhcp-allocated address configured. Configuring IPv4 link-local
addresses result in timeouts from the metadata service. We introduced a
DataSourceEc2Local subclass which will perform a sandboxed dhclient
discovery which obtains an authorized IP address on eth0 and crawl
metadata about full instance network configuration.

Since ec2 IPv6 metadata is not sufficient in itself to tell us all the
ipv6 knownledge we need, it only be used as a boolean to tell us which
nics need IPv6. Cloud-init will then configure desired interfaces to
DHCPv6 versus DHCPv4.

Performance side note: Shifting the dhcp work into init-local for Ec2
actually gets us 1 second faster deployments by skipping init-network
phase of alternate datasource checks because Ec2Local is configured in
an ealier boot stage. In 3 test runs prior to this change: cloud-init
runs were 5.5 seconds, with the change we now average 4.6 seconds.

This efficiency could be even further improved if we avoiding dhcp
discovery in order to talk to the metadata service from an AWS
authorized dhcp address if there were some way to advertize the dhcp
configuration via DMI/SMBIOS or system environment variables.

Inspecting time costs of the dhclient setup/teardown in 3 live runs the
time cost for the dhcp setup round trip on AWS is:
test 1: 76 milliseconds
         dhcp discovery + metadata: 0.347 seconds
         metadata alone: 0.271 seconds
test 2: 88 milliseconds
         dhcp discovery + metadata: 0.388 seconds
         metadata alone: 0.300 seconds
test 3: 75 milliseconds
         dhcp discovery + metadata: 0.366 seconds
         metadata alone: 0.291 seconds

LP: #1709772

11 files changed, 511 insertions, 73 deletions

diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 1ff8fae0..a1b0db10 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -175,13 +175,8 @@ def is_disabled_cfg(cfg):
     return cfg.get('config') == "disabled"
 
 
-def generate_fallback_config(blacklist_drivers=None, config_driver=None):
-    """Determine which attached net dev is most likely to have a connection and
-       generate network state to run dhcp on that interface"""
-
-    if not config_driver:
-        config_driver = False
-
+def find_fallback_nic(blacklist_drivers=None):
+    """Return the name of the 'fallback' network device."""
     if not blacklist_drivers:
         blacklist_drivers = []
 
@@ -233,15 +228,24 @@ def generate_fallback_config(blacklist_drivers=None, config_driver=None):
     if DEFAULT_PRIMARY_INTERFACE in names:
         names.remove(DEFAULT_PRIMARY_INTERFACE)
         names.insert(0, DEFAULT_PRIMARY_INTERFACE)
-    target_name = None
-    target_mac = None
+
+    # pick the first that has a mac-address
     for name in names:
-        mac = read_sys_net_safe(name, 'address')
-        if mac:
-            target_name = name
-            target_mac = mac
-            break
-    if target_mac and target_name:
+        if read_sys_net_safe(name, 'address'):
+            return name
+    return None
+
+
+def generate_fallback_config(blacklist_drivers=None, config_driver=None):
+    """Determine which attached net dev is most likely to have a connection and
+       generate network state to run dhcp on that interface"""
+
+    if not config_driver:
+        config_driver = False
+
+    target_name = find_fallback_nic(blacklist_drivers=blacklist_drivers)
+    if target_name:
+        target_mac = read_sys_net_safe(target_name, 'address')
         nconf = {'config': [], 'version': 1}
         cfg = {'type': 'physical', 'name': target_name,
                'mac_address': target_mac, 'subnets': [{'type': 'dhcp'}]}
@@ -585,6 +589,7 @@ class EphemeralIPv4Network(object):
             self._bringup_router()
 
     def __exit__(self, excp_type, excp_value, excp_traceback):
+        """Teardown anything we set up."""
         for cmd in self.cleanup_cmds:
             util.subp(cmd, capture=True)
 
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
new file mode 100644
index 00000000..c7febc57
--- /dev/null
+++ b/cloudinit/net/dhcp.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# Author: Chad Smith <chad.smith@canonical.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import logging
+import os
+import re
+
+from cloudinit.net import find_fallback_nic, get_devicelist
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidDHCPLeaseFileError(Exception):
+    """Raised when parsing an empty or invalid dhcp.leases file.
+
+    Current uses are DataSourceAzure and DataSourceEc2 during ephemeral
+    boot to scrape metadata.
+    """
+    pass
+
+
+def maybe_perform_dhcp_discovery(nic=None):
+    """Perform dhcp discovery if nic valid and dhclient command exists.
+
+    If the nic is invalid or undiscoverable or dhclient command is not found,
+    skip dhcp_discovery and return an empty dict.
+
+    @param nic: Name of the network interface we want to run dhclient on.
+    @return: A dict of dhcp options from the dhclient discovery if run,
+        otherwise an empty dict is returned.
+    """
+    if nic is None:
+        nic = find_fallback_nic()
+        if nic is None:
+            LOG.debug(
+                'Skip dhcp_discovery: Unable to find fallback nic.')
+            return {}
+    elif nic not in get_devicelist():
+        LOG.debug(
+            'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic)
+        return {}
+    dhclient_path = util.which('dhclient')
+    if not dhclient_path:
+        LOG.debug('Skip dhclient configuration: No dhclient command found.')
+        return {}
+    with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir:
+        return dhcp_discovery(dhclient_path, nic, tmpdir)
+
+
+def parse_dhcp_lease_file(lease_file):
+    """Parse the given dhcp lease file for the most recent lease.
+
+    Return a dict of dhcp options as key value pairs for the most recent lease
+    block.
+
+    @raises: InvalidDHCPLeaseFileError on empty of unparseable leasefile
+        content.
+    """
+    lease_regex = re.compile(r"lease {(?P<lease>[^}]*)}\n")
+    dhcp_leases = []
+    lease_content = util.load_file(lease_file)
+    if len(lease_content) == 0:
+        raise InvalidDHCPLeaseFileError(
+            'Cannot parse empty dhcp lease file {0}'.format(lease_file))
+    for lease in lease_regex.findall(lease_content):
+        lease_options = []
+        for line in lease.split(';'):
+            # Strip newlines, double-quotes and option prefix
+            line = line.strip().replace('"', '').replace('option ', '')
+            if not line:
+                continue
+            lease_options.append(line.split(' ', 1))
+        dhcp_leases.append(dict(lease_options))
+    if not dhcp_leases:
+        raise InvalidDHCPLeaseFileError(
+            'Cannot parse dhcp lease file {0}. No leases found'.format(
+                lease_file))
+    return dhcp_leases
+
+
+def dhcp_discovery(dhclient_cmd_path, interface, cleandir):
+    """Run dhclient on the interface without scripts or filesystem artifacts.
+
+    @param dhclient_cmd_path: Full path to the dhclient used.
+    @param interface: Name of the network inteface on which to dhclient.
+    @param cleandir: The directory from which to run dhclient as well as store
+        dhcp leases.
+
+    @return: A dict of dhcp options parsed from the dhcp.leases file or empty
+        dict.
+    """
+    LOG.debug('Performing a dhcp discovery on %s', interface)
+
+    # XXX We copy dhclient out of /sbin/dhclient to avoid dealing with strict
+    # app armor profiles which disallow running dhclient -sf <our-script-file>.
+    # We want to avoid running /sbin/dhclient-script because of side-effects in
+    # /etc/resolv.conf any any other vendor specific scripts in
+    # /etc/dhcp/dhclient*hooks.d.
+    sandbox_dhclient_cmd = os.path.join(cleandir, 'dhclient')
+    util.copy(dhclient_cmd_path, sandbox_dhclient_cmd)
+    pid_file = os.path.join(cleandir, 'dhclient.pid')
+    lease_file = os.path.join(cleandir, 'dhcp.leases')
+
+    # ISC dhclient needs the interface up to send initial discovery packets.
+    # Generally dhclient relies on dhclient-script PREINIT action to bring the
+    # link up before attempting discovery. Since we are using -sf /bin/true,
+    # we need to do that "link up" ourselves first.
+    util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True)
+    cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file,
+           '-pf', pid_file, interface, '-sf', '/bin/true']
+    util.subp(cmd, capture=True)
+    return parse_dhcp_lease_file(lease_file)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
new file mode 100644
index 00000000..47d8d461
--- /dev/null
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -0,0 +1,144 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import mock
+import os
+from textwrap import dedent
+
+from cloudinit.net.dhcp import (
+    InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
+    parse_dhcp_lease_file, dhcp_discovery)
+from cloudinit.util import ensure_file, write_file
+from tests.unittests.helpers import CiTestCase
+
+
+class TestParseDHCPLeasesFile(CiTestCase):
+
+    def test_parse_empty_lease_file_errors(self):
+        """parse_dhcp_lease_file errors when file content is empty."""
+        empty_file = self.tmp_path('leases')
+        ensure_file(empty_file)
+        with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager:
+            parse_dhcp_lease_file(empty_file)
+        error = context_manager.exception
+        self.assertIn('Cannot parse empty dhcp lease file', str(error))
+
+    def test_parse_malformed_lease_file_content_errors(self):
+        """parse_dhcp_lease_file errors when file content isn't dhcp leases."""
+        non_lease_file = self.tmp_path('leases')
+        write_file(non_lease_file, 'hi mom.')
+        with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager:
+            parse_dhcp_lease_file(non_lease_file)
+        error = context_manager.exception
+        self.assertIn('Cannot parse dhcp lease file', str(error))
+
+    def test_parse_multiple_leases(self):
+        """parse_dhcp_lease_file returns a list of all leases within."""
+        lease_file = self.tmp_path('leases')
+        content = dedent("""
+            lease {
+              interface "wlp3s0";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+              renew 4 2017/07/27 18:02:30;
+              expire 5 2017/07/28 07:08:15;
+            }
+            lease {
+              interface "wlp3s0";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+            }
+        """)
+        expected = [
+            {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+             'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1',
+             'renew': '4 2017/07/27 18:02:30',
+             'expire': '5 2017/07/28 07:08:15'},
+            {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+             'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}]
+        write_file(lease_file, content)
+        self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+
+
+class TestDHCPDiscoveryClean(CiTestCase):
+    with_logs = True
+
+    @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+    def test_no_fallback_nic_found(self, m_fallback_nic):
+        """Log and do nothing when nic is absent and no fallback is found."""
+        m_fallback_nic.return_value = None  # No fallback nic found
+        self.assertEqual({}, maybe_perform_dhcp_discovery())
+        self.assertIn(
+            'Skip dhcp_discovery: Unable to find fallback nic.',
+            self.logs.getvalue())
+
+    def test_provided_nic_does_not_exist(self):
+        """When the provided nic doesn't exist, log a message and no-op."""
+        self.assertEqual({}, maybe_perform_dhcp_discovery('idontexist'))
+        self.assertIn(
+            'Skip dhcp_discovery: nic idontexist not found in get_devicelist.',
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+    def test_absent_dhclient_command(self, m_fallback, m_which):
+        """When dhclient doesn't exist in the OS, log the issue and no-op."""
+        m_fallback.return_value = 'eth9'
+        m_which.return_value = None  # dhclient isn't found
+        self.assertEqual({}, maybe_perform_dhcp_discovery())
+        self.assertIn(
+            'Skip dhclient configuration: No dhclient command found.',
+            self.logs.getvalue())
+
+    @mock.patch('cloudinit.net.dhcp.dhcp_discovery')
+    @mock.patch('cloudinit.net.dhcp.util.which')
+    @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+    def test_dhclient_run_with_tmpdir(self, m_fallback, m_which, m_dhcp):
+        """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery."""
+        m_fallback.return_value = 'eth9'
+        m_which.return_value = '/sbin/dhclient'
+        m_dhcp.return_value = {'address': '192.168.2.2'}
+        self.assertEqual(
+            {'address': '192.168.2.2'}, maybe_perform_dhcp_discovery())
+        m_dhcp.assert_called_once()
+        call = m_dhcp.call_args_list[0]
+        self.assertEqual('/sbin/dhclient', call[0][0])
+        self.assertEqual('eth9', call[0][1])
+        self.assertIn('/tmp/cloud-init-dhcp-', call[0][2])
+
+    @mock.patch('cloudinit.net.dhcp.util.subp')
+    def test_dhcp_discovery_run_in_sandbox(self, m_subp):
+        """dhcp_discovery brings up the interface and runs dhclient.
+
+        It also returns the parsed dhcp.leases file generated in the sandbox.
+        """
+        tmpdir = self.tmp_dir()
+        dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
+        script_content = '#!/bin/bash\necho fake-dhclient'
+        write_file(dhclient_script, script_content, mode=0o755)
+        lease_content = dedent("""
+            lease {
+              interface "eth9";
+              fixed-address 192.168.2.74;
+              option subnet-mask 255.255.255.0;
+              option routers 192.168.2.1;
+            }
+        """)
+        lease_file = os.path.join(tmpdir, 'dhcp.leases')
+        write_file(lease_file, lease_content)
+        self.assertItemsEqual(
+            [{'interface': 'eth9', 'fixed-address': '192.168.2.74',
+              'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}],
+            dhcp_discovery(dhclient_script, 'eth9', tmpdir))
+        # dhclient script got copied
+        with open(os.path.join(tmpdir, 'dhclient')) as stream:
+            self.assertEqual(script_content, stream.read())
+        # Interface was brought up before dhclient called from sandbox
+        m_subp.assert_has_calls([
+            mock.call(
+                ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True),
+            mock.call(
+                [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf',
+                 lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'),
+                 'eth9', '-sf', '/bin/true'], capture=True)])
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 272a6ebd..cc052a7d 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -414,7 +414,7 @@ class TestEphemeralIPV4Network(CiTestCase):
             self.assertIn('Cannot init network on', str(error))
             self.assertEqual(0, m_subp.call_count)
 
-    def test_ephemeral_ipv4_network_errors_invalid_mask(self, m_subp):
+    def test_ephemeral_ipv4_network_errors_invalid_mask_prefix(self, m_subp):
         """Raise an error when prefix_or_mask is not a netmask or prefix."""
         params = {
             'interface': 'eth0', 'ip': '192.168.2.2',
diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 380e27cb..43a7e42c 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -6,17 +6,20 @@ from cloudinit import sources
 from cloudinit.sources import DataSourceEc2 as EC2
 from cloudinit import util
 
-DEF_MD_VERSION = "2016-01-01"
 ALIYUN_PRODUCT = "Alibaba Cloud ECS"
 
 
 class DataSourceAliYun(EC2.DataSourceEc2):
-    metadata_urls = ["http://100.100.100.200"]
+
+    metadata_urls = ['http://100.100.100.200']
+
+    # The minimum supported metadata_version from the ec2 metadata apis
+    min_metadata_version = '2016-01-01'
+    extended_metadata_versions = []
 
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)
         self.seed_dir = os.path.join(paths.seed_dir, "AliYun")
-        self.api_ver = DEF_MD_VERSION
 
     def get_hostname(self, fqdn=False, _resolve_ip=False):
         return self.metadata.get('hostname', 'localhost.localdomain')
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 4ec9592f..8e5f8ee4 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -13,6 +13,8 @@ import time
 
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
+from cloudinit import net
+from cloudinit.net import dhcp
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
@@ -20,8 +22,7 @@ from cloudinit import warnings
 
 LOG = logging.getLogger(__name__)
 
-# Which version we are requesting of the ec2 metadata apis
-DEF_MD_VERSION = '2009-04-04'
+SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND])
 
 STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
 STRICT_ID_DEFAULT = "warn"
@@ -41,17 +42,28 @@ class Platforms(object):
 
 
 class DataSourceEc2(sources.DataSource):
+
     # Default metadata urls that will be used if none are provided
     # They will be checked for 'resolveability' and some of the
     # following may be discarded if they do not resolve
     metadata_urls = ["http://169.254.169.254", "http://instance-data.:8773"]
+
+    # The minimum supported metadata_version from the ec2 metadata apis
+    min_metadata_version = '2009-04-04'
+
+    # Priority ordered list of additional metadata versions which will be tried
+    # for extended metadata content. IPv6 support comes in 2016-09-02
+    extended_metadata_versions = ['2016-09-02']
+
     _cloud_platform = None
 
+    # Whether we want to get network configuration from the metadata service.
+    get_network_metadata = False
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.metadata_address = None
         self.seed_dir = os.path.join(paths.seed_dir, "ec2")
-        self.api_ver = DEF_MD_VERSION
 
     def get_data(self):
         seed_ret = {}
@@ -73,21 +85,27 @@ class DataSourceEc2(sources.DataSource):
         elif self.cloud_platform == Platforms.NO_EC2_METADATA:
             return False
 
-        try:
-            if not self.wait_for_metadata_service():
+        if self.get_network_metadata:  # Setup networking in init-local stage.
+            if util.is_FreeBSD():
+                LOG.debug("FreeBSD doesn't support running dhclient with -sf")
                 return False
-            start_time = time.time()
-            self.userdata_raw = \
-                ec2.get_instance_userdata(self.api_ver, self.metadata_address)
-            self.metadata = ec2.get_instance_metadata(self.api_ver,
-                                                      self.metadata_address)
-            LOG.debug("Crawl of metadata service took %.3f seconds",
-                      time.time() - start_time)
-            return True
-        except Exception:
-            util.logexc(LOG, "Failed reading from metadata address %s",
-                        self.metadata_address)
-            return False
+            dhcp_leases = dhcp.maybe_perform_dhcp_discovery()
+            if not dhcp_leases:
+                # DataSourceEc2Local failed in init-local stage. DataSourceEc2
+                # will still run in init-network stage.
+                return False
+            dhcp_opts = dhcp_leases[-1]
+            net_params = {'interface': dhcp_opts.get('interface'),
+                          'ip': dhcp_opts.get('fixed-address'),
+                          'prefix_or_mask': dhcp_opts.get('subnet-mask'),
+                          'broadcast': dhcp_opts.get('broadcast-address'),
+                          'router': dhcp_opts.get('routers')}
+            with net.EphemeralIPv4Network(**net_params):
+                return util.log_time(
+                    logfunc=LOG.debug, msg='Crawl of metadata service',
+                    func=self._crawl_metadata)
+        else:
+            return self._crawl_metadata()
 
     @property
     def launch_index(self):
@@ -95,6 +113,32 @@ class DataSourceEc2(sources.DataSource):
             return None
         return self.metadata.get('ami-launch-index')
 
+    def get_metadata_api_version(self):
+        """Get the best supported api version from the metadata service.
+
+        Loop through all extended support metadata versions in order and
+        return the most-fully featured metadata api version discovered.
+
+        If extended_metadata_versions aren't present, return the datasource's
+        min_metadata_version.
+        """
+        # Assumes metadata service is already up
+        for api_ver in self.extended_metadata_versions:
+            url = '{0}/{1}/meta-data/instance-id'.format(
+                self.metadata_address, api_ver)
+            try:
+                resp = uhelp.readurl(url=url)
+            except uhelp.UrlError as e:
+                LOG.debug('url %s raised exception %s', url, e)
+            else:
+                if resp.code == 200:
+                    LOG.debug('Found preferred metadata version %s', api_ver)
+                    return api_ver
+                elif resp.code == 404:
+                    msg = 'Metadata api version %s not present. Headers: %s'
+                    LOG.debug(msg, api_ver, resp.headers)
+        return self.min_metadata_version
+
     def get_instance_id(self):
         return self.metadata['instance-id']
 
@@ -138,21 +182,22 @@ class DataSourceEc2(sources.DataSource):
         urls = []
         url2base = {}
         for url in mdurls:
-            cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
+            cur = '{0}/{1}/meta-data/instance-id'.format(
+                url, self.min_metadata_version)
             urls.append(cur)
             url2base[cur] = url
 
         start_time = time.time()
-        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                 timeout=timeout, status_cb=LOG.warn)
+        url = uhelp.wait_for_url(
+            urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn)
 
         if url:
-            LOG.debug("Using metadata source: '%s'", url2base[url])
+            self.metadata_address = url2base[url]
+            LOG.debug("Using metadata source: '%s'", self.metadata_address)
         else:
             LOG.critical("Giving up on md from %s after %s seconds",
                          urls, int(time.time() - start_time))
 
-        self.metadata_address = url2base.get(url)
         return bool(url)
 
     def device_name_to_device(self, name):
@@ -234,6 +279,37 @@ class DataSourceEc2(sources.DataSource):
                 util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
                 cfg)
 
+    def _crawl_metadata(self):
+        """Crawl metadata service when available.
+
+        @returns: True on success, False otherwise.
+        """
+        if not self.wait_for_metadata_service():
+            return False
+        api_version = self.get_metadata_api_version()
+        try:
+            self.userdata_raw = ec2.get_instance_userdata(
+                api_version, self.metadata_address)
+            self.metadata = ec2.get_instance_metadata(
+                api_version, self.metadata_address)
+        except Exception:
+            util.logexc(
+                LOG, "Failed reading from metadata address %s",
+                self.metadata_address)
+            return False
+        return True
+
+
+class DataSourceEc2Local(DataSourceEc2):
+    """Datasource run at init-local which sets up network to query metadata.
+
+    In init-local, no network is available. This subclass sets up minimal
+    networking with dhclient on a viable nic so that it can talk to the
+    metadata service. If the metadata service provides network configuration
+    then render the network configuration for that instance based on metadata.
+    """
+    get_network_metadata = True  # Get metadata network config if present
+
 
 def read_strict_mode(cfgval, default):
     try:
@@ -349,6 +425,7 @@ def _collect_platform_data():
 
 # Used to match classes to dependencies
 datasources = [
+    (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)),  # Run at init-local
     (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 08c5c469..bf1dc5df 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -278,7 +278,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
         return root
 
 
-class HttprettyTestCase(TestCase):
+class HttprettyTestCase(CiTestCase):
     # necessary as http_proxy gets in the way of httpretty
     # https://github.com/gabrielfalcao/HTTPretty/issues/122
     def setUp(self):
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 990bff2c..996560e4 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         self.ds = ay.DataSourceAliYun(cfg, distro, paths)
         self.metadata_address = self.ds.metadata_urls[0]
-        self.api_ver = self.ds.api_ver
 
     @property
     def default_metadata(self):
@@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
 
     @property
     def metadata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'meta-data') + '/'
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'meta-data') + '/'
 
     @property
     def userdata_url(self):
-        return os.path.join(self.metadata_address,
-                            self.api_ver, 'user-data')
+        return os.path.join(
+            self.metadata_address,
+            self.ds.min_metadata_version, 'user-data')
 
     def regist_default_server(self):
         register_mock_metaserver(self.metadata_url, self.default_metadata)
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 413e87ac..4802f105 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -35,6 +35,7 @@ DEFAULT_LOCAL = [
     OpenNebula.DataSourceOpenNebula,
     OVF.DataSourceOVF,
     SmartOS.DataSourceSmartOS,
+    Ec2.DataSourceEc2Local,
 ]
 
 DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 12230ae2..33d02619 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -8,35 +8,67 @@ from cloudinit import helpers
 from cloudinit.sources import DataSourceEc2 as ec2
 
 
-# collected from api version 2009-04-04/ with
+# collected from api version 2016-09-02/ with
 # python3 -c 'import json
 # from cloudinit.ec2_utils import get_instance_metadata as gm
-# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))'
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
 DEFAULT_METADATA = {
-    "ami-id": "ami-80861296",
+    "ami-id": "ami-8b92b4ee",
     "ami-launch-index": "0",
     "ami-manifest-path": "(unknown)",
     "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
-    "hostname": "ip-10-0-0-149",
+    "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
     "instance-action": "none",
-    "instance-id": "i-0052913950685138c",
-    "instance-type": "t2.micro",
-    "local-hostname": "ip-10-0-0-149",
-    "local-ipv4": "10.0.0.149",
-    "placement": {"availability-zone": "us-east-1b"},
+    "instance-id": "i-0a33f80f09c96477f",
+    "instance-type": "t2.small",
+    "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+    "local-ipv4": "172.3.3.15",
+    "mac": "06:17:04:d7:26:09",
+    "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+    "network": {
+        "interfaces": {
+            "macs": {
+                "06:17:04:d7:26:09": {
+                    "device-number": "0",
+                    "interface-id": "eni-e44ef49e",
+                    "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+                    "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+                    "local-hostname": ("ip-172-3-3-15.us-east-2."
+                                       "compute.internal"),
+                    "local-ipv4s": "172.3.3.15",
+                    "mac": "06:17:04:d7:26:09",
+                    "owner-id": "950047163771",
+                    "public-hostname": ("ec2-13-59-77-202.us-east-2."
+                                        "compute.amazonaws.com"),
+                    "public-ipv4s": "13.59.77.202",
+                    "security-group-ids": "sg-5a61d333",
+                    "security-groups": "wide-open",
+                    "subnet-id": "subnet-20b8565b",
+                    "subnet-ipv4-cidr-block": "172.31.16.0/20",
+                    "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+                    "vpc-id": "vpc-87e72bee",
+                    "vpc-ipv4-cidr-block": "172.31.0.0/16",
+                    "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+                    "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+                }
+            }
+        }
+    },
+    "placement": {"availability-zone": "us-east-2b"},
     "profile": "default-hvm",
-    "public-hostname": "",
-    "public-ipv4": "107.23.188.247",
+    "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+    "public-ipv4": "13.59.77.202",
     "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
-    "reservation-id": "r-00a2c173fb5782a08",
-    "security-groups": "wide-open"
+    "reservation-id": "r-01efbc9996bac1bd6",
+    "security-groups": "my-wide-open",
+    "services": {"domain": "amazonaws.com", "partition": "aws"}
 }
 
 
 def _register_ssh_keys(rfunc, base_url, keys_data):
     """handle ssh key inconsistencies.
 
-    public-keys in the ec2 metadata is inconsistently formatted compared
+    public-keys in the ec2 metadata is inconsistently formated compared
     to other entries.
     Given keys_data of {name1: pubkey1, name2: pubkey2}
 
@@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data):
 
 
 class TestEc2(test_helpers.HttprettyTestCase):
+    with_logs = True
+
     valid_platform_data = {
         'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
         'uuid_source': 'dmi',
@@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase):
 
     def setUp(self):
         super(TestEc2, self).setUp()
-        self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0]
-        self.api_ver = '2009-04-04'
+        self.datasource = ec2.DataSourceEc2
+        self.metadata_addr = self.datasource.metadata_urls[0]
 
     @property
     def metadata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', ''])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'meta-data', ''])
 
     @property
     def userdata_url(self):
-        return '/'.join([self.metadata_addr, self.api_ver, 'user-data'])
+        return '/'.join([
+            self.metadata_addr,
+            self.datasource.min_metadata_version, 'user-data'])
 
     def _patch_add_cleanup(self, mpath, *args, **kwargs):
         p = mock.patch(mpath, *args, **kwargs)
@@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
         paths = helpers.Paths({})
         if sys_cfg is None:
             sys_cfg = {}
-        ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths)
+        ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
         if platform_data is not None:
             self._patch_add_cleanup(
                 "cloudinit.sources.DataSourceEc2._collect_platform_data",
@@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase):
         return ds
 
     @httpretty.activate
-    def test_valid_platform_with_strict_true(self):
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    def test_valid_platform_with_strict_true(self, m_dhcp):
         """Valid platform data should return true with strict_id true."""
         ds = self._setup_ds(
             platform_data=self.valid_platform_data,
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+        self.assertEqual(0, m_dhcp.call_count)
 
     @httpretty.activate
     def test_valid_platform_with_strict_false(self):
@@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_true(self):
@@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(False, ret)
+        self.assertFalse(ret)
 
     @httpretty.activate
     def test_unknown_platform_with_strict_false(self):
@@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase):
             sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
             md=DEFAULT_METADATA)
         ret = ds.get_data()
-        self.assertEqual(True, ret)
+        self.assertTrue(ret)
+
+    @httpretty.activate
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+        """DataSourceEc2Local returns False on BSD.
+
+        FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+        """
+        m_is_freebsd.return_value = True
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertFalse(ret)
+        self.assertIn(
+            "FreeBSD doesn't support running dhclient with -sf",
+            self.logs.getvalue())
+
+    @httpretty.activate
+    @mock.patch('cloudinit.net.EphemeralIPv4Network')
+    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+    def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net):
+        """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+        DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+        Then the metadata services is crawled for more network config info.
+        When the platform data is valid, return True.
+        """
+        m_is_bsd.return_value = False
+        m_dhcp.return_value = [{
+            'interface': 'eth9', 'fixed-address': '192.168.2.9',
+            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+            'broadcast-address': '192.168.2.255'}]
+        self.datasource = ec2.DataSourceEc2Local
+        ds = self._setup_ds(
+            platform_data=self.valid_platform_data,
+            sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+            md=DEFAULT_METADATA)
+        ret = ds.get_data()
+        self.assertTrue(ret)
+        m_dhcp.assert_called_once_with()
+        m_net.assert_called_once_with(
+            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+            prefix_or_mask='255.255.255.0', router='192.168.2.1')
+        self.assertIn('Crawl of metadata service took', self.logs.getvalue())
 
 
 # vi: ts=4 expandtab
diff --git a/tox.ini b/tox.ini
index ef768847..1e7ca2d3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,10 +21,10 @@ setenv =
     LC_ALL = en_US.utf-8
 
 [testenv:pylint]
-deps = 
+deps =
     # requirements
     pylint==1.7.1
-    # test-requirements because unit tests are now present in cloudinit tree 
+    # test-requirements because unit tests are now present in cloudinit tree
     -r{toxinidir}/test-requirements.txt
 commands = {envpython} -m pylint {posargs:cloudinit}