summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--cloudinit/net/__init__.py35
-rw-r--r--cloudinit/net/dhcp.py119
-rw-r--r--cloudinit/net/tests/test_dhcp.py144
-rw-r--r--cloudinit/net/tests/test_init.py2
-rw-r--r--cloudinit/sources/DataSourceAliYun.py9
-rw-r--r--cloudinit/sources/DataSourceEc2.py121
-rw-r--r--tests/unittests/helpers.py2
-rw-r--r--tests/unittests/test_datasource/test_aliyun.py11
-rw-r--r--tests/unittests/test_datasource/test_common.py1
-rw-r--r--tests/unittests/test_datasource/test_ec2.py136
-rw-r--r--tox.ini4
11 files changed, 511 insertions, 73 deletions
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 1ff8fae0..a1b0db10 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -175,13 +175,8 @@ def is_disabled_cfg(cfg):
return cfg.get('config') == "disabled"
-def generate_fallback_config(blacklist_drivers=None, config_driver=None):
- """Determine which attached net dev is most likely to have a connection and
- generate network state to run dhcp on that interface"""
-
- if not config_driver:
- config_driver = False
-
+def find_fallback_nic(blacklist_drivers=None):
+ """Return the name of the 'fallback' network device."""
if not blacklist_drivers:
blacklist_drivers = []
@@ -233,15 +228,24 @@ def generate_fallback_config(blacklist_drivers=None, config_driver=None):
if DEFAULT_PRIMARY_INTERFACE in names:
names.remove(DEFAULT_PRIMARY_INTERFACE)
names.insert(0, DEFAULT_PRIMARY_INTERFACE)
- target_name = None
- target_mac = None
+
+ # pick the first that has a mac-address
for name in names:
- mac = read_sys_net_safe(name, 'address')
- if mac:
- target_name = name
- target_mac = mac
- break
- if target_mac and target_name:
+ if read_sys_net_safe(name, 'address'):
+ return name
+ return None
+
+
+def generate_fallback_config(blacklist_drivers=None, config_driver=None):
+ """Determine which attached net dev is most likely to have a connection and
+ generate network state to run dhcp on that interface"""
+
+ if not config_driver:
+ config_driver = False
+
+ target_name = find_fallback_nic(blacklist_drivers=blacklist_drivers)
+ if target_name:
+ target_mac = read_sys_net_safe(target_name, 'address')
nconf = {'config': [], 'version': 1}
cfg = {'type': 'physical', 'name': target_name,
'mac_address': target_mac, 'subnets': [{'type': 'dhcp'}]}
@@ -585,6 +589,7 @@ class EphemeralIPv4Network(object):
self._bringup_router()
def __exit__(self, excp_type, excp_value, excp_traceback):
+ """Teardown anything we set up."""
for cmd in self.cleanup_cmds:
util.subp(cmd, capture=True)
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
new file mode 100644
index 00000000..c7febc57
--- /dev/null
+++ b/cloudinit/net/dhcp.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2017 Canonical Ltd.
+#
+# Author: Chad Smith <chad.smith@canonical.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import logging
+import os
+import re
+
+from cloudinit.net import find_fallback_nic, get_devicelist
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class InvalidDHCPLeaseFileError(Exception):
+ """Raised when parsing an empty or invalid dhcp.leases file.
+
+ Current uses are DataSourceAzure and DataSourceEc2 during ephemeral
+ boot to scrape metadata.
+ """
+ pass
+
+
+def maybe_perform_dhcp_discovery(nic=None):
+ """Perform dhcp discovery if nic valid and dhclient command exists.
+
+ If the nic is invalid or undiscoverable or dhclient command is not found,
+ skip dhcp_discovery and return an empty dict.
+
+ @param nic: Name of the network interface we want to run dhclient on.
+ @return: A dict of dhcp options from the dhclient discovery if run,
+ otherwise an empty dict is returned.
+ """
+ if nic is None:
+ nic = find_fallback_nic()
+ if nic is None:
+ LOG.debug(
+ 'Skip dhcp_discovery: Unable to find fallback nic.')
+ return {}
+ elif nic not in get_devicelist():
+ LOG.debug(
+ 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic)
+ return {}
+ dhclient_path = util.which('dhclient')
+ if not dhclient_path:
+ LOG.debug('Skip dhclient configuration: No dhclient command found.')
+ return {}
+ with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir:
+ return dhcp_discovery(dhclient_path, nic, tmpdir)
+
+
+def parse_dhcp_lease_file(lease_file):
+ """Parse the given dhcp lease file for the most recent lease.
+
+ Return a dict of dhcp options as key value pairs for the most recent lease
+ block.
+
+ @raises: InvalidDHCPLeaseFileError on empty of unparseable leasefile
+ content.
+ """
+ lease_regex = re.compile(r"lease {(?P<lease>[^}]*)}\n")
+ dhcp_leases = []
+ lease_content = util.load_file(lease_file)
+ if len(lease_content) == 0:
+ raise InvalidDHCPLeaseFileError(
+ 'Cannot parse empty dhcp lease file {0}'.format(lease_file))
+ for lease in lease_regex.findall(lease_content):
+ lease_options = []
+ for line in lease.split(';'):
+ # Strip newlines, double-quotes and option prefix
+ line = line.strip().replace('"', '').replace('option ', '')
+ if not line:
+ continue
+ lease_options.append(line.split(' ', 1))
+ dhcp_leases.append(dict(lease_options))
+ if not dhcp_leases:
+ raise InvalidDHCPLeaseFileError(
+ 'Cannot parse dhcp lease file {0}. No leases found'.format(
+ lease_file))
+ return dhcp_leases
+
+
+def dhcp_discovery(dhclient_cmd_path, interface, cleandir):
+ """Run dhclient on the interface without scripts or filesystem artifacts.
+
+ @param dhclient_cmd_path: Full path to the dhclient used.
+ @param interface: Name of the network inteface on which to dhclient.
+ @param cleandir: The directory from which to run dhclient as well as store
+ dhcp leases.
+
+ @return: A dict of dhcp options parsed from the dhcp.leases file or empty
+ dict.
+ """
+ LOG.debug('Performing a dhcp discovery on %s', interface)
+
+ # XXX We copy dhclient out of /sbin/dhclient to avoid dealing with strict
+ # app armor profiles which disallow running dhclient -sf <our-script-file>.
+ # We want to avoid running /sbin/dhclient-script because of side-effects in
+ # /etc/resolv.conf any any other vendor specific scripts in
+ # /etc/dhcp/dhclient*hooks.d.
+ sandbox_dhclient_cmd = os.path.join(cleandir, 'dhclient')
+ util.copy(dhclient_cmd_path, sandbox_dhclient_cmd)
+ pid_file = os.path.join(cleandir, 'dhclient.pid')
+ lease_file = os.path.join(cleandir, 'dhcp.leases')
+
+ # ISC dhclient needs the interface up to send initial discovery packets.
+ # Generally dhclient relies on dhclient-script PREINIT action to bring the
+ # link up before attempting discovery. Since we are using -sf /bin/true,
+ # we need to do that "link up" ourselves first.
+ util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True)
+ cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file,
+ '-pf', pid_file, interface, '-sf', '/bin/true']
+ util.subp(cmd, capture=True)
+ return parse_dhcp_lease_file(lease_file)
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
new file mode 100644
index 00000000..47d8d461
--- /dev/null
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -0,0 +1,144 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import mock
+import os
+from textwrap import dedent
+
+from cloudinit.net.dhcp import (
+ InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery,
+ parse_dhcp_lease_file, dhcp_discovery)
+from cloudinit.util import ensure_file, write_file
+from tests.unittests.helpers import CiTestCase
+
+
+class TestParseDHCPLeasesFile(CiTestCase):
+
+ def test_parse_empty_lease_file_errors(self):
+ """parse_dhcp_lease_file errors when file content is empty."""
+ empty_file = self.tmp_path('leases')
+ ensure_file(empty_file)
+ with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager:
+ parse_dhcp_lease_file(empty_file)
+ error = context_manager.exception
+ self.assertIn('Cannot parse empty dhcp lease file', str(error))
+
+ def test_parse_malformed_lease_file_content_errors(self):
+ """parse_dhcp_lease_file errors when file content isn't dhcp leases."""
+ non_lease_file = self.tmp_path('leases')
+ write_file(non_lease_file, 'hi mom.')
+ with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager:
+ parse_dhcp_lease_file(non_lease_file)
+ error = context_manager.exception
+ self.assertIn('Cannot parse dhcp lease file', str(error))
+
+ def test_parse_multiple_leases(self):
+ """parse_dhcp_lease_file returns a list of all leases within."""
+ lease_file = self.tmp_path('leases')
+ content = dedent("""
+ lease {
+ interface "wlp3s0";
+ fixed-address 192.168.2.74;
+ option subnet-mask 255.255.255.0;
+ option routers 192.168.2.1;
+ renew 4 2017/07/27 18:02:30;
+ expire 5 2017/07/28 07:08:15;
+ }
+ lease {
+ interface "wlp3s0";
+ fixed-address 192.168.2.74;
+ option subnet-mask 255.255.255.0;
+ option routers 192.168.2.1;
+ }
+ """)
+ expected = [
+ {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+ 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1',
+ 'renew': '4 2017/07/27 18:02:30',
+ 'expire': '5 2017/07/28 07:08:15'},
+ {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74',
+ 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}]
+ write_file(lease_file, content)
+ self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file))
+
+
+class TestDHCPDiscoveryClean(CiTestCase):
+ with_logs = True
+
+ @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+ def test_no_fallback_nic_found(self, m_fallback_nic):
+ """Log and do nothing when nic is absent and no fallback is found."""
+ m_fallback_nic.return_value = None # No fallback nic found
+ self.assertEqual({}, maybe_perform_dhcp_discovery())
+ self.assertIn(
+ 'Skip dhcp_discovery: Unable to find fallback nic.',
+ self.logs.getvalue())
+
+ def test_provided_nic_does_not_exist(self):
+ """When the provided nic doesn't exist, log a message and no-op."""
+ self.assertEqual({}, maybe_perform_dhcp_discovery('idontexist'))
+ self.assertIn(
+ 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.',
+ self.logs.getvalue())
+
+ @mock.patch('cloudinit.net.dhcp.util.which')
+ @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+ def test_absent_dhclient_command(self, m_fallback, m_which):
+ """When dhclient doesn't exist in the OS, log the issue and no-op."""
+ m_fallback.return_value = 'eth9'
+ m_which.return_value = None # dhclient isn't found
+ self.assertEqual({}, maybe_perform_dhcp_discovery())
+ self.assertIn(
+ 'Skip dhclient configuration: No dhclient command found.',
+ self.logs.getvalue())
+
+ @mock.patch('cloudinit.net.dhcp.dhcp_discovery')
+ @mock.patch('cloudinit.net.dhcp.util.which')
+ @mock.patch('cloudinit.net.dhcp.find_fallback_nic')
+ def test_dhclient_run_with_tmpdir(self, m_fallback, m_which, m_dhcp):
+ """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery."""
+ m_fallback.return_value = 'eth9'
+ m_which.return_value = '/sbin/dhclient'
+ m_dhcp.return_value = {'address': '192.168.2.2'}
+ self.assertEqual(
+ {'address': '192.168.2.2'}, maybe_perform_dhcp_discovery())
+ m_dhcp.assert_called_once()
+ call = m_dhcp.call_args_list[0]
+ self.assertEqual('/sbin/dhclient', call[0][0])
+ self.assertEqual('eth9', call[0][1])
+ self.assertIn('/tmp/cloud-init-dhcp-', call[0][2])
+
+ @mock.patch('cloudinit.net.dhcp.util.subp')
+ def test_dhcp_discovery_run_in_sandbox(self, m_subp):
+ """dhcp_discovery brings up the interface and runs dhclient.
+
+ It also returns the parsed dhcp.leases file generated in the sandbox.
+ """
+ tmpdir = self.tmp_dir()
+ dhclient_script = os.path.join(tmpdir, 'dhclient.orig')
+ script_content = '#!/bin/bash\necho fake-dhclient'
+ write_file(dhclient_script, script_content, mode=0o755)
+ lease_content = dedent("""
+ lease {
+ interface "eth9";
+ fixed-address 192.168.2.74;
+ option subnet-mask 255.255.255.0;
+ option routers 192.168.2.1;
+ }
+ """)
+ lease_file = os.path.join(tmpdir, 'dhcp.leases')
+ write_file(lease_file, lease_content)
+ self.assertItemsEqual(
+ [{'interface': 'eth9', 'fixed-address': '192.168.2.74',
+ 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}],
+ dhcp_discovery(dhclient_script, 'eth9', tmpdir))
+ # dhclient script got copied
+ with open(os.path.join(tmpdir, 'dhclient')) as stream:
+ self.assertEqual(script_content, stream.read())
+ # Interface was brought up before dhclient called from sandbox
+ m_subp.assert_has_calls([
+ mock.call(
+ ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True),
+ mock.call(
+ [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf',
+ lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'),
+ 'eth9', '-sf', '/bin/true'], capture=True)])
diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py
index 272a6ebd..cc052a7d 100644
--- a/cloudinit/net/tests/test_init.py
+++ b/cloudinit/net/tests/test_init.py
@@ -414,7 +414,7 @@ class TestEphemeralIPV4Network(CiTestCase):
self.assertIn('Cannot init network on', str(error))
self.assertEqual(0, m_subp.call_count)
- def test_ephemeral_ipv4_network_errors_invalid_mask(self, m_subp):
+ def test_ephemeral_ipv4_network_errors_invalid_mask_prefix(self, m_subp):
"""Raise an error when prefix_or_mask is not a netmask or prefix."""
params = {
'interface': 'eth0', 'ip': '192.168.2.2',
diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 380e27cb..43a7e42c 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -6,17 +6,20 @@ from cloudinit import sources
from cloudinit.sources import DataSourceEc2 as EC2
from cloudinit import util
-DEF_MD_VERSION = "2016-01-01"
ALIYUN_PRODUCT = "Alibaba Cloud ECS"
class DataSourceAliYun(EC2.DataSourceEc2):
- metadata_urls = ["http://100.100.100.200"]
+
+ metadata_urls = ['http://100.100.100.200']
+
+ # The minimum supported metadata_version from the ec2 metadata apis
+ min_metadata_version = '2016-01-01'
+ extended_metadata_versions = []
def __init__(self, sys_cfg, distro, paths):
super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)
self.seed_dir = os.path.join(paths.seed_dir, "AliYun")
- self.api_ver = DEF_MD_VERSION
def get_hostname(self, fqdn=False, _resolve_ip=False):
return self.metadata.get('hostname', 'localhost.localdomain')
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 4ec9592f..8e5f8ee4 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -13,6 +13,8 @@ import time
from cloudinit import ec2_utils as ec2
from cloudinit import log as logging
+from cloudinit import net
+from cloudinit.net import dhcp
from cloudinit import sources
from cloudinit import url_helper as uhelp
from cloudinit import util
@@ -20,8 +22,7 @@ from cloudinit import warnings
LOG = logging.getLogger(__name__)
-# Which version we are requesting of the ec2 metadata apis
-DEF_MD_VERSION = '2009-04-04'
+SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND])
STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
STRICT_ID_DEFAULT = "warn"
@@ -41,17 +42,28 @@ class Platforms(object):
class DataSourceEc2(sources.DataSource):
+
# Default metadata urls that will be used if none are provided
# They will be checked for 'resolveability' and some of the
# following may be discarded if they do not resolve
metadata_urls = ["http://169.254.169.254", "http://instance-data.:8773"]
+
+ # The minimum supported metadata_version from the ec2 metadata apis
+ min_metadata_version = '2009-04-04'
+
+ # Priority ordered list of additional metadata versions which will be tried
+ # for extended metadata content. IPv6 support comes in 2016-09-02
+ extended_metadata_versions = ['2016-09-02']
+
_cloud_platform = None
+ # Whether we want to get network configuration from the metadata service.
+ get_network_metadata = False
+
def __init__(self, sys_cfg, distro, paths):
sources.DataSource.__init__(self, sys_cfg, distro, paths)
self.metadata_address = None
self.seed_dir = os.path.join(paths.seed_dir, "ec2")
- self.api_ver = DEF_MD_VERSION
def get_data(self):
seed_ret = {}
@@ -73,21 +85,27 @@ class DataSourceEc2(sources.DataSource):
elif self.cloud_platform == Platforms.NO_EC2_METADATA:
return False
- try:
- if not self.wait_for_metadata_service():
+ if self.get_network_metadata: # Setup networking in init-local stage.
+ if util.is_FreeBSD():
+ LOG.debug("FreeBSD doesn't support running dhclient with -sf")
return False
- start_time = time.time()
- self.userdata_raw = \
- ec2.get_instance_userdata(self.api_ver, self.metadata_address)
- self.metadata = ec2.get_instance_metadata(self.api_ver,
- self.metadata_address)
- LOG.debug("Crawl of metadata service took %.3f seconds",
- time.time() - start_time)
- return True
- except Exception:
- util.logexc(LOG, "Failed reading from metadata address %s",
- self.metadata_address)
- return False
+ dhcp_leases = dhcp.maybe_perform_dhcp_discovery()
+ if not dhcp_leases:
+ # DataSourceEc2Local failed in init-local stage. DataSourceEc2
+ # will still run in init-network stage.
+ return False
+ dhcp_opts = dhcp_leases[-1]
+ net_params = {'interface': dhcp_opts.get('interface'),
+ 'ip': dhcp_opts.get('fixed-address'),
+ 'prefix_or_mask': dhcp_opts.get('subnet-mask'),
+ 'broadcast': dhcp_opts.get('broadcast-address'),
+ 'router': dhcp_opts.get('routers')}
+ with net.EphemeralIPv4Network(**net_params):
+ return util.log_time(
+ logfunc=LOG.debug, msg='Crawl of metadata service',
+ func=self._crawl_metadata)
+ else:
+ return self._crawl_metadata()
@property
def launch_index(self):
@@ -95,6 +113,32 @@ class DataSourceEc2(sources.DataSource):
return None
return self.metadata.get('ami-launch-index')
+ def get_metadata_api_version(self):
+ """Get the best supported api version from the metadata service.
+
+ Loop through all extended support metadata versions in order and
+ return the most-fully featured metadata api version discovered.
+
+ If extended_metadata_versions aren't present, return the datasource's
+ min_metadata_version.
+ """
+ # Assumes metadata service is already up
+ for api_ver in self.extended_metadata_versions:
+ url = '{0}/{1}/meta-data/instance-id'.format(
+ self.metadata_address, api_ver)
+ try:
+ resp = uhelp.readurl(url=url)
+ except uhelp.UrlError as e:
+ LOG.debug('url %s raised exception %s', url, e)
+ else:
+ if resp.code == 200:
+ LOG.debug('Found preferred metadata version %s', api_ver)
+ return api_ver
+ elif resp.code == 404:
+ msg = 'Metadata api version %s not present. Headers: %s'
+ LOG.debug(msg, api_ver, resp.headers)
+ return self.min_metadata_version
+
def get_instance_id(self):
return self.metadata['instance-id']
@@ -138,21 +182,22 @@ class DataSourceEc2(sources.DataSource):
urls = []
url2base = {}
for url in mdurls:
- cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
+ cur = '{0}/{1}/meta-data/instance-id'.format(
+ url, self.min_metadata_version)
urls.append(cur)
url2base[cur] = url
start_time = time.time()
- url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
- timeout=timeout, status_cb=LOG.warn)
+ url = uhelp.wait_for_url(
+ urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn)
if url:
- LOG.debug("Using metadata source: '%s'", url2base[url])
+ self.metadata_address = url2base[url]
+ LOG.debug("Using metadata source: '%s'", self.metadata_address)
else:
LOG.critical("Giving up on md from %s after %s seconds",
urls, int(time.time() - start_time))
- self.metadata_address = url2base.get(url)
return bool(url)
def device_name_to_device(self, name):
@@ -234,6 +279,37 @@ class DataSourceEc2(sources.DataSource):
util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
cfg)
+ def _crawl_metadata(self):
+ """Crawl metadata service when available.
+
+ @returns: True on success, False otherwise.
+ """
+ if not self.wait_for_metadata_service():
+ return False
+ api_version = self.get_metadata_api_version()
+ try:
+ self.userdata_raw = ec2.get_instance_userdata(
+ api_version, self.metadata_address)
+ self.metadata = ec2.get_instance_metadata(
+ api_version, self.metadata_address)
+ except Exception:
+ util.logexc(
+ LOG, "Failed reading from metadata address %s",
+ self.metadata_address)
+ return False
+ return True
+
+
+class DataSourceEc2Local(DataSourceEc2):
+ """Datasource run at init-local which sets up network to query metadata.
+
+ In init-local, no network is available. This subclass sets up minimal
+ networking with dhclient on a viable nic so that it can talk to the
+ metadata service. If the metadata service provides network configuration
+ then render the network configuration for that instance based on metadata.
+ """
+ get_network_metadata = True # Get metadata network config if present
+
def read_strict_mode(cfgval, default):
try:
@@ -349,6 +425,7 @@ def _collect_platform_data():
# Used to match classes to dependencies
datasources = [
+ (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local
(DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
]
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 08c5c469..bf1dc5df 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -278,7 +278,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
return root
-class HttprettyTestCase(TestCase):
+class HttprettyTestCase(CiTestCase):
# necessary as http_proxy gets in the way of httpretty
# https://github.com/gabrielfalcao/HTTPretty/issues/122
def setUp(self):
diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py
index 990bff2c..996560e4 100644
--- a/tests/unittests/test_datasource/test_aliyun.py
+++ b/tests/unittests/test_datasource/test_aliyun.py
@@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
paths = helpers.Paths({})
self.ds = ay.DataSourceAliYun(cfg, distro, paths)
self.metadata_address = self.ds.metadata_urls[0]
- self.api_ver = self.ds.api_ver
@property
def default_metadata(self):
@@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase):
@property
def metadata_url(self):
- return os.path.join(self.metadata_address,
- self.api_ver, 'meta-data') + '/'
+ return os.path.join(
+ self.metadata_address,
+ self.ds.min_metadata_version, 'meta-data') + '/'
@property
def userdata_url(self):
- return os.path.join(self.metadata_address,
- self.api_ver, 'user-data')
+ return os.path.join(
+ self.metadata_address,
+ self.ds.min_metadata_version, 'user-data')
def regist_default_server(self):
register_mock_metaserver(self.metadata_url, self.default_metadata)
diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py
index 413e87ac..4802f105 100644
--- a/tests/unittests/test_datasource/test_common.py
+++ b/tests/unittests/test_datasource/test_common.py
@@ -35,6 +35,7 @@ DEFAULT_LOCAL = [
OpenNebula.DataSourceOpenNebula,
OVF.DataSourceOVF,
SmartOS.DataSourceSmartOS,
+ Ec2.DataSourceEc2Local,
]
DEFAULT_NETWORK = [
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index 12230ae2..33d02619 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -8,35 +8,67 @@ from cloudinit import helpers
from cloudinit.sources import DataSourceEc2 as ec2
-# collected from api version 2009-04-04/ with
+# collected from api version 2016-09-02/ with
# python3 -c 'import json
# from cloudinit.ec2_utils import get_instance_metadata as gm
-# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))'
+# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))'
DEFAULT_METADATA = {
- "ami-id": "ami-80861296",
+ "ami-id": "ami-8b92b4ee",
"ami-launch-index": "0",
"ami-manifest-path": "(unknown)",
"block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"},
- "hostname": "ip-10-0-0-149",
+ "hostname": "ip-172-31-31-158.us-east-2.compute.internal",
"instance-action": "none",
- "instance-id": "i-0052913950685138c",
- "instance-type": "t2.micro",
- "local-hostname": "ip-10-0-0-149",
- "local-ipv4": "10.0.0.149",
- "placement": {"availability-zone": "us-east-1b"},
+ "instance-id": "i-0a33f80f09c96477f",
+ "instance-type": "t2.small",
+ "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal",
+ "local-ipv4": "172.3.3.15",
+ "mac": "06:17:04:d7:26:09",
+ "metrics": {"vhostmd": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"},
+ "network": {
+ "interfaces": {
+ "macs": {
+ "06:17:04:d7:26:09": {
+ "device-number": "0",
+ "interface-id": "eni-e44ef49e",
+ "ipv4-associations": {"13.59.77.202": "172.3.3.15"},
+ "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc",
+ "local-hostname": ("ip-172-3-3-15.us-east-2."
+ "compute.internal"),
+ "local-ipv4s": "172.3.3.15",
+ "mac": "06:17:04:d7:26:09",
+ "owner-id": "950047163771",
+ "public-hostname": ("ec2-13-59-77-202.us-east-2."
+ "compute.amazonaws.com"),
+ "public-ipv4s": "13.59.77.202",
+ "security-group-ids": "sg-5a61d333",
+ "security-groups": "wide-open",
+ "subnet-id": "subnet-20b8565b",
+ "subnet-ipv4-cidr-block": "172.31.16.0/20",
+ "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64",
+ "vpc-id": "vpc-87e72bee",
+ "vpc-ipv4-cidr-block": "172.31.0.0/16",
+ "vpc-ipv4-cidr-blocks": "172.31.0.0/16",
+ "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56"
+ }
+ }
+ }
+ },
+ "placement": {"availability-zone": "us-east-2b"},
"profile": "default-hvm",
- "public-hostname": "",
- "public-ipv4": "107.23.188.247",
+ "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com",
+ "public-ipv4": "13.59.77.202",
"public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]},
- "reservation-id": "r-00a2c173fb5782a08",
- "security-groups": "wide-open"
+ "reservation-id": "r-01efbc9996bac1bd6",
+ "security-groups": "my-wide-open",
+ "services": {"domain": "amazonaws.com", "partition": "aws"}
}
def _register_ssh_keys(rfunc, base_url, keys_data):
"""handle ssh key inconsistencies.
- public-keys in the ec2 metadata is inconsistently formatted compared
+ public-keys in the ec2 metadata is inconsistently formated compared
to other entries.
Given keys_data of {name1: pubkey1, name2: pubkey2}
@@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data):
class TestEc2(test_helpers.HttprettyTestCase):
+ with_logs = True
+
valid_platform_data = {
'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412',
'uuid_source': 'dmi',
@@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase):
def setUp(self):
super(TestEc2, self).setUp()
- self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0]
- self.api_ver = '2009-04-04'
+ self.datasource = ec2.DataSourceEc2
+ self.metadata_addr = self.datasource.metadata_urls[0]
@property
def metadata_url(self):
- return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', ''])
+ return '/'.join([
+ self.metadata_addr,
+ self.datasource.min_metadata_version, 'meta-data', ''])
@property
def userdata_url(self):
- return '/'.join([self.metadata_addr, self.api_ver, 'user-data'])
+ return '/'.join([
+ self.metadata_addr,
+ self.datasource.min_metadata_version, 'user-data'])
def _patch_add_cleanup(self, mpath, *args, **kwargs):
p = mock.patch(mpath, *args, **kwargs)
@@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
paths = helpers.Paths({})
if sys_cfg is None:
sys_cfg = {}
- ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths)
+ ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths)
if platform_data is not None:
self._patch_add_cleanup(
"cloudinit.sources.DataSourceEc2._collect_platform_data",
@@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase):
return ds
@httpretty.activate
- def test_valid_platform_with_strict_true(self):
+ @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+ def test_valid_platform_with_strict_true(self, m_dhcp):
"""Valid platform data should return true with strict_id true."""
ds = self._setup_ds(
platform_data=self.valid_platform_data,
sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
md=DEFAULT_METADATA)
ret = ds.get_data()
- self.assertEqual(True, ret)
+ self.assertTrue(ret)
+ self.assertEqual(0, m_dhcp.call_count)
@httpretty.activate
def test_valid_platform_with_strict_false(self):
@@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
md=DEFAULT_METADATA)
ret = ds.get_data()
- self.assertEqual(True, ret)
+ self.assertTrue(ret)
@httpretty.activate
def test_unknown_platform_with_strict_true(self):
@@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
sys_cfg={'datasource': {'Ec2': {'strict_id': True}}},
md=DEFAULT_METADATA)
ret = ds.get_data()
- self.assertEqual(False, ret)
+ self.assertFalse(ret)
@httpretty.activate
def test_unknown_platform_with_strict_false(self):
@@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase):
sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
md=DEFAULT_METADATA)
ret = ds.get_data()
- self.assertEqual(True, ret)
+ self.assertTrue(ret)
+
+ @httpretty.activate
+ @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+ def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd):
+ """DataSourceEc2Local returns False on BSD.
+
+ FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox.
+ """
+ m_is_freebsd.return_value = True
+ self.datasource = ec2.DataSourceEc2Local
+ ds = self._setup_ds(
+ platform_data=self.valid_platform_data,
+ sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+ md=DEFAULT_METADATA)
+ ret = ds.get_data()
+ self.assertFalse(ret)
+ self.assertIn(
+ "FreeBSD doesn't support running dhclient with -sf",
+ self.logs.getvalue())
+
+ @httpretty.activate
+ @mock.patch('cloudinit.net.EphemeralIPv4Network')
+ @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
+ @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
+ def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net):
+ """Ec2Local returns True for valid platform data on non-BSD with dhcp.
+
+ DataSourceEc2Local will setup initial IPv4 network via dhcp discovery.
+ Then the metadata services is crawled for more network config info.
+ When the platform data is valid, return True.
+ """
+ m_is_bsd.return_value = False
+ m_dhcp.return_value = [{
+ 'interface': 'eth9', 'fixed-address': '192.168.2.9',
+ 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
+ 'broadcast-address': '192.168.2.255'}]
+ self.datasource = ec2.DataSourceEc2Local
+ ds = self._setup_ds(
+ platform_data=self.valid_platform_data,
+ sys_cfg={'datasource': {'Ec2': {'strict_id': False}}},
+ md=DEFAULT_METADATA)
+ ret = ds.get_data()
+ self.assertTrue(ret)
+ m_dhcp.assert_called_once_with()
+ m_net.assert_called_once_with(
+ broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
+ prefix_or_mask='255.255.255.0', router='192.168.2.1')
+ self.assertIn('Crawl of metadata service took', self.logs.getvalue())
# vi: ts=4 expandtab
diff --git a/tox.ini b/tox.ini
index ef768847..1e7ca2d3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,10 +21,10 @@ setenv =
LC_ALL = en_US.utf-8
[testenv:pylint]
-deps =
+deps =
# requirements
pylint==1.7.1
- # test-requirements because unit tests are now present in cloudinit tree
+ # test-requirements because unit tests are now present in cloudinit tree
-r{toxinidir}/test-requirements.txt
commands = {envpython} -m pylint {posargs:cloudinit}
generated by cgit v1.2.3 (git 2.39.1) at 2025-02-08 10:46:38 +0000