From d5f855dd96ccbea77f61b0515b574ad2c43d116d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 9 Aug 2017 21:55:52 -0600 Subject: ec2: Allow Ec2 to run in init-local using dhclient in a sandbox. This branch is a prerequisite for IPv6 support in AWS by allowing Ec2 datasource to query the metadata source version 2016-09-02 about whether or not it needs to configure IPv6 on interfaces. If version 2016-09-02 is not present, fallback to the min_metadata_version of 2009-04-04. The DataSourceEc2Local not run on FreeBSD because dhclient in doesn't support the -sf flag allowing us to run dhclient without filesystem side-effects. To query AWS' metadata address @ 169.254.169.254, the instance must have a dhcp-allocated address configured. Configuring IPv4 link-local addresses result in timeouts from the metadata service. We introduced a DataSourceEc2Local subclass which will perform a sandboxed dhclient discovery which obtains an authorized IP address on eth0 and crawl metadata about full instance network configuration. Since ec2 IPv6 metadata is not sufficient in itself to tell us all the ipv6 knownledge we need, it only be used as a boolean to tell us which nics need IPv6. Cloud-init will then configure desired interfaces to DHCPv6 versus DHCPv4. Performance side note: Shifting the dhcp work into init-local for Ec2 actually gets us 1 second faster deployments by skipping init-network phase of alternate datasource checks because Ec2Local is configured in an ealier boot stage. In 3 test runs prior to this change: cloud-init runs were 5.5 seconds, with the change we now average 4.6 seconds. This efficiency could be even further improved if we avoiding dhcp discovery in order to talk to the metadata service from an AWS authorized dhcp address if there were some way to advertize the dhcp configuration via DMI/SMBIOS or system environment variables. Inspecting time costs of the dhclient setup/teardown in 3 live runs the time cost for the dhcp setup round trip on AWS is: test 1: 76 milliseconds dhcp discovery + metadata: 0.347 seconds metadata alone: 0.271 seconds test 2: 88 milliseconds dhcp discovery + metadata: 0.388 seconds metadata alone: 0.300 seconds test 3: 75 milliseconds dhcp discovery + metadata: 0.366 seconds metadata alone: 0.291 seconds LP: #1709772 --- cloudinit/net/dhcp.py | 119 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 cloudinit/net/dhcp.py (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py new file mode 100644 index 00000000..c7febc57 --- /dev/null +++ b/cloudinit/net/dhcp.py @@ -0,0 +1,119 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# Author: Chad Smith +# +# This file is part of cloud-init. See LICENSE file for license information. + +import logging +import os +import re + +from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class InvalidDHCPLeaseFileError(Exception): + """Raised when parsing an empty or invalid dhcp.leases file. + + Current uses are DataSourceAzure and DataSourceEc2 during ephemeral + boot to scrape metadata. + """ + pass + + +def maybe_perform_dhcp_discovery(nic=None): + """Perform dhcp discovery if nic valid and dhclient command exists. + + If the nic is invalid or undiscoverable or dhclient command is not found, + skip dhcp_discovery and return an empty dict. + + @param nic: Name of the network interface we want to run dhclient on. + @return: A dict of dhcp options from the dhclient discovery if run, + otherwise an empty dict is returned. + """ + if nic is None: + nic = find_fallback_nic() + if nic is None: + LOG.debug( + 'Skip dhcp_discovery: Unable to find fallback nic.') + return {} + elif nic not in get_devicelist(): + LOG.debug( + 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) + return {} + dhclient_path = util.which('dhclient') + if not dhclient_path: + LOG.debug('Skip dhclient configuration: No dhclient command found.') + return {} + with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + return dhcp_discovery(dhclient_path, nic, tmpdir) + + +def parse_dhcp_lease_file(lease_file): + """Parse the given dhcp lease file for the most recent lease. + + Return a dict of dhcp options as key value pairs for the most recent lease + block. + + @raises: InvalidDHCPLeaseFileError on empty of unparseable leasefile + content. + """ + lease_regex = re.compile(r"lease {(?P[^}]*)}\n") + dhcp_leases = [] + lease_content = util.load_file(lease_file) + if len(lease_content) == 0: + raise InvalidDHCPLeaseFileError( + 'Cannot parse empty dhcp lease file {0}'.format(lease_file)) + for lease in lease_regex.findall(lease_content): + lease_options = [] + for line in lease.split(';'): + # Strip newlines, double-quotes and option prefix + line = line.strip().replace('"', '').replace('option ', '') + if not line: + continue + lease_options.append(line.split(' ', 1)) + dhcp_leases.append(dict(lease_options)) + if not dhcp_leases: + raise InvalidDHCPLeaseFileError( + 'Cannot parse dhcp lease file {0}. No leases found'.format( + lease_file)) + return dhcp_leases + + +def dhcp_discovery(dhclient_cmd_path, interface, cleandir): + """Run dhclient on the interface without scripts or filesystem artifacts. + + @param dhclient_cmd_path: Full path to the dhclient used. + @param interface: Name of the network inteface on which to dhclient. + @param cleandir: The directory from which to run dhclient as well as store + dhcp leases. + + @return: A dict of dhcp options parsed from the dhcp.leases file or empty + dict. + """ + LOG.debug('Performing a dhcp discovery on %s', interface) + + # XXX We copy dhclient out of /sbin/dhclient to avoid dealing with strict + # app armor profiles which disallow running dhclient -sf . + # We want to avoid running /sbin/dhclient-script because of side-effects in + # /etc/resolv.conf any any other vendor specific scripts in + # /etc/dhcp/dhclient*hooks.d. + sandbox_dhclient_cmd = os.path.join(cleandir, 'dhclient') + util.copy(dhclient_cmd_path, sandbox_dhclient_cmd) + pid_file = os.path.join(cleandir, 'dhclient.pid') + lease_file = os.path.join(cleandir, 'dhcp.leases') + + # ISC dhclient needs the interface up to send initial discovery packets. + # Generally dhclient relies on dhclient-script PREINIT action to bring the + # link up before attempting discovery. Since we are using -sf /bin/true, + # we need to do that "link up" ourselves first. + util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, + '-pf', pid_file, interface, '-sf', '/bin/true'] + util.subp(cmd, capture=True) + return parse_dhcp_lease_file(lease_file) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From 409918f9ba83e45e9bc5cc0b6c589e2fc8ae9b60 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 29 Aug 2017 09:59:20 -0400 Subject: Use /run/cloud-init for tempfile operations. During boot, the usage of /tmp is not safe. In systemd systems, systemd-tmpfiles-clean may run at any point and clear out a temp file while cloud-init is using it. The solution here is to use /run/cloud-init/tmp. LP: #1707222 --- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_snappy.py | 4 +- cloudinit/net/dhcp.py | 3 +- cloudinit/sources/helpers/azure.py | 4 +- cloudinit/temp_utils.py | 93 ++++++++++++++++++++++ cloudinit/util.py | 36 +-------- packages/bddeb | 5 +- .../unittests/test_datasource/test_azure_helper.py | 4 +- tests/unittests/test_net.py | 3 +- 10 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 cloudinit/temp_utils.py (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 604f93b0..9c0476af 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -37,6 +37,7 @@ specified either as lists or strings. For invocation details, see ``runcmd``. import os from cloudinit.settings import PER_ALWAYS +from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS @@ -49,7 +50,7 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return - with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: + with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 02c70b10..c192dd32 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -71,6 +71,7 @@ import itertools import json import os +from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -303,7 +304,7 @@ def install_chef(cloud, chef_cfg, log): "omnibus_url_retries", default=OMNIBUS_URL_RETRIES)) content = url_helper.readurl(url=url, retries=retries).contents - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd util.write_file(tmpf, content, mode=0o700) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a9682f19..eecb8178 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,11 +63,11 @@ is ``auto``. Options are: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import temp_utils from cloudinit import util import glob import os -import tempfile LOG = logging.getLogger(__name__) @@ -183,7 +183,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # config # Note, however, we do not touch config files on disk. nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = tempfile.mkstemp() + (fd, cfg_tmpf) = temp_utils.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c7febc57..c842c839 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -9,6 +9,7 @@ import os import re from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import temp_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: return dhcp_discovery(dhclient_path, nic, tmpdir) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index e22409d1..28ed0ae2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -6,10 +6,10 @@ import os import re import socket import struct -import tempfile import time from cloudinit import stages +from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree @@ -111,7 +111,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.mkdtemp() + self.tmpdir = temp_utils.mkdtemp() self.certificate = None self.generate_certificate() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py new file mode 100644 index 00000000..0355f19d --- /dev/null +++ b/cloudinit/temp_utils.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import contextlib +import errno +import os +import shutil +import tempfile + +_TMPDIR = None +_ROOT_TMPDIR = "/run/cloud-init/tmp" + + +def _tempfile_dir_arg(odir=None): + """Return the proper 'dir' argument for tempfile functions. + + When root, cloud-init will use /run/cloud-init/tmp to avoid + any cleaning that a distro boot might do on /tmp (such as + systemd-tmpfiles-clean). + + If the caller of this function (mkdtemp or mkstemp) was provided + with a 'dir' argument, then that is respected. + + @param odir: original 'dir' arg to 'mkdtemp' or other.""" + + if odir is not None: + return odir + + global _TMPDIR + if _TMPDIR: + return _TMPDIR + + if os.getuid() == 0: + tdir = _ROOT_TMPDIR + else: + tdir = os.environ.get('TMPDIR', '/tmp') + if not os.path.isdir(tdir): + os.makedirs(tdir) + os.chmod(tdir, 0o1777) + + _TMPDIR = tdir + return tdir + + +def ExtendedTemporaryFile(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + + def _unlink_if_exists(path): + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + + fh.unlink = _unlink_if_exists + + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + + setattr(fh, 'unlink_now', unlink_now) + return fh + + +@contextlib.contextmanager +def tempdir(**kwargs): + # This seems like it was only added in python 3.2 + # Make it since its useful... + # See: http://bugs.python.org/file12970/tempdir.patch + tdir = mkdtemp(**kwargs) + try: + yield tdir + finally: + shutil.rmtree(tdir) + + +def mkdtemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkdtemp(**kwargs) + + +def mkstemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkstemp(**kwargs) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 609e94c8..ae5cda8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -30,7 +30,6 @@ import stat import string import subprocess import sys -import tempfile import time from errno import ENOENT, ENOEXEC @@ -45,6 +44,7 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import mergers from cloudinit import safeyaml +from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import url_helper from cloudinit import version @@ -349,26 +349,6 @@ class DecompressionError(Exception): pass -def ExtendedTemporaryFile(**kwargs): - fh = tempfile.NamedTemporaryFile(**kwargs) - # Replace its unlink with a quiet version - # that does not raise errors when the - # file to unlink has been unlinked elsewhere.. - LOG.debug("Created temporary file %s", fh.name) - fh.unlink = del_file - - # Add a new method that will unlink - # right 'now' but still lets the exit - # method attempt to remove it (which will - # not throw due to our del file being quiet - # about files that are not there) - def unlink_now(): - fh.unlink(fh.name) - - setattr(fh, 'unlink_now', unlink_now) - return fh - - def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: @@ -790,18 +770,6 @@ def umask(n_msk): os.umask(old) -@contextlib.contextmanager -def tempdir(**kwargs): - # This seems like it was only added in python 3.2 - # Make it since its useful... - # See: http://bugs.python.org/file12970/tempdir.patch - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - del_dir(tdir) - - def center(text, fill, max_len): return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len) @@ -1587,7 +1555,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): mtypes = [''] mounted = mounts() - with tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: umount = False if os.path.realpath(device) in mounted: mountpoint = mounted[os.path.realpath(device)]['mountpoint'] diff --git a/packages/bddeb b/packages/bddeb index 7c123548..4f2e2ddf 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -21,8 +21,9 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import templater from cloudinit import util + from cloudinit import temp_utils + from cloudinit import templater DEBUILD_ARGS = ["-S", "-d"] @@ -148,7 +149,7 @@ def main(): capture = False templ_data = {'debian_release': args.release} - with util.tempdir() as tdir: + with temp_utils.tempdir() as tdir: # output like 0.7.6-1022-g36e92d3 ver_data = read_version() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 80ce003d..44b99eca 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -275,7 +275,7 @@ class TestOpenSSLManager(TestCase): mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp') def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() self.assertEqual(mkdtemp.return_value, manager.tmpdir) @@ -292,7 +292,7 @@ class TestOpenSSLManager(TestCase): manager.clean_up() @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock()) @mock.patch.object(azure_helper.util, 'del_dir') def test_clean_up(self, del_dir): manager = azure_helper.OpenSSLManager() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c10ef905..f2496151 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -9,6 +9,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack +from cloudinit import temp_utils from cloudinit import util from cloudinit.tests.helpers import CiTestCase @@ -2150,7 +2151,7 @@ class TestCmdlineConfigParsing(CiTestCase): static['mac_address'] = macs['eth1'] expected = {'version': 1, 'config': [dhcp, static]} - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: for fname, content in pairs: fp = os.path.join(tmpd, fname) files.append(fp) -- cgit v1.2.3 From 7eb3460b0d6d3e362a246958a7ea0a9ee5d91d5e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 15 Sep 2017 20:07:11 -0600 Subject: ec2: Fix maybe_perform_dhcp_discovery to use /var/tmp as a tmpdir /run/cloud-init/tmp is on a filesystem mounted noexec, so running dchlient in Ec2Local during discovery breaks with 'Permission denied'. This branch allows us to run from a different tmp dir so we have exec rights. LP: #1717627 --- cloudinit/net/dhcp.py | 5 +- cloudinit/net/tests/test_dhcp.py | 18 ++++--- cloudinit/temp_utils.py | 22 +++++--- cloudinit/tests/test_temp_utils.py | 101 +++++++++++++++++++++++++++++++++++++ cloudinit/util.py | 2 +- 5 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 cloudinit/tests/test_temp_utils.py (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c842c839..05350639 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -48,8 +48,9 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: - return dhcp_discovery(dhclient_path, nic, tmpdir) + with temp_utils.tempdir(prefix='cloud-init-dhcp-', needs_exe=True) as tdir: + # Use /var/tmp because /run/cloud-init/tmp is mounted noexec + return dhcp_discovery(dhclient_path, nic, tdir) def parse_dhcp_lease_file(lease_file): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 4a37e98a..1324c3d0 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -8,7 +8,7 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery) from cloudinit.util import ensure_file, write_file -from cloudinit.tests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase, wrap_and_call class TestParseDHCPLeasesFile(CiTestCase): @@ -91,21 +91,27 @@ class TestDHCPDiscoveryClean(CiTestCase): 'Skip dhclient configuration: No dhclient command found.', self.logs.getvalue()) + @mock.patch('cloudinit.temp_utils.os.getuid') @mock.patch('cloudinit.net.dhcp.dhcp_discovery') @mock.patch('cloudinit.net.dhcp.util.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') - def test_dhclient_run_with_tmpdir(self, m_fallback, m_which, m_dhcp): + def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid): """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" - m_fallback.return_value = 'eth9' + m_uid.return_value = 0 # Fake root user for tmpdir + m_fback.return_value = 'eth9' m_which.return_value = '/sbin/dhclient' m_dhcp.return_value = {'address': '192.168.2.2'} - self.assertEqual( - {'address': '192.168.2.2'}, maybe_perform_dhcp_discovery()) + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'_TMPDIR': {'new': None}, + 'os.getuid': 0}, + maybe_perform_dhcp_discovery) + self.assertEqual({'address': '192.168.2.2'}, retval) m_dhcp.assert_called_once() call = m_dhcp.call_args_list[0] self.assertEqual('/sbin/dhclient', call[0][0]) self.assertEqual('eth9', call[0][1]) - self.assertIn('/tmp/cloud-init-dhcp-', call[0][2]) + self.assertIn('/var/tmp/cloud-init/cloud-init-dhcp-', call[0][2]) @mock.patch('cloudinit.net.dhcp.util.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp): diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py index 0355f19d..5d7adf70 100644 --- a/cloudinit/temp_utils.py +++ b/cloudinit/temp_utils.py @@ -8,9 +8,10 @@ import tempfile _TMPDIR = None _ROOT_TMPDIR = "/run/cloud-init/tmp" +_EXE_ROOT_TMPDIR = "/var/tmp/cloud-init" -def _tempfile_dir_arg(odir=None): +def _tempfile_dir_arg(odir=None, needs_exe=False): """Return the proper 'dir' argument for tempfile functions. When root, cloud-init will use /run/cloud-init/tmp to avoid @@ -20,8 +21,10 @@ def _tempfile_dir_arg(odir=None): If the caller of this function (mkdtemp or mkstemp) was provided with a 'dir' argument, then that is respected. - @param odir: original 'dir' arg to 'mkdtemp' or other.""" - + @param odir: original 'dir' arg to 'mkdtemp' or other. + @param needs_exe: Boolean specifying whether or not exe permissions are + needed for tempdir. This is needed because /run is mounted noexec. + """ if odir is not None: return odir @@ -29,7 +32,9 @@ def _tempfile_dir_arg(odir=None): if _TMPDIR: return _TMPDIR - if os.getuid() == 0: + if needs_exe: + tdir = _EXE_ROOT_TMPDIR + elif os.getuid() == 0: tdir = _ROOT_TMPDIR else: tdir = os.environ.get('TMPDIR', '/tmp') @@ -42,7 +47,8 @@ def _tempfile_dir_arg(odir=None): def ExtendedTemporaryFile(**kwargs): - kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + kwargs['dir'] = _tempfile_dir_arg( + kwargs.pop('dir', None), kwargs.pop('needs_exe', False)) fh = tempfile.NamedTemporaryFile(**kwargs) # Replace its unlink with a quiet version # that does not raise errors when the @@ -82,12 +88,14 @@ def tempdir(**kwargs): def mkdtemp(**kwargs): - kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + kwargs['dir'] = _tempfile_dir_arg( + kwargs.pop('dir', None), kwargs.pop('needs_exe', False)) return tempfile.mkdtemp(**kwargs) def mkstemp(**kwargs): - kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + kwargs['dir'] = _tempfile_dir_arg( + kwargs.pop('dir', None), kwargs.pop('needs_exe', False)) return tempfile.mkstemp(**kwargs) # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_temp_utils.py b/cloudinit/tests/test_temp_utils.py new file mode 100644 index 00000000..ffbb92cd --- /dev/null +++ b/cloudinit/tests/test_temp_utils.py @@ -0,0 +1,101 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Tests for cloudinit.temp_utils""" + +from cloudinit.temp_utils import mkdtemp, mkstemp +from cloudinit.tests.helpers import CiTestCase, wrap_and_call + + +class TestTempUtils(CiTestCase): + + def test_mkdtemp_default_non_root(self): + """mkdtemp creates a dir under /tmp for the unprivileged.""" + calls = [] + + def fake_mkdtemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 1000, + 'tempfile.mkdtemp': {'side_effect': fake_mkdtemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkdtemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/tmp'}], calls) + + def test_mkdtemp_default_non_root_needs_exe(self): + """mkdtemp creates a dir under /var/tmp/cloud-init when needs_exe.""" + calls = [] + + def fake_mkdtemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 1000, + 'tempfile.mkdtemp': {'side_effect': fake_mkdtemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkdtemp, needs_exe=True) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/var/tmp/cloud-init'}], calls) + + def test_mkdtemp_default_root(self): + """mkdtemp creates a dir under /run/cloud-init for the privileged.""" + calls = [] + + def fake_mkdtemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 0, + 'tempfile.mkdtemp': {'side_effect': fake_mkdtemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkdtemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/run/cloud-init/tmp'}], calls) + + def test_mkstemp_default_non_root(self): + """mkstemp creates secure tempfile under /tmp for the unprivileged.""" + calls = [] + + def fake_mkstemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 1000, + 'tempfile.mkstemp': {'side_effect': fake_mkstemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkstemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/tmp'}], calls) + + def test_mkstemp_default_root(self): + """mkstemp creates a secure tempfile in /run/cloud-init for root.""" + calls = [] + + def fake_mkstemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 0, + 'tempfile.mkstemp': {'side_effect': fake_mkstemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkstemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/run/cloud-init/tmp'}], calls) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 7e9d94fc..4c01f449 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1755,7 +1755,7 @@ def subp_blob_in_tempfile(blob, *args, **kwargs): args = [tuple()] # Use tmpdir over tmpfile to avoid 'text file busy' on execute - with temp_utils.tempdir() as tmpd: + with temp_utils.tempdir(needs_exe=True) as tmpd: tmpf = os.path.join(tmpd, basename) if 'args' in kwargs: kwargs['args'] = [tmpf] + list(kwargs['args']) -- cgit v1.2.3 From 9d2a87dc386b7aed1a8243d599676e78ed358749 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Sat, 30 Sep 2017 00:00:18 -0400 Subject: Azure, CloudStack: Support reading dhcp options from systemd-networkd. Systems that used systemd-networkd's dhcp client would not be able to get information on the Azure endpoint (placed in Option 245) or the CloudStack server (in 'server_address'). The change here supports reading these files in /run/systemd/netif/leases. The files declare that "This is private data. Do not parse.", but at this point we do not have another option. LP: #1718029 --- cloudinit/net/dhcp.py | 42 ++++++ cloudinit/net/tests/test_dhcp.py | 113 +++++++++++++++- cloudinit/sources/DataSourceCloudStack.py | 17 ++- cloudinit/sources/helpers/azure.py | 20 ++- .../unittests/test_datasource/test_azure_helper.py | 143 ++++++++++++++------- tests/unittests/test_datasource/test_cloudstack.py | 11 +- 6 files changed, 282 insertions(+), 64 deletions(-) (limited to 'cloudinit/net/dhcp.py') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 05350639..0cba7032 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -4,6 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import configobj import logging import os import re @@ -11,9 +12,12 @@ import re from cloudinit.net import find_fallback_nic, get_devicelist from cloudinit import temp_utils from cloudinit import util +from six import StringIO LOG = logging.getLogger(__name__) +NETWORKD_LEASES_DIR = '/run/systemd/netif/leases' + class InvalidDHCPLeaseFileError(Exception): """Raised when parsing an empty or invalid dhcp.leases file. @@ -118,4 +122,42 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): return parse_dhcp_lease_file(lease_file) +def networkd_parse_lease(content): + """Parse a systemd lease file content as in /run/systemd/netif/leases/ + + Parse this (almost) ini style file even though it says: + # This is private data. Do not parse. + + Simply return a dictionary of key/values.""" + + return dict(configobj.ConfigObj(StringIO(content), list_values=False)) + + +def networkd_load_leases(leases_d=None): + """Return a dictionary of dictionaries representing each lease + found in lease_d.i + + The top level key will be the filename, which is typically the ifindex.""" + + if leases_d is None: + leases_d = NETWORKD_LEASES_DIR + + ret = {} + if not os.path.isdir(leases_d): + return ret + for lfile in os.listdir(leases_d): + ret[lfile] = networkd_parse_lease( + util.load_file(os.path.join(leases_d, lfile))) + return ret + + +def networkd_get_option_from_leases(keyname, leases_d=None): + if leases_d is None: + leases_d = NETWORKD_LEASES_DIR + leases = networkd_load_leases(leases_d=leases_d) + for ifindex, data in sorted(leases.items()): + if data.get(keyname): + return data[keyname] + return None + # vi: ts=4 expandtab diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index a38edaec..1c1f504a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -6,9 +6,9 @@ from textwrap import dedent from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, - parse_dhcp_lease_file, dhcp_discovery) + parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases) from cloudinit.util import ensure_file, write_file -from cloudinit.tests.helpers import CiTestCase, wrap_and_call +from cloudinit.tests.helpers import CiTestCase, wrap_and_call, populate_dir class TestParseDHCPLeasesFile(CiTestCase): @@ -149,3 +149,112 @@ class TestDHCPDiscoveryClean(CiTestCase): [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf', lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), 'eth9', '-sf', '/bin/true'], capture=True)]) + + +class TestSystemdParseLeases(CiTestCase): + + lxd_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.75.205.242 + NETMASK=255.255.255.0 + ROUTER=10.75.205.1 + SERVER_ADDRESS=10.75.205.1 + NEXT_SERVER=10.75.205.1 + BROADCAST=10.75.205.255 + T1=1580 + T2=2930 + LIFETIME=3600 + DNS=10.75.205.1 + DOMAINNAME=lxd + HOSTNAME=a1 + CLIENTID=ffe617693400020000ab110c65a6a0866931c2 + """) + + lxd_parsed = { + 'ADDRESS': '10.75.205.242', + 'NETMASK': '255.255.255.0', + 'ROUTER': '10.75.205.1', + 'SERVER_ADDRESS': '10.75.205.1', + 'NEXT_SERVER': '10.75.205.1', + 'BROADCAST': '10.75.205.255', + 'T1': '1580', + 'T2': '2930', + 'LIFETIME': '3600', + 'DNS': '10.75.205.1', + 'DOMAINNAME': 'lxd', + 'HOSTNAME': 'a1', + 'CLIENTID': 'ffe617693400020000ab110c65a6a0866931c2', + } + + azure_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.132.0.5 + NETMASK=255.255.255.255 + ROUTER=10.132.0.1 + SERVER_ADDRESS=169.254.169.254 + NEXT_SERVER=10.132.0.1 + MTU=1460 + T1=43200 + T2=75600 + LIFETIME=86400 + DNS=169.254.169.254 + NTP=169.254.169.254 + DOMAINNAME=c.ubuntu-foundations.internal + DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal + HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal + ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 + CLIENTID=ff405663a200020000ab11332859494d7a8b4c + OPTION_245=624c3620 + """) + + azure_parsed = { + 'ADDRESS': '10.132.0.5', + 'NETMASK': '255.255.255.255', + 'ROUTER': '10.132.0.1', + 'SERVER_ADDRESS': '169.254.169.254', + 'NEXT_SERVER': '10.132.0.1', + 'MTU': '1460', + 'T1': '43200', + 'T2': '75600', + 'LIFETIME': '86400', + 'DNS': '169.254.169.254', + 'NTP': '169.254.169.254', + 'DOMAINNAME': 'c.ubuntu-foundations.internal', + 'DOMAIN_SEARCH_LIST': 'c.ubuntu-foundations.internal google.internal', + 'HOSTNAME': 'tribaal-test-171002-1349.c.ubuntu-foundations.internal', + 'ROUTES': '10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1', + 'CLIENTID': 'ff405663a200020000ab11332859494d7a8b4c', + 'OPTION_245': '624c3620'} + + def setUp(self): + super(TestSystemdParseLeases, self).setUp() + self.lease_d = self.tmp_dir() + + def test_no_leases_returns_empty_dict(self): + """A leases dir with no lease files should return empty dictionary.""" + self.assertEqual({}, networkd_load_leases(self.lease_d)) + + def test_no_leases_dir_returns_empty_dict(self): + """A non-existing leases dir should return empty dict.""" + enodir = os.path.join(self.lease_d, 'does-not-exist') + self.assertEqual({}, networkd_load_leases(enodir)) + + def test_single_leases_file(self): + """A leases dir with one leases file.""" + populate_dir(self.lease_d, {'2': self.lxd_lease}) + self.assertEqual( + {'2': self.lxd_parsed}, networkd_load_leases(self.lease_d)) + + def test_single_azure_leases_file(self): + """On Azure, option 245 should be present, verify it specifically.""" + populate_dir(self.lease_d, {'1': self.azure_lease}) + self.assertEqual( + {'1': self.azure_parsed}, networkd_load_leases(self.lease_d)) + + def test_multiple_files(self): + """Multiple leases files on azure with one found return that value.""" + self.maxDiff = None + populate_dir(self.lease_d, {'1': self.azure_lease, + '9': self.lxd_lease}) + self.assertEqual({'1': self.azure_parsed, '9': self.lxd_parsed}, + networkd_load_leases(self.lease_d)) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7e0f9bb8..9dc473fc 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -19,6 +19,7 @@ import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util @@ -224,20 +225,28 @@ def get_vr_address(): # Get the address of the virtual router via dhcp leases # If no virtual router is detected, fallback on default gateway. # See http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/4.8/virtual_machines/user-data.html # noqa + + # Try networkd first... + latest_address = dhcp.networkd_get_option_from_leases('SERVER_ADDRESS') + if latest_address: + LOG.debug("Found SERVER_ADDRESS '%s' via networkd_leases", + latest_address) + return latest_address + + # Try dhcp lease files next... lease_file = get_latest_lease() if not lease_file: LOG.debug("No lease file found, using default gateway") return get_default_gateway() - latest_address = None with open(lease_file, "r") as fd: for line in fd: if "dhcp-server-identifier" in line: words = line.strip(" ;\r\n").split(" ") if len(words) > 2: - dhcp = words[2] - LOG.debug("Found DHCP identifier %s", dhcp) - latest_address = dhcp + dhcptok = words[2] + LOG.debug("Found DHCP identifier %s", dhcptok) + latest_address = dhcptok if not latest_address: # No virtual router found, fallback on default gateway LOG.debug("No DHCP found, using default gateway") diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 28ed0ae2..959b1bda 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -8,6 +8,7 @@ import socket import struct import time +from cloudinit.net import dhcp from cloudinit import stages from cloudinit import temp_utils from contextlib import contextmanager @@ -15,7 +16,6 @@ from xml.etree import ElementTree from cloudinit import util - LOG = logging.getLogger(__name__) @@ -238,6 +238,11 @@ class WALinuxAgentShim(object): packed_bytes = unescaped_value.encode('utf-8') return socket.inet_ntoa(packed_bytes) + @staticmethod + def _networkd_get_value_from_leases(leases_d=None): + return dhcp.networkd_get_option_from_leases( + 'OPTION_245', leases_d=leases_d) + @staticmethod def _get_value_from_leases_file(fallback_lease_file): leases = [] @@ -287,12 +292,15 @@ class WALinuxAgentShim(object): @staticmethod def find_endpoint(fallback_lease_file=None): - LOG.debug('Finding Azure endpoint...') value = None - # Option-245 stored in /run/cloud-init/dhclient.hooks/.json - # a dhclient exit hook that calls cloud-init-dhclient-hook - dhcp_options = WALinuxAgentShim._load_dhclient_json() - value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) + LOG.debug('Finding Azure endpoint from networkd...') + value = WALinuxAgentShim._networkd_get_value_from_leases() + if value is None: + # Option-245 stored in /run/cloud-init/dhclient.hooks/.json + # a dhclient exit hook that calls cloud-init-dhclient-hook + LOG.debug('Finding Azure endpoint from hook json...') + dhcp_options = WALinuxAgentShim._load_dhclient_json() + value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) if value is None: # Fallback and check the leases file if unsuccessful LOG.debug("Unable to find endpoint in dhclient logs. " diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 44b99eca..b42b073f 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,10 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import os +from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper -from cloudinit.tests.helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, populate_dir +from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim GOAL_STATE_TEMPLATE = """\ @@ -45,7 +47,7 @@ GOAL_STATE_TEMPLATE = """\ """ -class TestFindEndpoint(TestCase): +class TestFindEndpoint(CiTestCase): def setUp(self): super(TestFindEndpoint, self).setUp() @@ -56,18 +58,19 @@ class TestFindEndpoint(TestCase): mock.patch.object(azure_helper.util, 'load_file')) self.dhcp_options = patches.enter_context( - mock.patch.object(azure_helper.WALinuxAgentShim, - '_load_dhclient_json')) + mock.patch.object(wa_shim, '_load_dhclient_json')) + + self.networkd_leases = patches.enter_context( + mock.patch.object(wa_shim, '_networkd_get_value_from_leases')) + self.networkd_leases.return_value = None def test_missing_file(self): - self.assertRaises(ValueError, - azure_helper.WALinuxAgentShim.find_endpoint) + self.assertRaises(ValueError, wa_shim.find_endpoint) def test_missing_special_azure_line(self): self.load_file.return_value = '' self.dhcp_options.return_value = {'eth0': {'key': 'value'}} - self.assertRaises(ValueError, - azure_helper.WALinuxAgentShim.find_endpoint) + self.assertRaises(ValueError, wa_shim.find_endpoint) @staticmethod def _build_lease_content(encoded_address): @@ -80,8 +83,7 @@ class TestFindEndpoint(TestCase): def test_from_dhcp_client(self): self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}} - self.assertEqual('5.4.3.2', - azure_helper.WALinuxAgentShim.find_endpoint(None)) + self.assertEqual('5.4.3.2', wa_shim.find_endpoint(None)) def test_latest_lease_used(self): encoded_addresses = ['5:4:3:2', '4:3:2:1'] @@ -89,53 +91,38 @@ class TestFindEndpoint(TestCase): for encoded_address in encoded_addresses]) self.load_file.return_value = file_content self.assertEqual(encoded_addresses[-1].replace(':', '.'), - azure_helper.WALinuxAgentShim.find_endpoint("foobar")) + wa_shim.find_endpoint("foobar")) -class TestExtractIpAddressFromLeaseValue(TestCase): +class TestExtractIpAddressFromLeaseValue(CiTestCase): def test_hex_string(self): ip_address, encoded_address = '98.76.54.32', '62:4c:36:20' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_hex_string_with_single_character_part(self): ip_address, encoded_address = '4.3.2.1', '4:3:2:1' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string(self): ip_address, encoded_address = '98.76.54.32', 'bL6 ' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string_with_escaped_quote(self): ip_address, encoded_address = '100.72.34.108', 'dH\\"l' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string_containing_a_colon(self): ip_address, encoded_address = '100.72.58.108', 'dH:l' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) -class TestGoalStateParsing(TestCase): +class TestGoalStateParsing(CiTestCase): default_parameters = { 'incarnation': 1, @@ -195,7 +182,7 @@ class TestGoalStateParsing(TestCase): self.assertIsNone(certificates_xml) -class TestAzureEndpointHttpClient(TestCase): +class TestAzureEndpointHttpClient(CiTestCase): regular_headers = { 'x-ms-agent-name': 'WALinuxAgent', @@ -258,7 +245,7 @@ class TestAzureEndpointHttpClient(TestCase): self.read_file_or_url.call_args) -class TestOpenSSLManager(TestCase): +class TestOpenSSLManager(CiTestCase): def setUp(self): super(TestOpenSSLManager, self).setUp() @@ -300,7 +287,7 @@ class TestOpenSSLManager(TestCase): self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list) -class TestWALinuxAgentShim(TestCase): +class TestWALinuxAgentShim(CiTestCase): def setUp(self): super(TestWALinuxAgentShim, self).setUp() @@ -310,8 +297,7 @@ class TestWALinuxAgentShim(TestCase): self.AzureEndpointHttpClient = patches.enter_context( mock.patch.object(azure_helper, 'AzureEndpointHttpClient')) self.find_endpoint = patches.enter_context( - mock.patch.object( - azure_helper.WALinuxAgentShim, 'find_endpoint')) + mock.patch.object(wa_shim, 'find_endpoint')) self.GoalState = patches.enter_context( mock.patch.object(azure_helper, 'GoalState')) self.OpenSSLManager = patches.enter_context( @@ -320,7 +306,7 @@ class TestWALinuxAgentShim(TestCase): mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) def test_http_client_uses_certificate(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.OpenSSLManager.return_value.certificate)], @@ -328,7 +314,7 @@ class TestWALinuxAgentShim(TestCase): def test_correct_url_used_for_goalstate(self): self.find_endpoint.return_value = 'test_endpoint' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() get = self.AzureEndpointHttpClient.return_value.get self.assertEqual( @@ -340,7 +326,7 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() data = shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.GoalState.return_value.certificates_xml)], @@ -351,13 +337,13 @@ class TestWALinuxAgentShim(TestCase): def test_absent_certificates_produces_empty_public_keys(self): self.GoalState.return_value.certificates_xml = None - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() data = shim.register_with_azure_and_fetch_data() self.assertEqual([], data['public-keys']) def test_correct_url_used_for_report_ready(self): self.find_endpoint.return_value = 'test_endpoint' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() expected_url = 'http://test_endpoint/machine?comp=health' self.assertEqual( @@ -368,7 +354,7 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.return_value.incarnation = 'TestIncarnation' self.GoalState.return_value.container_id = 'TestContainerId' self.GoalState.return_value.instance_id = 'TestInstanceId' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]['data'] @@ -378,11 +364,11 @@ class TestWALinuxAgentShim(TestCase): self.assertIn('TestInstanceId', posted_document) def test_clean_up_can_be_called_at_any_time(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.clean_up() def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() shim.clean_up() self.assertEqual( @@ -393,12 +379,12 @@ class TestWALinuxAgentShim(TestCase): pass self.AzureEndpointHttpClient.return_value.get.side_effect = ( SentinelException) - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() self.assertRaises(SentinelException, shim.register_with_azure_and_fetch_data) -class TestGetMetadataFromFabric(TestCase): +class TestGetMetadataFromFabric(CiTestCase): @mock.patch.object(azure_helper, 'WALinuxAgentShim') def test_data_from_shim_returned(self, shim): @@ -422,4 +408,65 @@ class TestGetMetadataFromFabric(TestCase): azure_helper.get_metadata_from_fabric) self.assertEqual(1, shim.return_value.clean_up.call_count) + +class TestExtractIpAddressFromNetworkd(CiTestCase): + + azure_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.132.0.5 + NETMASK=255.255.255.255 + ROUTER=10.132.0.1 + SERVER_ADDRESS=169.254.169.254 + NEXT_SERVER=10.132.0.1 + MTU=1460 + T1=43200 + T2=75600 + LIFETIME=86400 + DNS=169.254.169.254 + NTP=169.254.169.254 + DOMAINNAME=c.ubuntu-foundations.internal + DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal + HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal + ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 + CLIENTID=ff405663a200020000ab11332859494d7a8b4c + OPTION_245=624c3620 + """) + + def setUp(self): + super(TestExtractIpAddressFromNetworkd, self).setUp() + self.lease_d = self.tmp_dir() + + def test_no_valid_leases_is_none(self): + """No valid leases should return None.""" + self.assertIsNone( + wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_option_245_is_found_in_single(self): + """A single valid lease with 245 option should return it.""" + populate_dir(self.lease_d, {'9': self.azure_lease}) + self.assertEqual( + '624c3620', wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_option_245_not_found_returns_None(self): + """A valid lease, but no option 245 should return None.""" + populate_dir( + self.lease_d, + {'9': self.azure_lease.replace("OPTION_245", "OPTION_999")}) + self.assertIsNone( + wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_multiple_returns_first(self): + """Somewhat arbitrarily return the first address when multiple. + + Most important at the moment is that this is consistent behavior + rather than changing randomly as in order of a dictionary.""" + myval = "624c3601" + populate_dir( + self.lease_d, + {'9': self.azure_lease, + '2': self.azure_lease.replace("624c3620", myval)}) + self.assertEqual( + myval, wa_shim._networkd_get_value_from_leases(self.lease_d)) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 8e98e1bb..96144b64 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -23,13 +23,16 @@ class TestCloudStackPasswordFetching(CiTestCase): default_gw = "192.201.20.0" get_latest_lease = mock.MagicMock(return_value=None) self.patches.enter_context(mock.patch( - 'cloudinit.sources.DataSourceCloudStack.get_latest_lease', - get_latest_lease)) + mod_name + '.get_latest_lease', get_latest_lease)) get_default_gw = mock.MagicMock(return_value=default_gw) self.patches.enter_context(mock.patch( - 'cloudinit.sources.DataSourceCloudStack.get_default_gateway', - get_default_gw)) + mod_name + '.get_default_gateway', get_default_gw)) + + get_networkd_server_address = mock.MagicMock(return_value=None) + self.patches.enter_context(mock.patch( + mod_name + '.dhcp.networkd_get_option_from_leases', + get_networkd_server_address)) def _set_password_server_response(self, response_string): subp = mock.MagicMock(return_value=(response_string, '')) -- cgit v1.2.3