From 1f8183ff4750cc7f8798749987ef10912719544d Mon Sep 17 00:00:00 2001 From: Maitreyee Saikia Date: Tue, 15 Aug 2017 09:33:50 -0600 Subject: vcloud directory: Guest Customization support for passwords This feature enables the following VMware VCloud Director functionality: 1. Setting admin password 2. Expire password. 3. Set admin password and expire. Password configuration is triggered only as part of a full recustomization, that happens either on first power on or when "poweron and full recustomization" is selected. Full customization flow is determined by marker files. Unique marker ids are generated when full recustomization is requested. And marker file based on these marker ids help to determine if we need to execute the above configuration. --- cloudinit/sources/helpers/vmware/imc/config.py | 24 +++++++- .../sources/helpers/vmware/imc/config_passwd.py | 67 ++++++++++++++++++++++ 2 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/config_passwd.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 9a5e3a8a..49d441db 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -5,6 +5,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. + from .nic import Nic @@ -14,13 +15,16 @@ class Config(object): Specification file. """ + CUSTOM_SCRIPT = 'CUSTOM-SCRIPT|SCRIPT-NAME' DNS = 'DNS|NAMESERVER|' - SUFFIX = 'DNS|SUFFIX|' + DOMAINNAME = 'NETWORK|DOMAINNAME' + HOSTNAME = 'NETWORK|HOSTNAME' + MARKERID = 'MISC|MARKER-ID' PASS = 'PASSWORD|-PASS' + RESETPASS = 'PASSWORD|RESET' + SUFFIX = 'DNS|SUFFIX|' TIMEZONE = 'DATETIME|TIMEZONE' UTC = 'DATETIME|UTC' - HOSTNAME = 'NETWORK|HOSTNAME' - DOMAINNAME = 'NETWORK|DOMAINNAME' def __init__(self, configFile): self._configFile = configFile @@ -82,4 +86,18 @@ class Config(object): return res + @property + def reset_password(self): + """Retreives if the root password needs to be reset.""" + resetPass = self._configFile.get(Config.RESETPASS, 'no') + resetPass = resetPass.lower() + if resetPass not in ('yes', 'no'): + raise ValueError('ResetPassword value should be yes/no') + return resetPass == 'yes' + + @property + def marker_id(self): + """Returns marker id.""" + return self._configFile.get(Config.MARKERID, None) + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py new file mode 100644 index 00000000..75cfbaaf --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py @@ -0,0 +1,67 @@ +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware INC. +# +# Author: Maitreyee Saikia +# +# This file is part of cloud-init. See LICENSE file for license information. + + +import logging +import os + +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class PasswordConfigurator(object): + """ + Class for changing configurations related to passwords in a VM. Includes + setting and expiring passwords. + """ + def configure(self, passwd, resetPasswd, distro): + """ + Main method to perform all functionalities based on configuration file + inputs. + @param passwd: encoded admin password. + @param resetPasswd: boolean to determine if password needs to be reset. + @return cfg: dict to be used by cloud-init set_passwd code. + """ + LOG.info('Starting password configuration') + if passwd: + passwd = util.b64d(passwd) + allRootUsers = [] + for line in open('/etc/passwd', 'r'): + if line.split(':')[2] == '0': + allRootUsers.append(line.split(':')[0]) + # read shadow file and check for each user, if its uid0 or root. + uidUsersList = [] + for line in open('/etc/shadow', 'r'): + user = line.split(':')[0] + if user in allRootUsers: + uidUsersList.append(user) + if passwd: + LOG.info('Setting admin password') + distro.set_passwd('root', passwd) + if resetPasswd: + self.reset_password(uidUsersList) + LOG.info('Configure Password completed!') + + def reset_password(self, uidUserList): + """ + Method to reset password. Use passwd --expire command. Use chage if + not succeeded using passwd command. Log failure message otherwise. + @param: list of users for which to expire password. + """ + LOG.info('Expiring password.') + for user in uidUserList: + try: + out, err = util.subp(['passwd', '--expire', user]) + except util.ProcessExecutionError as e: + if os.path.exists('/usr/bin/chage'): + out, e = util.subp(['chage', '-d', '0', user]) + else: + LOG.warning('Failed to expire password for %s with error: ' + '%s', user, e) + +# vi: ts=4 expandtab -- cgit v1.2.3 From 409918f9ba83e45e9bc5cc0b6c589e2fc8ae9b60 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 29 Aug 2017 09:59:20 -0400 Subject: Use /run/cloud-init for tempfile operations. During boot, the usage of /tmp is not safe. In systemd systems, systemd-tmpfiles-clean may run at any point and clear out a temp file while cloud-init is using it. The solution here is to use /run/cloud-init/tmp. LP: #1707222 --- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_snappy.py | 4 +- cloudinit/net/dhcp.py | 3 +- cloudinit/sources/helpers/azure.py | 4 +- cloudinit/temp_utils.py | 93 ++++++++++++++++++++++ cloudinit/util.py | 36 +-------- packages/bddeb | 5 +- .../unittests/test_datasource/test_azure_helper.py | 4 +- tests/unittests/test_net.py | 3 +- 10 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 cloudinit/temp_utils.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 604f93b0..9c0476af 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -37,6 +37,7 @@ specified either as lists or strings. For invocation details, see ``runcmd``. import os from cloudinit.settings import PER_ALWAYS +from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS @@ -49,7 +50,7 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return - with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: + with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 02c70b10..c192dd32 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -71,6 +71,7 @@ import itertools import json import os +from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -303,7 +304,7 @@ def install_chef(cloud, chef_cfg, log): "omnibus_url_retries", default=OMNIBUS_URL_RETRIES)) content = url_helper.readurl(url=url, retries=retries).contents - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd util.write_file(tmpf, content, mode=0o700) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a9682f19..eecb8178 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,11 +63,11 @@ is ``auto``. Options are: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import temp_utils from cloudinit import util import glob import os -import tempfile LOG = logging.getLogger(__name__) @@ -183,7 +183,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # config # Note, however, we do not touch config files on disk. nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = tempfile.mkstemp() + (fd, cfg_tmpf) = temp_utils.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c7febc57..c842c839 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -9,6 +9,7 @@ import os import re from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import temp_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: return dhcp_discovery(dhclient_path, nic, tmpdir) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index e22409d1..28ed0ae2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -6,10 +6,10 @@ import os import re import socket import struct -import tempfile import time from cloudinit import stages +from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree @@ -111,7 +111,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.mkdtemp() + self.tmpdir = temp_utils.mkdtemp() self.certificate = None self.generate_certificate() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py new file mode 100644 index 00000000..0355f19d --- /dev/null +++ b/cloudinit/temp_utils.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import contextlib +import errno +import os +import shutil +import tempfile + +_TMPDIR = None +_ROOT_TMPDIR = "/run/cloud-init/tmp" + + +def _tempfile_dir_arg(odir=None): + """Return the proper 'dir' argument for tempfile functions. + + When root, cloud-init will use /run/cloud-init/tmp to avoid + any cleaning that a distro boot might do on /tmp (such as + systemd-tmpfiles-clean). + + If the caller of this function (mkdtemp or mkstemp) was provided + with a 'dir' argument, then that is respected. + + @param odir: original 'dir' arg to 'mkdtemp' or other.""" + + if odir is not None: + return odir + + global _TMPDIR + if _TMPDIR: + return _TMPDIR + + if os.getuid() == 0: + tdir = _ROOT_TMPDIR + else: + tdir = os.environ.get('TMPDIR', '/tmp') + if not os.path.isdir(tdir): + os.makedirs(tdir) + os.chmod(tdir, 0o1777) + + _TMPDIR = tdir + return tdir + + +def ExtendedTemporaryFile(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + + def _unlink_if_exists(path): + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + + fh.unlink = _unlink_if_exists + + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + + setattr(fh, 'unlink_now', unlink_now) + return fh + + +@contextlib.contextmanager +def tempdir(**kwargs): + # This seems like it was only added in python 3.2 + # Make it since its useful... + # See: http://bugs.python.org/file12970/tempdir.patch + tdir = mkdtemp(**kwargs) + try: + yield tdir + finally: + shutil.rmtree(tdir) + + +def mkdtemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkdtemp(**kwargs) + + +def mkstemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkstemp(**kwargs) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 609e94c8..ae5cda8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -30,7 +30,6 @@ import stat import string import subprocess import sys -import tempfile import time from errno import ENOENT, ENOEXEC @@ -45,6 +44,7 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import mergers from cloudinit import safeyaml +from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import url_helper from cloudinit import version @@ -349,26 +349,6 @@ class DecompressionError(Exception): pass -def ExtendedTemporaryFile(**kwargs): - fh = tempfile.NamedTemporaryFile(**kwargs) - # Replace its unlink with a quiet version - # that does not raise errors when the - # file to unlink has been unlinked elsewhere.. - LOG.debug("Created temporary file %s", fh.name) - fh.unlink = del_file - - # Add a new method that will unlink - # right 'now' but still lets the exit - # method attempt to remove it (which will - # not throw due to our del file being quiet - # about files that are not there) - def unlink_now(): - fh.unlink(fh.name) - - setattr(fh, 'unlink_now', unlink_now) - return fh - - def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: @@ -790,18 +770,6 @@ def umask(n_msk): os.umask(old) -@contextlib.contextmanager -def tempdir(**kwargs): - # This seems like it was only added in python 3.2 - # Make it since its useful... - # See: http://bugs.python.org/file12970/tempdir.patch - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - del_dir(tdir) - - def center(text, fill, max_len): return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len) @@ -1587,7 +1555,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): mtypes = [''] mounted = mounts() - with tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: umount = False if os.path.realpath(device) in mounted: mountpoint = mounted[os.path.realpath(device)]['mountpoint'] diff --git a/packages/bddeb b/packages/bddeb index 7c123548..4f2e2ddf 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -21,8 +21,9 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import templater from cloudinit import util + from cloudinit import temp_utils + from cloudinit import templater DEBUILD_ARGS = ["-S", "-d"] @@ -148,7 +149,7 @@ def main(): capture = False templ_data = {'debian_release': args.release} - with util.tempdir() as tdir: + with temp_utils.tempdir() as tdir: # output like 0.7.6-1022-g36e92d3 ver_data = read_version() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 80ce003d..44b99eca 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -275,7 +275,7 @@ class TestOpenSSLManager(TestCase): mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp') def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() self.assertEqual(mkdtemp.return_value, manager.tmpdir) @@ -292,7 +292,7 @@ class TestOpenSSLManager(TestCase): manager.clean_up() @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock()) @mock.patch.object(azure_helper.util, 'del_dir') def test_clean_up(self, del_dir): manager = azure_helper.OpenSSLManager() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c10ef905..f2496151 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -9,6 +9,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack +from cloudinit import temp_utils from cloudinit import util from cloudinit.tests.helpers import CiTestCase @@ -2150,7 +2151,7 @@ class TestCmdlineConfigParsing(CiTestCase): static['mac_address'] = macs['eth1'] expected = {'version': 1, 'config': [dhcp, static]} - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: for fname, content in pairs: fp = os.path.join(tmpd, fname) files.append(fp) -- cgit v1.2.3 From a1dfdda2a2ae20fe026881980ddf7d16110f06e2 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Thu, 7 Sep 2017 22:16:16 -0600 Subject: vmware customization: return network config format For customizing the machines hosted on 'VMWare' hypervisor, the datasource should return the 'network config' data in 'curtin' format. This branch also fixes /etc/network/interfaces replacing the line "source /etc/network/interfaces.d/*.cfg" which is incorrectly removed when VMWare's Perl Customization Engine writes /etc/network/interfaces. Modify the code to read the customization configuration and return the converted data. Added few tests. LP: #1675063 --- cloudinit/sources/DataSourceOVF.py | 91 ++++++--- cloudinit/sources/helpers/vmware/imc/config_nic.py | 201 ++++++++++++------- .../sources/helpers/vmware/imc/guestcust_util.py | 12 +- tests/unittests/test_vmware_config_file.py | 217 +++++++++++++++++++++ 4 files changed, 418 insertions(+), 103 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 73d38771..aa5f798d 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -51,6 +51,10 @@ class DataSourceOVF(sources.DataSource): self.cfg = {} self.supported_seed_starts = ("/", "file://") self.vmware_customization_supported = True + self._network_config = None + self._vmware_nics_to_enable = None + self._vmware_cust_conf = None + self._vmware_cust_found = False def __str__(self): root = sources.DataSource.__str__(self) @@ -60,8 +64,8 @@ class DataSourceOVF(sources.DataSource): found = [] md = {} ud = "" - vmwarePlatformFound = False - vmwareImcConfigFilePath = '' + vmwareImcConfigFilePath = None + nicspath = None defaults = { "instance-id": "iid-dsovf", @@ -101,25 +105,26 @@ class DataSourceOVF(sources.DataSource): logfunc=LOG.debug, msg="waiting for configuration file", func=wait_for_imc_cfg_file, - args=("/var/run/vmware-imc", "cust.cfg", max_wait)) + args=("cust.cfg", max_wait)) if vmwareImcConfigFilePath: LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) + nicspath = wait_for_imc_cfg_file( + filename="nics.txt", maxwait=10, naplen=5) else: LOG.debug("Did not find VMware Customization Config File") else: LOG.debug("Customization for VMware platform is disabled.") if vmwareImcConfigFilePath: - nics = "" + self._vmware_nics_to_enable = "" try: cf = ConfigFile(vmwareImcConfigFilePath) - conf = Config(cf) - (md, ud, cfg) = read_vmware_imc(conf) - dirpath = os.path.dirname(vmwareImcConfigFilePath) - nics = get_nics_to_enable(dirpath) - markerid = conf.marker_id + self._vmware_cust_conf = Config(cf) + (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) + self._vmware_nics_to_enable = get_nics_to_enable(nicspath) + markerid = self._vmware_cust_conf.marker_id markerexists = check_marker_exists(markerid) except Exception as e: LOG.debug("Error parsing the customization Config File") @@ -127,28 +132,29 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) - return False + raise e finally: util.del_dir(os.path.dirname(vmwareImcConfigFilePath)) try: - LOG.debug("Applying the Network customization") - nicConfigurator = NicConfigurator(conf.nics) - nicConfigurator.configure() + LOG.debug("Preparing the Network configuration") + self._network_config = get_network_config_from_conf( + self._vmware_cust_conf, + True, + True, + self.distro.osfamily) except Exception as e: - LOG.debug("Error applying the Network Configuration") LOG.exception(e) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) - enable_nics(nics) - return False + raise e + if markerid and not markerexists: LOG.debug("Applying password customization") pwdConfigurator = PasswordConfigurator() - adminpwd = conf.admin_password + adminpwd = self._vmware_cust_conf.admin_password try: - resetpwd = conf.reset_password + resetpwd = self._vmware_cust_conf.reset_password if adminpwd or resetpwd: pwdConfigurator.configure(adminpwd, resetpwd, self.distro) @@ -159,7 +165,6 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) return False if markerid: LOG.debug("Handle marker creation") @@ -170,14 +175,18 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) return False - vmwarePlatformFound = True + self._vmware_cust_found = True + found.append('vmware-tools') + + # TODO: Need to set the status to DONE only when the + # customization is done successfully. set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) - enable_nics(nics) + enable_nics(self._vmware_nics_to_enable) + else: np = {'iso': transport_iso9660, 'vmware-guestd': transport_vmware_guestd, } @@ -192,7 +201,7 @@ class DataSourceOVF(sources.DataSource): found.append(name) # There was no OVF transports found - if len(found) == 0 and not vmwarePlatformFound: + if len(found) == 0: return False if 'seedfrom' in md and md['seedfrom']: @@ -237,6 +246,10 @@ class DataSourceOVF(sources.DataSource): def get_config_obj(self): return self.cfg + @property + def network_config(self): + return self._network_config + class DataSourceOVFNet(DataSourceOVF): def __init__(self, sys_cfg, distro, paths): @@ -268,12 +281,13 @@ def get_max_wait_from_cfg(cfg): return max_wait -def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): +def wait_for_imc_cfg_file(filename, maxwait=180, naplen=5, + dirpath="/var/run/vmware-imc"): waited = 0 while waited < maxwait: - fileFullPath = search_file(dirpath, filename) - if fileFullPath: + fileFullPath = os.path.join(dirpath, filename) + if os.path.isfile(fileFullPath): return fileFullPath LOG.debug("Waiting for VMware Customization Config File") time.sleep(naplen) @@ -281,6 +295,26 @@ def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): return None +def get_network_config_from_conf(config, use_system_devices=True, + configure=False, osfamily=None): + nicConfigurator = NicConfigurator(config.nics, use_system_devices) + nics_cfg_list = nicConfigurator.generate(configure, osfamily) + + return get_network_config(nics_cfg_list, + config.name_servers, + config.dns_suffixes) + + +def get_network_config(nics=None, nameservers=None, search=None): + config_list = nics + + if nameservers or search: + config_list.append({'type': 'nameserver', 'address': nameservers, + 'search': search}) + + return {'version': 1, 'config': config_list} + + # This will return a dict with some content # meta-data, user-data, some config def read_vmware_imc(config): @@ -296,6 +330,9 @@ def read_vmware_imc(config): if config.timezone: cfg['timezone'] = config.timezone + # Generate a unique instance-id so that re-customization will + # happen in cloud-init + md['instance-id'] = "iid-vmware-" + util.rand_str(strlen=8) return (md, ud, cfg) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 67ac21db..2fb07c59 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -9,22 +9,48 @@ import logging import os import re +from cloudinit.net.network_state import mask_to_net_prefix from cloudinit import util logger = logging.getLogger(__name__) +def gen_subnet(ip, netmask): + """ + Return the subnet for a given ip address and a netmask + @return (str): the subnet + @param ip: ip address + @param netmask: netmask + """ + ip_array = ip.split(".") + mask_array = netmask.split(".") + result = [] + for index in list(range(4)): + result.append(int(ip_array[index]) & int(mask_array[index])) + + return ".".join([str(x) for x in result]) + + class NicConfigurator(object): - def __init__(self, nics): + def __init__(self, nics, use_system_devices=True): """ Initialize the Nic Configurator @param nics (list) an array of nics to configure + @param use_system_devices (Bool) Get the MAC names from the system + if this is True. If False, then mac names will be retrieved from + the specified nics. """ self.nics = nics self.mac2Name = {} self.ipv4PrimaryGateway = None self.ipv6PrimaryGateway = None - self.find_devices() + + if use_system_devices: + self.find_devices() + else: + for nic in self.nics: + self.mac2Name[nic.mac.lower()] = nic.name + self._primaryNic = self.get_primary_nic() def get_primary_nic(self): @@ -61,138 +87,163 @@ class NicConfigurator(object): def gen_one_nic(self, nic): """ - Return the lines needed to configure a nic - @return (str list): the string list to configure the nic + Return the config list needed to configure a nic + @return (list): the subnets and routes list to configure the nic @param nic (NicBase): the nic to configure """ - lines = [] - name = self.mac2Name.get(nic.mac.lower()) + mac = nic.mac.lower() + name = self.mac2Name.get(mac) if not name: raise ValueError('No known device has MACADDR: %s' % nic.mac) - if nic.onboot: - lines.append('auto %s' % name) + nics_cfg_list = [] + + cfg = {'type': 'physical', 'name': name, 'mac_address': mac} + + subnet_list = [] + route_list = [] # Customize IPv4 - lines.extend(self.gen_ipv4(name, nic)) + (subnets, routes) = self.gen_ipv4(name, nic) + subnet_list.extend(subnets) + route_list.extend(routes) # Customize IPv6 - lines.extend(self.gen_ipv6(name, nic)) + (subnets, routes) = self.gen_ipv6(name, nic) + subnet_list.extend(subnets) + route_list.extend(routes) + + cfg.update({'subnets': subnet_list}) - lines.append('') + nics_cfg_list.append(cfg) + if route_list: + nics_cfg_list.extend(route_list) - return lines + return nics_cfg_list def gen_ipv4(self, name, nic): """ - Return the lines needed to configure the IPv4 setting of a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic + Return the set of subnets and routes needed to configure the + IPv4 settings of a nic + @return (set): the set of subnet and routes to configure the gateways + @param name (str): subnet and route list for the nic @param nic (NicBase): the nic to configure """ - lines = [] + + subnet = {} + route_list = [] + + if nic.onboot: + subnet.update({'control': 'auto'}) bootproto = nic.bootProto.lower() if nic.ipv4_mode.lower() == 'disabled': bootproto = 'manual' - lines.append('iface %s inet %s' % (name, bootproto)) if bootproto != 'static': - return lines + subnet.update({'type': 'dhcp'}) + return ([subnet], route_list) + else: + subnet.update({'type': 'static'}) # Static Ipv4 addrs = nic.staticIpv4 if not addrs: - return lines + return ([subnet], route_list) v4 = addrs[0] if v4.ip: - lines.append(' address %s' % v4.ip) + subnet.update({'address': v4.ip}) if v4.netmask: - lines.append(' netmask %s' % v4.netmask) + subnet.update({'netmask': v4.netmask}) # Add the primary gateway if nic.primary and v4.gateways: self.ipv4PrimaryGateway = v4.gateways[0] - lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) - return lines + subnet.update({'gateway': self.ipv4PrimaryGateway}) + return [subnet] # Add routes if there is no primary nic if not self._primaryNic: - lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + route_list.extend(self.gen_ipv4_route(nic, + v4.gateways, + v4.netmask)) - return lines + return ([subnet], route_list) - def gen_ipv4_route(self, nic, gateways): + def gen_ipv4_route(self, nic, gateways, netmask): """ - Return the lines needed to configure additional Ipv4 route - @return (str list): the string list to configure the gateways + Return the routes list needed to configure additional Ipv4 route + @return (list): the route list to configure the gateways @param nic (NicBase): the nic to configure @param gateways (str list): the list of gateways """ - lines = [] + route_list = [] + + cidr = mask_to_net_prefix(netmask) for gateway in gateways: - lines.append(' up route add default gw %s metric 10000' % - gateway) + destination = "%s/%d" % (gen_subnet(gateway, netmask), cidr) + route_list.append({'destination': destination, + 'type': 'route', + 'gateway': gateway, + 'metric': 10000}) - return lines + return route_list def gen_ipv6(self, name, nic): """ - Return the lines needed to configure the gateways for a nic - @return (str list): the string list to configure the gateways + Return the set of subnets and routes needed to configure the + gateways for a nic + @return (set): the set of subnets and routes to configure the gateways @param name (str): name of the nic @param nic (NicBase): the nic to configure """ - lines = [] if not nic.staticIpv6: - return lines + return ([], []) + subnet_list = [] # Static Ipv6 addrs = nic.staticIpv6 - lines.append('iface %s inet6 static' % name) - lines.append(' address %s' % addrs[0].ip) - lines.append(' netmask %s' % addrs[0].netmask) - for addr in addrs[1:]: - lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, - addr.netmask)) - # Add the primary gateway - if nic.primary: - for addr in addrs: - if addr.gateway: - self.ipv6PrimaryGateway = addr.gateway - lines.append(' gateway %s' % self.ipv6PrimaryGateway) - return lines + for addr in addrs: + subnet = {'type': 'static6', + 'address': addr.ip, + 'netmask': addr.netmask} + subnet_list.append(subnet) - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self._genIpv6Route(name, nic, addrs)) + # TODO: Add the primary gateway + + route_list = [] + # TODO: Add routes if there is no primary nic + # if not self._primaryNic: + # route_list.extend(self._genIpv6Route(name, nic, addrs)) - return lines + return (subnet_list, route_list) def _genIpv6Route(self, name, nic, addrs): - lines = [] + route_list = [] for addr in addrs: - lines.append(' up route -A inet6 add default gw ' - '%s metric 10000' % addr.gateway) + route_list.append({'type': 'route', + 'gateway': addr.gateway, + 'metric': 10000}) + + return route_list - return lines + def generate(self, configure=False, osfamily=None): + """Return the config elements that are needed to configure the nics""" + if configure: + logger.info("Configuring the interfaces file") + self.configure(osfamily) - def generate(self): - """Return the lines that is needed to configure the nics""" - lines = [] - lines.append('iface lo inet loopback') - lines.append('auto lo') - lines.append('') + nics_cfg_list = [] for nic in self.nics: - lines.extend(self.gen_one_nic(nic)) + nics_cfg_list.extend(self.gen_one_nic(nic)) - return lines + return nics_cfg_list def clear_dhcp(self): logger.info('Clearing DHCP leases') @@ -201,11 +252,16 @@ class NicConfigurator(object): util.subp(["pkill", "dhclient"], rcs=[0, 1]) util.subp(["rm", "-f", "/var/lib/dhcp/*"]) - def configure(self): + def configure(self, osfamily=None): """ - Configure the /etc/network/intefaces + Configure the /etc/network/interfaces Make a back up of the original """ + + if not osfamily or osfamily != "debian": + logger.info("Debian OS not detected. Skipping the configure step") + return + containingDir = '/etc/network' interfaceFile = os.path.join(containingDir, 'interfaces') @@ -215,10 +271,13 @@ class NicConfigurator(object): if not os.path.exists(originalFile) and os.path.exists(interfaceFile): os.rename(interfaceFile, originalFile) - lines = self.generate() - with open(interfaceFile, 'w') as fp: - for line in lines: - fp.write('%s\n' % line) + lines = [ + "# DO NOT EDIT THIS FILE BY HAND --" + " AUTOMATICALLY GENERATED BY cloud-init", + "source /etc/network/interfaces.d/*.cfg", + ] + + util.write_file(interfaceFile, content='\n'.join(lines)) self.clear_dhcp() diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 1ab6bd41..44075255 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -59,14 +59,16 @@ def set_customization_status(custstate, custerror, errormessage=None): return (out, err) -# This will read the file nics.txt in the specified directory -# and return the content -def get_nics_to_enable(dirpath): - if not dirpath: +def get_nics_to_enable(nicsfilepath): + """Reads the NICS from the specified file path and returns the content + + @param nicsfilepath: Absolute file path to the NICS.txt file. + """ + + if not nicsfilepath: return None NICS_SIZE = 1024 - nicsfilepath = os.path.join(dirpath, "nics.txt") if not os.path.exists(nicsfilepath): return None diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index d8651077..808d303a 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,9 +8,13 @@ import logging import sys +from cloudinit.sources.DataSourceOVF import get_network_config_from_conf +from cloudinit.sources.DataSourceOVF import read_vmware_imc from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic import gen_subnet +from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) @@ -20,6 +24,7 @@ logger = logging.getLogger(__name__) class TestVmwareConfigFile(CiTestCase): def test_utility_methods(self): + """Tests basic utility methods of ConfigFile class""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") cf.clear() @@ -43,7 +48,26 @@ class TestVmwareConfigFile(CiTestCase): self.assertFalse(cf.should_keep_current_value("BAR"), "keepBar") self.assertTrue(cf.should_remove_current_value("BAR"), "removeBar") + def test_datasource_instance_id(self): + """Tests instance id for the DatasourceOVF""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + instance_id_prefix = 'iid-vmware-' + + conf = Config(cf) + + (md1, _, _) = read_vmware_imc(conf) + self.assertIn(instance_id_prefix, md1["instance-id"]) + self.assertEqual(len(md1["instance-id"]), len(instance_id_prefix) + 8) + + (md2, _, _) = read_vmware_imc(conf) + self.assertIn(instance_id_prefix, md2["instance-id"]) + self.assertEqual(len(md2["instance-id"]), len(instance_id_prefix) + 8) + + self.assertNotEqual(md1["instance-id"], md2["instance-id"]) + def test_configfile_static_2nics(self): + """Tests Config class for a configuration with two static NICs.""" cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") conf = Config(cf) @@ -81,6 +105,7 @@ class TestVmwareConfigFile(CiTestCase): self.assertTrue(not nics[1].staticIpv6, "ipv61 dhcp") def test_config_file_dhcp_2nics(self): + """Tests Config class for a configuration with two DHCP NICs.""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") conf = Config(cf) @@ -117,5 +142,197 @@ class TestVmwareConfigFile(CiTestCase): conf = Config(cf) self.assertTrue(conf.reset_password, "reset password") + def test_get_config_nameservers(self): + """Tests DNS and nameserver settings in a configuration.""" + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + config = Config(cf) + + network_config = get_network_config_from_conf(config, False) + + self.assertEqual(1, network_config.get('version')) + + config_types = network_config.get('config') + name_servers = None + dns_suffixes = None + + for type in config_types: + if type.get('type') == 'nameserver': + name_servers = type.get('address') + dns_suffixes = type.get('search') + break + + self.assertEqual(['10.20.145.1', '10.20.145.2'], + name_servers, + "dns") + self.assertEqual(['eng.vmware.com', 'proxy.vmware.com'], + dns_suffixes, + "suffixes") + + def test_gen_subnet(self): + """Tests if gen_subnet properly calculates network subnet from + IPv4 address and netmask""" + ip_subnet_list = [['10.20.87.253', '255.255.252.0', '10.20.84.0'], + ['10.20.92.105', '255.255.252.0', '10.20.92.0'], + ['192.168.0.10', '255.255.0.0', '192.168.0.0']] + for entry in ip_subnet_list: + self.assertEqual(entry[2], gen_subnet(entry[0], entry[1]), + "Subnet for a specified ip and netmask") + + def test_get_config_dns_suffixes(self): + """Tests if get_network_config_from_conf properly + generates nameservers and dns settings from a + specified configuration""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + config = Config(cf) + + network_config = get_network_config_from_conf(config, False) + + self.assertEqual(1, network_config.get('version')) + + config_types = network_config.get('config') + name_servers = None + dns_suffixes = None + + for type in config_types: + if type.get('type') == 'nameserver': + name_servers = type.get('address') + dns_suffixes = type.get('search') + break + + self.assertEqual([], + name_servers, + "dns") + self.assertEqual(['eng.vmware.com'], + dns_suffixes, + "suffixes") + + def test_get_nics_list_dhcp(self): + """Tests if NicConfigurator properly calculates network subnets + for a configuration with a list of DHCP NICs""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + config = Config(cf) + + nicConfigurator = NicConfigurator(config.nics, False) + nics_cfg_list = nicConfigurator.generate() + + self.assertEqual(2, len(nics_cfg_list), "number of config elements") + + nic1 = {'name': 'NIC1'} + nic2 = {'name': 'NIC2'} + for cfg in nics_cfg_list: + if cfg.get('name') == nic1.get('name'): + nic1.update(cfg) + elif cfg.get('name') == nic2.get('name'): + nic2.update(cfg) + + self.assertEqual('physical', nic1.get('type'), 'type of NIC1') + self.assertEqual('NIC1', nic1.get('name'), 'name of NIC1') + self.assertEqual('00:50:56:a6:8c:08', nic1.get('mac_address'), + 'mac address of NIC1') + subnets = nic1.get('subnets') + self.assertEqual(1, len(subnets), 'number of subnets for NIC1') + subnet = subnets[0] + self.assertEqual('dhcp', subnet.get('type'), 'DHCP type for NIC1') + self.assertEqual('auto', subnet.get('control'), 'NIC1 Control type') + + self.assertEqual('physical', nic2.get('type'), 'type of NIC2') + self.assertEqual('NIC2', nic2.get('name'), 'name of NIC2') + self.assertEqual('00:50:56:a6:5a:de', nic2.get('mac_address'), + 'mac address of NIC2') + subnets = nic2.get('subnets') + self.assertEqual(1, len(subnets), 'number of subnets for NIC2') + subnet = subnets[0] + self.assertEqual('dhcp', subnet.get('type'), 'DHCP type for NIC2') + self.assertEqual('auto', subnet.get('control'), 'NIC2 Control type') + + def test_get_nics_list_static(self): + """Tests if NicConfigurator properly calculates network subnets + for a configuration with 2 static NICs""" + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + config = Config(cf) + + nicConfigurator = NicConfigurator(config.nics, False) + nics_cfg_list = nicConfigurator.generate() + + self.assertEqual(5, len(nics_cfg_list), "number of elements") + + nic1 = {'name': 'NIC1'} + nic2 = {'name': 'NIC2'} + route_list = [] + for cfg in nics_cfg_list: + cfg_type = cfg.get('type') + if cfg_type == 'physical': + if cfg.get('name') == nic1.get('name'): + nic1.update(cfg) + elif cfg.get('name') == nic2.get('name'): + nic2.update(cfg) + elif cfg_type == 'route': + route_list.append(cfg) + + self.assertEqual('physical', nic1.get('type'), 'type of NIC1') + self.assertEqual('NIC1', nic1.get('name'), 'name of NIC1') + self.assertEqual('00:50:56:a6:8c:08', nic1.get('mac_address'), + 'mac address of NIC1') + + subnets = nic1.get('subnets') + self.assertEqual(2, len(subnets), 'Number of subnets') + + static_subnet = [] + static6_subnet = [] + + for subnet in subnets: + subnet_type = subnet.get('type') + if subnet_type == 'static': + static_subnet.append(subnet) + elif subnet_type == 'static6': + static6_subnet.append(subnet) + else: + self.assertEqual(True, False, 'Unknown type') + + self.assertEqual(1, len(static_subnet), 'Number of static subnet') + self.assertEqual(1, len(static6_subnet), 'Number of static6 subnet') + + subnet = static_subnet[0] + self.assertEqual('10.20.87.154', subnet.get('address'), + 'IPv4 address of static subnet') + self.assertEqual('255.255.252.0', subnet.get('netmask'), + 'NetMask of static subnet') + self.assertEqual('auto', subnet.get('control'), + 'control for static subnet') + + subnet = static6_subnet[0] + self.assertEqual('fc00:10:20:87::154', subnet.get('address'), + 'IPv6 address of static subnet') + self.assertEqual('64', subnet.get('netmask'), + 'NetMask of static6 subnet') + + route_set = set(['10.20.87.253', '10.20.87.105', '192.168.0.10']) + for route in route_list: + self.assertEqual(10000, route.get('metric'), 'metric of route') + gateway = route.get('gateway') + if gateway in route_set: + route_set.discard(gateway) + else: + self.assertEqual(True, False, 'invalid gateway %s' % (gateway)) + + self.assertEqual('physical', nic2.get('type'), 'type of NIC2') + self.assertEqual('NIC2', nic2.get('name'), 'name of NIC2') + self.assertEqual('00:50:56:a6:ef:7d', nic2.get('mac_address'), + 'mac address of NIC2') + + subnets = nic2.get('subnets') + self.assertEqual(1, len(subnets), 'Number of subnets for NIC2') + + subnet = subnets[0] + self.assertEqual('static', subnet.get('type'), 'Subnet type') + self.assertEqual('192.168.6.102', subnet.get('address'), + 'Subnet address') + self.assertEqual('255.255.0.0', subnet.get('netmask'), + 'Subnet netmask') + # vi: ts=4 expandtab -- cgit v1.2.3 From 9d2a87dc386b7aed1a8243d599676e78ed358749 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Sat, 30 Sep 2017 00:00:18 -0400 Subject: Azure, CloudStack: Support reading dhcp options from systemd-networkd. Systems that used systemd-networkd's dhcp client would not be able to get information on the Azure endpoint (placed in Option 245) or the CloudStack server (in 'server_address'). The change here supports reading these files in /run/systemd/netif/leases. The files declare that "This is private data. Do not parse.", but at this point we do not have another option. LP: #1718029 --- cloudinit/net/dhcp.py | 42 ++++++ cloudinit/net/tests/test_dhcp.py | 113 +++++++++++++++- cloudinit/sources/DataSourceCloudStack.py | 17 ++- cloudinit/sources/helpers/azure.py | 20 ++- .../unittests/test_datasource/test_azure_helper.py | 143 ++++++++++++++------- tests/unittests/test_datasource/test_cloudstack.py | 11 +- 6 files changed, 282 insertions(+), 64 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 05350639..0cba7032 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -4,6 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import configobj import logging import os import re @@ -11,9 +12,12 @@ import re from cloudinit.net import find_fallback_nic, get_devicelist from cloudinit import temp_utils from cloudinit import util +from six import StringIO LOG = logging.getLogger(__name__) +NETWORKD_LEASES_DIR = '/run/systemd/netif/leases' + class InvalidDHCPLeaseFileError(Exception): """Raised when parsing an empty or invalid dhcp.leases file. @@ -118,4 +122,42 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): return parse_dhcp_lease_file(lease_file) +def networkd_parse_lease(content): + """Parse a systemd lease file content as in /run/systemd/netif/leases/ + + Parse this (almost) ini style file even though it says: + # This is private data. Do not parse. + + Simply return a dictionary of key/values.""" + + return dict(configobj.ConfigObj(StringIO(content), list_values=False)) + + +def networkd_load_leases(leases_d=None): + """Return a dictionary of dictionaries representing each lease + found in lease_d.i + + The top level key will be the filename, which is typically the ifindex.""" + + if leases_d is None: + leases_d = NETWORKD_LEASES_DIR + + ret = {} + if not os.path.isdir(leases_d): + return ret + for lfile in os.listdir(leases_d): + ret[lfile] = networkd_parse_lease( + util.load_file(os.path.join(leases_d, lfile))) + return ret + + +def networkd_get_option_from_leases(keyname, leases_d=None): + if leases_d is None: + leases_d = NETWORKD_LEASES_DIR + leases = networkd_load_leases(leases_d=leases_d) + for ifindex, data in sorted(leases.items()): + if data.get(keyname): + return data[keyname] + return None + # vi: ts=4 expandtab diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index a38edaec..1c1f504a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -6,9 +6,9 @@ from textwrap import dedent from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, - parse_dhcp_lease_file, dhcp_discovery) + parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases) from cloudinit.util import ensure_file, write_file -from cloudinit.tests.helpers import CiTestCase, wrap_and_call +from cloudinit.tests.helpers import CiTestCase, wrap_and_call, populate_dir class TestParseDHCPLeasesFile(CiTestCase): @@ -149,3 +149,112 @@ class TestDHCPDiscoveryClean(CiTestCase): [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf', lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), 'eth9', '-sf', '/bin/true'], capture=True)]) + + +class TestSystemdParseLeases(CiTestCase): + + lxd_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.75.205.242 + NETMASK=255.255.255.0 + ROUTER=10.75.205.1 + SERVER_ADDRESS=10.75.205.1 + NEXT_SERVER=10.75.205.1 + BROADCAST=10.75.205.255 + T1=1580 + T2=2930 + LIFETIME=3600 + DNS=10.75.205.1 + DOMAINNAME=lxd + HOSTNAME=a1 + CLIENTID=ffe617693400020000ab110c65a6a0866931c2 + """) + + lxd_parsed = { + 'ADDRESS': '10.75.205.242', + 'NETMASK': '255.255.255.0', + 'ROUTER': '10.75.205.1', + 'SERVER_ADDRESS': '10.75.205.1', + 'NEXT_SERVER': '10.75.205.1', + 'BROADCAST': '10.75.205.255', + 'T1': '1580', + 'T2': '2930', + 'LIFETIME': '3600', + 'DNS': '10.75.205.1', + 'DOMAINNAME': 'lxd', + 'HOSTNAME': 'a1', + 'CLIENTID': 'ffe617693400020000ab110c65a6a0866931c2', + } + + azure_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.132.0.5 + NETMASK=255.255.255.255 + ROUTER=10.132.0.1 + SERVER_ADDRESS=169.254.169.254 + NEXT_SERVER=10.132.0.1 + MTU=1460 + T1=43200 + T2=75600 + LIFETIME=86400 + DNS=169.254.169.254 + NTP=169.254.169.254 + DOMAINNAME=c.ubuntu-foundations.internal + DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal + HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal + ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 + CLIENTID=ff405663a200020000ab11332859494d7a8b4c + OPTION_245=624c3620 + """) + + azure_parsed = { + 'ADDRESS': '10.132.0.5', + 'NETMASK': '255.255.255.255', + 'ROUTER': '10.132.0.1', + 'SERVER_ADDRESS': '169.254.169.254', + 'NEXT_SERVER': '10.132.0.1', + 'MTU': '1460', + 'T1': '43200', + 'T2': '75600', + 'LIFETIME': '86400', + 'DNS': '169.254.169.254', + 'NTP': '169.254.169.254', + 'DOMAINNAME': 'c.ubuntu-foundations.internal', + 'DOMAIN_SEARCH_LIST': 'c.ubuntu-foundations.internal google.internal', + 'HOSTNAME': 'tribaal-test-171002-1349.c.ubuntu-foundations.internal', + 'ROUTES': '10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1', + 'CLIENTID': 'ff405663a200020000ab11332859494d7a8b4c', + 'OPTION_245': '624c3620'} + + def setUp(self): + super(TestSystemdParseLeases, self).setUp() + self.lease_d = self.tmp_dir() + + def test_no_leases_returns_empty_dict(self): + """A leases dir with no lease files should return empty dictionary.""" + self.assertEqual({}, networkd_load_leases(self.lease_d)) + + def test_no_leases_dir_returns_empty_dict(self): + """A non-existing leases dir should return empty dict.""" + enodir = os.path.join(self.lease_d, 'does-not-exist') + self.assertEqual({}, networkd_load_leases(enodir)) + + def test_single_leases_file(self): + """A leases dir with one leases file.""" + populate_dir(self.lease_d, {'2': self.lxd_lease}) + self.assertEqual( + {'2': self.lxd_parsed}, networkd_load_leases(self.lease_d)) + + def test_single_azure_leases_file(self): + """On Azure, option 245 should be present, verify it specifically.""" + populate_dir(self.lease_d, {'1': self.azure_lease}) + self.assertEqual( + {'1': self.azure_parsed}, networkd_load_leases(self.lease_d)) + + def test_multiple_files(self): + """Multiple leases files on azure with one found return that value.""" + self.maxDiff = None + populate_dir(self.lease_d, {'1': self.azure_lease, + '9': self.lxd_lease}) + self.assertEqual({'1': self.azure_parsed, '9': self.lxd_parsed}, + networkd_load_leases(self.lease_d)) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7e0f9bb8..9dc473fc 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -19,6 +19,7 @@ import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util @@ -224,20 +225,28 @@ def get_vr_address(): # Get the address of the virtual router via dhcp leases # If no virtual router is detected, fallback on default gateway. # See http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/4.8/virtual_machines/user-data.html # noqa + + # Try networkd first... + latest_address = dhcp.networkd_get_option_from_leases('SERVER_ADDRESS') + if latest_address: + LOG.debug("Found SERVER_ADDRESS '%s' via networkd_leases", + latest_address) + return latest_address + + # Try dhcp lease files next... lease_file = get_latest_lease() if not lease_file: LOG.debug("No lease file found, using default gateway") return get_default_gateway() - latest_address = None with open(lease_file, "r") as fd: for line in fd: if "dhcp-server-identifier" in line: words = line.strip(" ;\r\n").split(" ") if len(words) > 2: - dhcp = words[2] - LOG.debug("Found DHCP identifier %s", dhcp) - latest_address = dhcp + dhcptok = words[2] + LOG.debug("Found DHCP identifier %s", dhcptok) + latest_address = dhcptok if not latest_address: # No virtual router found, fallback on default gateway LOG.debug("No DHCP found, using default gateway") diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 28ed0ae2..959b1bda 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -8,6 +8,7 @@ import socket import struct import time +from cloudinit.net import dhcp from cloudinit import stages from cloudinit import temp_utils from contextlib import contextmanager @@ -15,7 +16,6 @@ from xml.etree import ElementTree from cloudinit import util - LOG = logging.getLogger(__name__) @@ -238,6 +238,11 @@ class WALinuxAgentShim(object): packed_bytes = unescaped_value.encode('utf-8') return socket.inet_ntoa(packed_bytes) + @staticmethod + def _networkd_get_value_from_leases(leases_d=None): + return dhcp.networkd_get_option_from_leases( + 'OPTION_245', leases_d=leases_d) + @staticmethod def _get_value_from_leases_file(fallback_lease_file): leases = [] @@ -287,12 +292,15 @@ class WALinuxAgentShim(object): @staticmethod def find_endpoint(fallback_lease_file=None): - LOG.debug('Finding Azure endpoint...') value = None - # Option-245 stored in /run/cloud-init/dhclient.hooks/.json - # a dhclient exit hook that calls cloud-init-dhclient-hook - dhcp_options = WALinuxAgentShim._load_dhclient_json() - value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) + LOG.debug('Finding Azure endpoint from networkd...') + value = WALinuxAgentShim._networkd_get_value_from_leases() + if value is None: + # Option-245 stored in /run/cloud-init/dhclient.hooks/.json + # a dhclient exit hook that calls cloud-init-dhclient-hook + LOG.debug('Finding Azure endpoint from hook json...') + dhcp_options = WALinuxAgentShim._load_dhclient_json() + value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) if value is None: # Fallback and check the leases file if unsuccessful LOG.debug("Unable to find endpoint in dhclient logs. " diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 44b99eca..b42b073f 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,10 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import os +from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper -from cloudinit.tests.helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, populate_dir +from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim GOAL_STATE_TEMPLATE = """\ @@ -45,7 +47,7 @@ GOAL_STATE_TEMPLATE = """\ """ -class TestFindEndpoint(TestCase): +class TestFindEndpoint(CiTestCase): def setUp(self): super(TestFindEndpoint, self).setUp() @@ -56,18 +58,19 @@ class TestFindEndpoint(TestCase): mock.patch.object(azure_helper.util, 'load_file')) self.dhcp_options = patches.enter_context( - mock.patch.object(azure_helper.WALinuxAgentShim, - '_load_dhclient_json')) + mock.patch.object(wa_shim, '_load_dhclient_json')) + + self.networkd_leases = patches.enter_context( + mock.patch.object(wa_shim, '_networkd_get_value_from_leases')) + self.networkd_leases.return_value = None def test_missing_file(self): - self.assertRaises(ValueError, - azure_helper.WALinuxAgentShim.find_endpoint) + self.assertRaises(ValueError, wa_shim.find_endpoint) def test_missing_special_azure_line(self): self.load_file.return_value = '' self.dhcp_options.return_value = {'eth0': {'key': 'value'}} - self.assertRaises(ValueError, - azure_helper.WALinuxAgentShim.find_endpoint) + self.assertRaises(ValueError, wa_shim.find_endpoint) @staticmethod def _build_lease_content(encoded_address): @@ -80,8 +83,7 @@ class TestFindEndpoint(TestCase): def test_from_dhcp_client(self): self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}} - self.assertEqual('5.4.3.2', - azure_helper.WALinuxAgentShim.find_endpoint(None)) + self.assertEqual('5.4.3.2', wa_shim.find_endpoint(None)) def test_latest_lease_used(self): encoded_addresses = ['5:4:3:2', '4:3:2:1'] @@ -89,53 +91,38 @@ class TestFindEndpoint(TestCase): for encoded_address in encoded_addresses]) self.load_file.return_value = file_content self.assertEqual(encoded_addresses[-1].replace(':', '.'), - azure_helper.WALinuxAgentShim.find_endpoint("foobar")) + wa_shim.find_endpoint("foobar")) -class TestExtractIpAddressFromLeaseValue(TestCase): +class TestExtractIpAddressFromLeaseValue(CiTestCase): def test_hex_string(self): ip_address, encoded_address = '98.76.54.32', '62:4c:36:20' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_hex_string_with_single_character_part(self): ip_address, encoded_address = '4.3.2.1', '4:3:2:1' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string(self): ip_address, encoded_address = '98.76.54.32', 'bL6 ' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string_with_escaped_quote(self): ip_address, encoded_address = '100.72.34.108', 'dH\\"l' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string_containing_a_colon(self): ip_address, encoded_address = '100.72.58.108', 'dH:l' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) -class TestGoalStateParsing(TestCase): +class TestGoalStateParsing(CiTestCase): default_parameters = { 'incarnation': 1, @@ -195,7 +182,7 @@ class TestGoalStateParsing(TestCase): self.assertIsNone(certificates_xml) -class TestAzureEndpointHttpClient(TestCase): +class TestAzureEndpointHttpClient(CiTestCase): regular_headers = { 'x-ms-agent-name': 'WALinuxAgent', @@ -258,7 +245,7 @@ class TestAzureEndpointHttpClient(TestCase): self.read_file_or_url.call_args) -class TestOpenSSLManager(TestCase): +class TestOpenSSLManager(CiTestCase): def setUp(self): super(TestOpenSSLManager, self).setUp() @@ -300,7 +287,7 @@ class TestOpenSSLManager(TestCase): self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list) -class TestWALinuxAgentShim(TestCase): +class TestWALinuxAgentShim(CiTestCase): def setUp(self): super(TestWALinuxAgentShim, self).setUp() @@ -310,8 +297,7 @@ class TestWALinuxAgentShim(TestCase): self.AzureEndpointHttpClient = patches.enter_context( mock.patch.object(azure_helper, 'AzureEndpointHttpClient')) self.find_endpoint = patches.enter_context( - mock.patch.object( - azure_helper.WALinuxAgentShim, 'find_endpoint')) + mock.patch.object(wa_shim, 'find_endpoint')) self.GoalState = patches.enter_context( mock.patch.object(azure_helper, 'GoalState')) self.OpenSSLManager = patches.enter_context( @@ -320,7 +306,7 @@ class TestWALinuxAgentShim(TestCase): mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) def test_http_client_uses_certificate(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.OpenSSLManager.return_value.certificate)], @@ -328,7 +314,7 @@ class TestWALinuxAgentShim(TestCase): def test_correct_url_used_for_goalstate(self): self.find_endpoint.return_value = 'test_endpoint' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() get = self.AzureEndpointHttpClient.return_value.get self.assertEqual( @@ -340,7 +326,7 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() data = shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.GoalState.return_value.certificates_xml)], @@ -351,13 +337,13 @@ class TestWALinuxAgentShim(TestCase): def test_absent_certificates_produces_empty_public_keys(self): self.GoalState.return_value.certificates_xml = None - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() data = shim.register_with_azure_and_fetch_data() self.assertEqual([], data['public-keys']) def test_correct_url_used_for_report_ready(self): self.find_endpoint.return_value = 'test_endpoint' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() expected_url = 'http://test_endpoint/machine?comp=health' self.assertEqual( @@ -368,7 +354,7 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.return_value.incarnation = 'TestIncarnation' self.GoalState.return_value.container_id = 'TestContainerId' self.GoalState.return_value.instance_id = 'TestInstanceId' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]['data'] @@ -378,11 +364,11 @@ class TestWALinuxAgentShim(TestCase): self.assertIn('TestInstanceId', posted_document) def test_clean_up_can_be_called_at_any_time(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.clean_up() def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() shim.clean_up() self.assertEqual( @@ -393,12 +379,12 @@ class TestWALinuxAgentShim(TestCase): pass self.AzureEndpointHttpClient.return_value.get.side_effect = ( SentinelException) - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() self.assertRaises(SentinelException, shim.register_with_azure_and_fetch_data) -class TestGetMetadataFromFabric(TestCase): +class TestGetMetadataFromFabric(CiTestCase): @mock.patch.object(azure_helper, 'WALinuxAgentShim') def test_data_from_shim_returned(self, shim): @@ -422,4 +408,65 @@ class TestGetMetadataFromFabric(TestCase): azure_helper.get_metadata_from_fabric) self.assertEqual(1, shim.return_value.clean_up.call_count) + +class TestExtractIpAddressFromNetworkd(CiTestCase): + + azure_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.132.0.5 + NETMASK=255.255.255.255 + ROUTER=10.132.0.1 + SERVER_ADDRESS=169.254.169.254 + NEXT_SERVER=10.132.0.1 + MTU=1460 + T1=43200 + T2=75600 + LIFETIME=86400 + DNS=169.254.169.254 + NTP=169.254.169.254 + DOMAINNAME=c.ubuntu-foundations.internal + DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal + HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal + ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 + CLIENTID=ff405663a200020000ab11332859494d7a8b4c + OPTION_245=624c3620 + """) + + def setUp(self): + super(TestExtractIpAddressFromNetworkd, self).setUp() + self.lease_d = self.tmp_dir() + + def test_no_valid_leases_is_none(self): + """No valid leases should return None.""" + self.assertIsNone( + wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_option_245_is_found_in_single(self): + """A single valid lease with 245 option should return it.""" + populate_dir(self.lease_d, {'9': self.azure_lease}) + self.assertEqual( + '624c3620', wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_option_245_not_found_returns_None(self): + """A valid lease, but no option 245 should return None.""" + populate_dir( + self.lease_d, + {'9': self.azure_lease.replace("OPTION_245", "OPTION_999")}) + self.assertIsNone( + wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_multiple_returns_first(self): + """Somewhat arbitrarily return the first address when multiple. + + Most important at the moment is that this is consistent behavior + rather than changing randomly as in order of a dictionary.""" + myval = "624c3601" + populate_dir( + self.lease_d, + {'9': self.azure_lease, + '2': self.azure_lease.replace("624c3620", myval)}) + self.assertEqual( + myval, wa_shim._networkd_get_value_from_leases(self.lease_d)) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 8e98e1bb..96144b64 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -23,13 +23,16 @@ class TestCloudStackPasswordFetching(CiTestCase): default_gw = "192.201.20.0" get_latest_lease = mock.MagicMock(return_value=None) self.patches.enter_context(mock.patch( - 'cloudinit.sources.DataSourceCloudStack.get_latest_lease', - get_latest_lease)) + mod_name + '.get_latest_lease', get_latest_lease)) get_default_gw = mock.MagicMock(return_value=default_gw) self.patches.enter_context(mock.patch( - 'cloudinit.sources.DataSourceCloudStack.get_default_gateway', - get_default_gw)) + mod_name + '.get_default_gateway', get_default_gw)) + + get_networkd_server_address = mock.MagicMock(return_value=None) + self.patches.enter_context(mock.patch( + mod_name + '.dhcp.networkd_get_option_from_leases', + get_networkd_server_address)) def _set_password_server_response(self, response_string): subp = mock.MagicMock(return_value=(response_string, '')) -- cgit v1.2.3