From 512145cd16b0dfa0cbbe8a20d732e6f2d943b869 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Jul 2017 17:18:22 -0400 Subject: archlinux: Fix bug with empty dns, do not render 'lo' devices. If no dns nameservers were provided a stack trace would occur. The changes here add some unit tests for the arch distro. Also avoids rendering an 'lo' interface. LP: #1663045 LP: #1706593 --- cloudinit/distros/arch.py | 90 ++++++++++++++++++++----------- tests/unittests/test_distros/__init__.py | 21 ++++++++ tests/unittests/test_distros/test_arch.py | 45 ++++++++++++++++ 3 files changed, 125 insertions(+), 31 deletions(-) create mode 100644 tests/unittests/test_distros/test_arch.py diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index b4c0ba72..f87a3432 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -14,6 +14,8 @@ from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit.settings import PER_INSTANCE +import os + LOG = logging.getLogger(__name__) @@ -52,31 +54,10 @@ class Distro(distros.Distro): entries = net_util.translate_network(settings) LOG.debug("Translated ubuntu style network settings %s into %s", settings, entries) - dev_names = entries.keys() - # Format for netctl - for (dev, info) in entries.items(): - nameservers = [] - net_fn = self.network_conf_dir + dev - net_cfg = { - 'Connection': 'ethernet', - 'Interface': dev, - 'IP': info.get('bootproto'), - 'Address': "('%s/%s')" % (info.get('address'), - info.get('netmask')), - 'Gateway': info.get('gateway'), - 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') - } - util.write_file(net_fn, convert_netctl(net_cfg)) - if info.get('auto'): - self._enable_interface(dev) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - - if nameservers: - util.write_file(self.resolve_conf_fn, - convert_resolv_conf(nameservers)) - - return dev_names + return _render_network( + entries, resolv_conf=self.resolve_conf_fn, + conf_dir=self.network_conf_dir, + enable_func=self._enable_interface) def _enable_interface(self, device_name): cmd = ['netctl', 'reenable', device_name] @@ -173,13 +154,60 @@ class Distro(distros.Distro): ["-y"], freq=PER_INSTANCE) +def _render_network(entries, target="/", conf_dir="etc/netctl", + resolv_conf="etc/resolv.conf", enable_func=None): + """Render the translate_network format into netctl files in target. + Paths will be rendered under target. + """ + + devs = [] + nameservers = [] + resolv_conf = util.target_path(target, resolv_conf) + conf_dir = util.target_path(target, conf_dir) + + for (dev, info) in entries.items(): + if dev == 'lo': + # no configuration should be rendered for 'lo' + continue + devs.append(dev) + net_fn = os.path.join(conf_dir, dev) + net_cfg = { + 'Connection': 'ethernet', + 'Interface': dev, + 'IP': info.get('bootproto'), + 'Address': "%s/%s" % (info.get('address'), + info.get('netmask')), + 'Gateway': info.get('gateway'), + 'DNS': info.get('dns-nameservers', []), + } + util.write_file(net_fn, convert_netctl(net_cfg)) + if enable_func and info.get('auto'): + enable_func(dev) + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + + if nameservers: + util.write_file(resolv_conf, + convert_resolv_conf(nameservers)) + return devs + + def convert_netctl(settings): - """Returns a settings string formatted for netctl.""" - result = '' - if isinstance(settings, dict): - for k, v in settings.items(): - result = result + '%s=%s\n' % (k, v) - return result + """Given a dictionary, returns a string in netctl profile format. + + netctl profile is described at: + https://git.archlinux.org/netctl.git/tree/docs/netctl.profile.5.txt + + Note that the 'Special Quoting Rules' are not handled here.""" + result = [] + for key in sorted(settings): + val = settings[key] + if val is None: + val = "" + elif isinstance(val, (tuple, list)): + val = "(" + ' '.join("'%s'" % v for v in val) + ")" + result.append("%s=%s\n" % (key, val)) + return ''.join(result) def convert_resolv_conf(settings): diff --git a/tests/unittests/test_distros/__init__.py b/tests/unittests/test_distros/__init__.py index e69de29b..5394aa56 100644 --- a/tests/unittests/test_distros/__init__.py +++ b/tests/unittests/test_distros/__init__.py @@ -0,0 +1,21 @@ +# This file is part of cloud-init. See LICENSE file for license information. +import copy + +from cloudinit import distros +from cloudinit import helpers +from cloudinit import settings + + +def _get_distro(dtype, system_info=None): + """Return a Distro class of distro 'dtype'. + + cfg is format of CFG_BUILTIN['system_info']. + + example: _get_distro("debian") + """ + if system_info is None: + system_info = copy.deepcopy(settings.CFG_BUILTIN['system_info']) + system_info['distro'] = dtype + paths = helpers.Paths(system_info['paths']) + distro_cls = distros.fetch(dtype) + return distro_cls(dtype, system_info, paths) diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py new file mode 100644 index 00000000..3d4c9a70 --- /dev/null +++ b/tests/unittests/test_distros/test_arch.py @@ -0,0 +1,45 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros.arch import _render_network +from cloudinit import util + +from ..helpers import (CiTestCase, dir2dict) + +from . import _get_distro + + +class TestArch(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("arch") + hostname = "myhostname" + hostfile = self.tmp_path("hostfile") + distro._write_hostname(hostname, hostfile) + self.assertEqual(hostname + "\n", util.load_file(hostfile)) + + +class TestRenderNetwork(CiTestCase): + def test_basic_static(self): + """Just the most basic static config. + + note 'lo' should not be rendered as an interface.""" + entries = {'eth0': {'auto': True, + 'dns-nameservers': ['8.8.8.8'], + 'bootproto': 'static', + 'address': '10.0.0.2', + 'gateway': '10.0.0.1', + 'netmask': '255.255.255.0'}, + 'lo': {'auto': True}} + target = self.tmp_dir() + devs = _render_network(entries, target=target) + files = dir2dict(target, prefix=target) + self.assertEqual(['eth0'], devs) + self.assertEqual( + {'/etc/netctl/eth0': '\n'.join([ + "Address=10.0.0.2/255.255.255.0", + "Connection=ethernet", + "DNS=('8.8.8.8')", + "Gateway=10.0.0.1", + "IP=static", + "Interface=eth0", ""]), + '/etc/resolv.conf': 'nameserver 8.8.8.8\n'}, files) -- cgit v1.2.3 From 56103567fbf486625cdf5bfe40eea5ddcb7e8e04 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 31 Jul 2017 13:35:07 -0500 Subject: sysconfig: Dont repeat header when rendering resolv.conf The sysconfig renderer duplicates the cloud-init header string when rendering resolv.conf file. This leads to resolv.conf file growing with every reboot of a system. Fix this by checking for the header when loading content from existing file. Update one of the sysconfig unittests with multiple render calls to simulate the reboot to check that we don't repeat the header. LP: #1701420 --- cloudinit/net/sysconfig.py | 6 +++++- tests/unittests/test_net.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index a550f97c..f5727969 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -484,7 +484,11 @@ class Renderer(renderer.Renderer): content.add_nameserver(nameserver) for searchdomain in network_state.dns_searchdomains: content.add_search_domain(searchdomain) - return "\n".join([_make_header(';'), str(content)]) + header = _make_header(';') + content_str = str(content) + if not content_str.startswith(header): + content_str = header + '\n' + content_str + return content_str @staticmethod def _render_networkmanager_conf(network_state): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index e49abcc4..4653be1a 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1683,6 +1683,9 @@ USERCTL=no ns = network_state.parse_net_config_data(network_cfg, skip_broken=False) renderer = sysconfig.Renderer() + # render a multiple times to simulate reboots + renderer.render_network_state(ns, render_dir) + renderer.render_network_state(ns, render_dir) renderer.render_network_state(ns, render_dir) for fn, expected_content in os_sample.get('out_sysconfig', []): with open(os.path.join(render_dir, fn)) as fh: -- cgit v1.2.3 From 9d0fdf1c6d39f8b6ff0f9e0172318bece56fed06 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Fri, 28 Jul 2017 13:24:37 -0700 Subject: tests: Fix build tree integration tests The build deb command was no longer working becasue it had assumed that you were in the root of the cloud-init directory. This changes where the deb is built and changes how the dependencies are determined as well as uses the built-in tools for determining build dependencies. --- tests/cloud_tests/bddeb.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index 53dbf74e..fe805356 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -11,7 +11,7 @@ from tests.cloud_tests import (config, LOG) from tests.cloud_tests import (platforms, images, snapshots, instances) from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) -build_deps = ['devscripts', 'equivs', 'git', 'tar'] +pre_reqs = ['devscripts', 'equivs', 'git', 'tar'] def _out(cmd_res): @@ -26,13 +26,10 @@ def build_deb(args, instance): @return_value: tuple of results and fail count """ # update remote system package list and install build deps - LOG.debug('installing build deps') - pkgs = ' '.join(build_deps) + LOG.debug('installing pre-reqs') + pkgs = ' '.join(pre_reqs) cmd = 'apt-get update && apt-get install --yes {}'.format(pkgs) instance.execute(['/bin/sh', '-c', cmd]) - # TODO Remove this call once we have a ci-deps Makefile target - instance.execute(['mk-build-deps', '--install', '-t', - 'apt-get --no-install-recommends --yes', 'cloud-init']) # local tmpfile that must be deleted local_tarball = tempfile.NamedTemporaryFile().name @@ -40,7 +37,7 @@ def build_deb(args, instance): # paths to use in remote system output_link = '/root/cloud-init_all.deb' remote_tarball = _out(instance.execute(['mktemp'])) - extract_dir = _out(instance.execute(['mktemp', '--directory'])) + extract_dir = '/root' bddeb_path = os.path.join(extract_dir, 'packages', 'bddeb') git_env = {'GIT_DIR': os.path.join(extract_dir, '.git'), 'GIT_WORK_TREE': extract_dir} @@ -56,6 +53,11 @@ def build_deb(args, instance): instance.execute(['git', 'commit', '-a', '-m', 'tmp', '--allow-empty'], env=git_env) + LOG.debug('installing deps') + deps_path = os.path.join(extract_dir, 'tools', 'read-dependencies') + instance.execute([deps_path, '--install', '--test-distro', + '--distro', 'ubuntu', '--python-version', '3']) + LOG.debug('building deb in remote system at: %s', output_link) bddeb_args = args.bddeb_args.split() if args.bddeb_args else [] instance.execute([bddeb_path, '-d'] + bddeb_args, env=git_env) -- cgit v1.2.3 From 9d923c1ab9c4556b980509513ece4a414269b5b9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Aug 2017 16:38:09 -0400 Subject: net: Reduce duplicate code. Have get_interfaces_by_mac use get_interfaces. get_interfaces_by_mac and get_interfaces just looked much alike. This makes get_interfaces_by_mac call get_interfaces. --- cloudinit/net/__init__.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 46cb9c85..1ff8fae0 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -511,21 +511,7 @@ def get_interfaces_by_mac(): Bridges and any devices that have a 'stolen' mac are excluded.""" ret = {} - devs = get_devicelist() - empty_mac = '00:00:00:00:00:00' - for name in devs: - if not interface_has_own_mac(name): - continue - if is_bridge(name): - continue - if is_vlan(name): - continue - mac = get_interface_mac(name) - # some devices may not have a mac (tun0) - if not mac: - continue - if mac == empty_mac and name != 'lo': - continue + for name, mac, _driver, _devid in get_interfaces(): if mac in ret: raise RuntimeError( "duplicate mac found! both '%s' and '%s' have mac '%s'" % -- cgit v1.2.3 From 5bba5db2655d88b8aba8fa06b30f8e91e2ca6836 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 1 Aug 2017 18:00:00 -0500 Subject: cc_ntp: fallback on timesyncd configuration if ntp is not installable Some systems like Ubuntu-Core do not provide an ntp package for installation but do include systemd-timesyncd (an ntp client). On such systems cloud-init will generate a timesyncd configuration using the 'servers' and 'pools' values as ntp hosts for timesyncd to use. LP: #1686485 --- cloudinit/config/cc_ntp.py | 58 ++++++++++--- templates/timesyncd.conf.tmpl | 8 ++ tests/unittests/test_handler/test_handler_ntp.py | 105 ++++++++++++++++++++++- 3 files changed, 154 insertions(+), 17 deletions(-) create mode 100644 templates/timesyncd.conf.tmpl diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 31ed64e3..a02b4bf1 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -50,6 +50,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' +TIMESYNCD_CONF = '/etc/systemd/timesyncd.conf.d/cloud-init.conf' NR_POOL_SERVERS = 4 distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu'] @@ -132,20 +133,50 @@ def handle(name, cfg, cloud, log, _args): " is a %s %instead"), type_utils.obj_name(ntp_cfg)) validate_cloudconfig_schema(cfg, schema) + if ntp_installable(): + service_name = 'ntp' + confpath = NTP_CONF + template_name = None + packages = ['ntp'] + check_exe = 'ntpd' + else: + service_name = 'systemd-timesyncd' + confpath = TIMESYNCD_CONF + template_name = 'timesyncd.conf' + packages = [] + check_exe = '/lib/systemd/systemd-timesyncd' + rename_ntp_conf() # ensure when ntp is installed it has a configuration file # to use instead of starting up with packaged defaults - write_ntp_config_template(ntp_cfg, cloud) - install_ntp(cloud.distro.install_packages, packages=['ntp'], - check_exe="ntpd") - # if ntp was already installed, it may not have started + write_ntp_config_template(ntp_cfg, cloud, confpath, template=template_name) + install_ntp(cloud.distro.install_packages, packages=packages, + check_exe=check_exe) + try: - reload_ntp(systemd=cloud.distro.uses_systemd()) + reload_ntp(service_name, systemd=cloud.distro.uses_systemd()) except util.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise +def ntp_installable(): + """Check if we can install ntp package + + Ubuntu-Core systems do not have an ntp package available, so + we always return False. Other systems require package managers to install + the ntp package If we fail to find one of the package managers, then we + cannot install ntp. + """ + if util.system_is_snappy(): + return False + + if any(map(util.which, ['apt-get', 'dnf', 'yum', 'zypper'])): + return True + + return False + + def install_ntp(install_func, packages=None, check_exe="ntpd"): if util.which(check_exe): return @@ -156,7 +187,7 @@ def install_ntp(install_func, packages=None, check_exe="ntpd"): def rename_ntp_conf(config=None): - """Rename any existing ntp.conf file and render from template""" + """Rename any existing ntp.conf file""" if config is None: # For testing config = NTP_CONF if os.path.exists(config): @@ -171,7 +202,7 @@ def generate_server_names(distro): return names -def write_ntp_config_template(cfg, cloud): +def write_ntp_config_template(cfg, cloud, path, template=None): servers = cfg.get('servers', []) pools = cfg.get('pools', []) @@ -185,19 +216,20 @@ def write_ntp_config_template(cfg, cloud): 'pools': pools, } - template_fn = cloud.get_template_filename('ntp.conf.%s' % - (cloud.distro.name)) + if template is None: + template = 'ntp.conf.%s' % cloud.distro.name + + template_fn = cloud.get_template_filename(template) if not template_fn: template_fn = cloud.get_template_filename('ntp.conf') if not template_fn: raise RuntimeError(("No template found, " - "not rendering %s"), NTP_CONF) + "not rendering %s"), path) - templater.render_to_file(template_fn, NTP_CONF, params) + templater.render_to_file(template_fn, path, params) -def reload_ntp(systemd=False): - service = 'ntp' +def reload_ntp(service, systemd=False): if systemd: cmd = ['systemctl', 'reload-or-restart', service] else: diff --git a/templates/timesyncd.conf.tmpl b/templates/timesyncd.conf.tmpl new file mode 100644 index 00000000..6b98301d --- /dev/null +++ b/templates/timesyncd.conf.tmpl @@ -0,0 +1,8 @@ +## template:jinja +# cloud-init generated file +# See timesyncd.conf(5) for details. + +[Time] +{% if servers or pools -%} +NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%} +{% endif -%} diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 7f278646..83d5faa2 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -16,6 +16,14 @@ servers {{servers}} pools {{pools}} """ +TIMESYNCD_TEMPLATE = b"""\ +## template:jinja +[Time] +{% if servers or pools -%} +NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%} +{% endif -%} +""" + try: import jsonschema assert jsonschema # avoid pyflakes error F401: import unused @@ -59,6 +67,14 @@ class TestNtp(FilesystemMockingTestCase): cc_ntp.install_ntp(install_func, packages=['ntp'], check_exe='ntpd') install_func.assert_not_called() + @mock.patch("cloudinit.config.cc_ntp.util") + def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util): + """ntp_install calls install_func with empty list""" + mock_util.which.return_value = None # check_exe not found + install_func = mock.MagicMock() + cc_ntp.install_ntp(install_func, packages=[], check_exe='timesyncd') + install_func.assert_called_once_with([]) + def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) @@ -68,6 +84,30 @@ class TestNtp(FilesystemMockingTestCase): self.assertFalse(os.path.exists(ntpconf)) self.assertTrue(os.path.exists("{0}.dist".format(ntpconf))) + @mock.patch("cloudinit.config.cc_ntp.util") + def test_reload_ntp_defaults(self, mock_util): + """Test service is restarted/reloaded (defaults)""" + service = 'ntp' + cmd = ['service', service, 'restart'] + cc_ntp.reload_ntp(service) + mock_util.subp.assert_called_with(cmd, capture=True) + + @mock.patch("cloudinit.config.cc_ntp.util") + def test_reload_ntp_systemd(self, mock_util): + """Test service is restarted/reloaded (systemd)""" + service = 'ntp' + cmd = ['systemctl', 'reload-or-restart', service] + cc_ntp.reload_ntp(service, systemd=True) + mock_util.subp.assert_called_with(cmd, capture=True) + + @mock.patch("cloudinit.config.cc_ntp.util") + def test_reload_ntp_systemd_timesycnd(self, mock_util): + """Test service is restarted/reloaded (systemd/timesyncd)""" + service = 'systemd-timesycnd' + cmd = ['systemctl', 'reload-or-restart', service] + cc_ntp.reload_ntp(service, systemd=True) + mock_util.subp.assert_called_with(cmd, capture=True) + def test_ntp_rename_ntp_conf_skip_missing(self): """When NTP_CONF doesn't exist rename_ntp doesn't create a file.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) @@ -94,7 +134,7 @@ class TestNtp(FilesystemMockingTestCase): with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: stream.write(NTP_TEMPLATE) with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud) + cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf) content = util.read_file_or_url('file://' + ntp_conf).contents self.assertEqual( "servers ['192.168.2.1', '192.168.2.2']\npools []\n", @@ -120,7 +160,7 @@ class TestNtp(FilesystemMockingTestCase): with open('{0}.{1}.tmpl'.format(ntp_conf, distro), 'wb') as stream: stream.write(NTP_TEMPLATE) with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud) + cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf) content = util.read_file_or_url('file://' + ntp_conf).contents self.assertEqual( "servers []\npools ['10.0.0.1', '10.0.0.2']\n", @@ -139,7 +179,7 @@ class TestNtp(FilesystemMockingTestCase): with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: stream.write(NTP_TEMPLATE) with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template({}, mycloud) + cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf) content = util.read_file_or_url('file://' + ntp_conf).contents default_pools = [ "{0}.{1}.pool.ntp.org".format(x, distro) @@ -152,7 +192,8 @@ class TestNtp(FilesystemMockingTestCase): ",".join(default_pools)), self.logs.getvalue()) - def test_ntp_handler_mocked_template(self): + @mock.patch("cloudinit.config.cc_ntp.ntp_installable") + def test_ntp_handler_mocked_template(self, m_ntp_install): """Test ntp handler renders ubuntu ntp.conf template.""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] servers = ['192.168.23.3', '192.168.23.4'] @@ -164,6 +205,8 @@ class TestNtp(FilesystemMockingTestCase): } mycloud = self._get_cloud('ubuntu') ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist + m_ntp_install.return_value = True + # Create ntp.conf.tmpl with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: stream.write(NTP_TEMPLATE) @@ -176,6 +219,34 @@ class TestNtp(FilesystemMockingTestCase): 'servers {0}\npools {1}\n'.format(servers, pools), content.decode()) + @mock.patch("cloudinit.config.cc_ntp.util") + def test_ntp_handler_mocked_template_snappy(self, m_util): + """Test ntp handler renders timesycnd.conf template on snappy.""" + pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] + servers = ['192.168.23.3', '192.168.23.4'] + cfg = { + 'ntp': { + 'pools': pools, + 'servers': servers + } + } + mycloud = self._get_cloud('ubuntu') + m_util.system_is_snappy.return_value = True + + # Create timesyncd.conf.tmpl + tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root) + template = '{0}.tmpl'.format(tsyncd_conf) + with open(template, 'wb') as stream: + stream.write(TIMESYNCD_TEMPLATE) + + with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf): + cc_ntp.handle('notimportant', cfg, mycloud, None, None) + + content = util.read_file_or_url('file://' + tsyncd_conf).contents + self.assertEqual( + "[Time]\nNTP=%s %s \n" % (" ".join(servers), " ".join(pools)), + content.decode()) + def test_ntp_handler_real_distro_templates(self): """Test ntp handler renders the shipped distro ntp.conf templates.""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] @@ -333,4 +404,30 @@ class TestNtp(FilesystemMockingTestCase): "pools ['0.mypool.org', '0.mypool.org']\n", content) + @mock.patch("cloudinit.config.cc_ntp.ntp_installable") + def test_ntp_handler_timesyncd(self, m_ntp_install): + """Test ntp handler configures timesyncd""" + m_ntp_install.return_value = False + distro = 'ubuntu' + cfg = { + 'servers': ['192.168.2.1', '192.168.2.2'], + 'pools': ['0.mypool.org'], + } + mycloud = self._get_cloud(distro) + tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root) + # Create timesyncd.conf.tmpl + template = '{0}.tmpl'.format(tsyncd_conf) + print(template) + with open(template, 'wb') as stream: + stream.write(TIMESYNCD_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf): + cc_ntp.write_ntp_config_template(cfg, mycloud, tsyncd_conf, + template='timesyncd.conf') + + content = util.read_file_or_url('file://' + tsyncd_conf).contents + self.assertEqual( + "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n", + content.decode()) + + # vi: ts=4 expandtab -- cgit v1.2.3 From d5f855dd96ccbea77f61b0515b574ad2c43d116d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 9 Aug 2017 21:55:52 -0600 Subject: ec2: Allow Ec2 to run in init-local using dhclient in a sandbox. This branch is a prerequisite for IPv6 support in AWS by allowing Ec2 datasource to query the metadata source version 2016-09-02 about whether or not it needs to configure IPv6 on interfaces. If version 2016-09-02 is not present, fallback to the min_metadata_version of 2009-04-04. The DataSourceEc2Local not run on FreeBSD because dhclient in doesn't support the -sf flag allowing us to run dhclient without filesystem side-effects. To query AWS' metadata address @ 169.254.169.254, the instance must have a dhcp-allocated address configured. Configuring IPv4 link-local addresses result in timeouts from the metadata service. We introduced a DataSourceEc2Local subclass which will perform a sandboxed dhclient discovery which obtains an authorized IP address on eth0 and crawl metadata about full instance network configuration. Since ec2 IPv6 metadata is not sufficient in itself to tell us all the ipv6 knownledge we need, it only be used as a boolean to tell us which nics need IPv6. Cloud-init will then configure desired interfaces to DHCPv6 versus DHCPv4. Performance side note: Shifting the dhcp work into init-local for Ec2 actually gets us 1 second faster deployments by skipping init-network phase of alternate datasource checks because Ec2Local is configured in an ealier boot stage. In 3 test runs prior to this change: cloud-init runs were 5.5 seconds, with the change we now average 4.6 seconds. This efficiency could be even further improved if we avoiding dhcp discovery in order to talk to the metadata service from an AWS authorized dhcp address if there were some way to advertize the dhcp configuration via DMI/SMBIOS or system environment variables. Inspecting time costs of the dhclient setup/teardown in 3 live runs the time cost for the dhcp setup round trip on AWS is: test 1: 76 milliseconds dhcp discovery + metadata: 0.347 seconds metadata alone: 0.271 seconds test 2: 88 milliseconds dhcp discovery + metadata: 0.388 seconds metadata alone: 0.300 seconds test 3: 75 milliseconds dhcp discovery + metadata: 0.366 seconds metadata alone: 0.291 seconds LP: #1709772 --- cloudinit/net/__init__.py | 35 +++--- cloudinit/net/dhcp.py | 119 ++++++++++++++++++++ cloudinit/net/tests/test_dhcp.py | 144 +++++++++++++++++++++++++ cloudinit/net/tests/test_init.py | 2 +- cloudinit/sources/DataSourceAliYun.py | 9 +- cloudinit/sources/DataSourceEc2.py | 121 +++++++++++++++++---- tests/unittests/helpers.py | 2 +- tests/unittests/test_datasource/test_aliyun.py | 11 +- tests/unittests/test_datasource/test_common.py | 1 + tests/unittests/test_datasource/test_ec2.py | 136 ++++++++++++++++++----- tox.ini | 4 +- 11 files changed, 511 insertions(+), 73 deletions(-) create mode 100644 cloudinit/net/dhcp.py create mode 100644 cloudinit/net/tests/test_dhcp.py diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 1ff8fae0..a1b0db10 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -175,13 +175,8 @@ def is_disabled_cfg(cfg): return cfg.get('config') == "disabled" -def generate_fallback_config(blacklist_drivers=None, config_driver=None): - """Determine which attached net dev is most likely to have a connection and - generate network state to run dhcp on that interface""" - - if not config_driver: - config_driver = False - +def find_fallback_nic(blacklist_drivers=None): + """Return the name of the 'fallback' network device.""" if not blacklist_drivers: blacklist_drivers = [] @@ -233,15 +228,24 @@ def generate_fallback_config(blacklist_drivers=None, config_driver=None): if DEFAULT_PRIMARY_INTERFACE in names: names.remove(DEFAULT_PRIMARY_INTERFACE) names.insert(0, DEFAULT_PRIMARY_INTERFACE) - target_name = None - target_mac = None + + # pick the first that has a mac-address for name in names: - mac = read_sys_net_safe(name, 'address') - if mac: - target_name = name - target_mac = mac - break - if target_mac and target_name: + if read_sys_net_safe(name, 'address'): + return name + return None + + +def generate_fallback_config(blacklist_drivers=None, config_driver=None): + """Determine which attached net dev is most likely to have a connection and + generate network state to run dhcp on that interface""" + + if not config_driver: + config_driver = False + + target_name = find_fallback_nic(blacklist_drivers=blacklist_drivers) + if target_name: + target_mac = read_sys_net_safe(target_name, 'address') nconf = {'config': [], 'version': 1} cfg = {'type': 'physical', 'name': target_name, 'mac_address': target_mac, 'subnets': [{'type': 'dhcp'}]} @@ -585,6 +589,7 @@ class EphemeralIPv4Network(object): self._bringup_router() def __exit__(self, excp_type, excp_value, excp_traceback): + """Teardown anything we set up.""" for cmd in self.cleanup_cmds: util.subp(cmd, capture=True) diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py new file mode 100644 index 00000000..c7febc57 --- /dev/null +++ b/cloudinit/net/dhcp.py @@ -0,0 +1,119 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# Author: Chad Smith +# +# This file is part of cloud-init. See LICENSE file for license information. + +import logging +import os +import re + +from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class InvalidDHCPLeaseFileError(Exception): + """Raised when parsing an empty or invalid dhcp.leases file. + + Current uses are DataSourceAzure and DataSourceEc2 during ephemeral + boot to scrape metadata. + """ + pass + + +def maybe_perform_dhcp_discovery(nic=None): + """Perform dhcp discovery if nic valid and dhclient command exists. + + If the nic is invalid or undiscoverable or dhclient command is not found, + skip dhcp_discovery and return an empty dict. + + @param nic: Name of the network interface we want to run dhclient on. + @return: A dict of dhcp options from the dhclient discovery if run, + otherwise an empty dict is returned. + """ + if nic is None: + nic = find_fallback_nic() + if nic is None: + LOG.debug( + 'Skip dhcp_discovery: Unable to find fallback nic.') + return {} + elif nic not in get_devicelist(): + LOG.debug( + 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) + return {} + dhclient_path = util.which('dhclient') + if not dhclient_path: + LOG.debug('Skip dhclient configuration: No dhclient command found.') + return {} + with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + return dhcp_discovery(dhclient_path, nic, tmpdir) + + +def parse_dhcp_lease_file(lease_file): + """Parse the given dhcp lease file for the most recent lease. + + Return a dict of dhcp options as key value pairs for the most recent lease + block. + + @raises: InvalidDHCPLeaseFileError on empty of unparseable leasefile + content. + """ + lease_regex = re.compile(r"lease {(?P[^}]*)}\n") + dhcp_leases = [] + lease_content = util.load_file(lease_file) + if len(lease_content) == 0: + raise InvalidDHCPLeaseFileError( + 'Cannot parse empty dhcp lease file {0}'.format(lease_file)) + for lease in lease_regex.findall(lease_content): + lease_options = [] + for line in lease.split(';'): + # Strip newlines, double-quotes and option prefix + line = line.strip().replace('"', '').replace('option ', '') + if not line: + continue + lease_options.append(line.split(' ', 1)) + dhcp_leases.append(dict(lease_options)) + if not dhcp_leases: + raise InvalidDHCPLeaseFileError( + 'Cannot parse dhcp lease file {0}. No leases found'.format( + lease_file)) + return dhcp_leases + + +def dhcp_discovery(dhclient_cmd_path, interface, cleandir): + """Run dhclient on the interface without scripts or filesystem artifacts. + + @param dhclient_cmd_path: Full path to the dhclient used. + @param interface: Name of the network inteface on which to dhclient. + @param cleandir: The directory from which to run dhclient as well as store + dhcp leases. + + @return: A dict of dhcp options parsed from the dhcp.leases file or empty + dict. + """ + LOG.debug('Performing a dhcp discovery on %s', interface) + + # XXX We copy dhclient out of /sbin/dhclient to avoid dealing with strict + # app armor profiles which disallow running dhclient -sf . + # We want to avoid running /sbin/dhclient-script because of side-effects in + # /etc/resolv.conf any any other vendor specific scripts in + # /etc/dhcp/dhclient*hooks.d. + sandbox_dhclient_cmd = os.path.join(cleandir, 'dhclient') + util.copy(dhclient_cmd_path, sandbox_dhclient_cmd) + pid_file = os.path.join(cleandir, 'dhclient.pid') + lease_file = os.path.join(cleandir, 'dhcp.leases') + + # ISC dhclient needs the interface up to send initial discovery packets. + # Generally dhclient relies on dhclient-script PREINIT action to bring the + # link up before attempting discovery. Since we are using -sf /bin/true, + # we need to do that "link up" ourselves first. + util.subp(['ip', 'link', 'set', 'dev', interface, 'up'], capture=True) + cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, + '-pf', pid_file, interface, '-sf', '/bin/true'] + util.subp(cmd, capture=True) + return parse_dhcp_lease_file(lease_file) + + +# vi: ts=4 expandtab diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py new file mode 100644 index 00000000..47d8d461 --- /dev/null +++ b/cloudinit/net/tests/test_dhcp.py @@ -0,0 +1,144 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import mock +import os +from textwrap import dedent + +from cloudinit.net.dhcp import ( + InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, + parse_dhcp_lease_file, dhcp_discovery) +from cloudinit.util import ensure_file, write_file +from tests.unittests.helpers import CiTestCase + + +class TestParseDHCPLeasesFile(CiTestCase): + + def test_parse_empty_lease_file_errors(self): + """parse_dhcp_lease_file errors when file content is empty.""" + empty_file = self.tmp_path('leases') + ensure_file(empty_file) + with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager: + parse_dhcp_lease_file(empty_file) + error = context_manager.exception + self.assertIn('Cannot parse empty dhcp lease file', str(error)) + + def test_parse_malformed_lease_file_content_errors(self): + """parse_dhcp_lease_file errors when file content isn't dhcp leases.""" + non_lease_file = self.tmp_path('leases') + write_file(non_lease_file, 'hi mom.') + with self.assertRaises(InvalidDHCPLeaseFileError) as context_manager: + parse_dhcp_lease_file(non_lease_file) + error = context_manager.exception + self.assertIn('Cannot parse dhcp lease file', str(error)) + + def test_parse_multiple_leases(self): + """parse_dhcp_lease_file returns a list of all leases within.""" + lease_file = self.tmp_path('leases') + content = dedent(""" + lease { + interface "wlp3s0"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + renew 4 2017/07/27 18:02:30; + expire 5 2017/07/28 07:08:15; + } + lease { + interface "wlp3s0"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + expected = [ + {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', + 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1', + 'renew': '4 2017/07/27 18:02:30', + 'expire': '5 2017/07/28 07:08:15'}, + {'interface': 'wlp3s0', 'fixed-address': '192.168.2.74', + 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}] + write_file(lease_file, content) + self.assertItemsEqual(expected, parse_dhcp_lease_file(lease_file)) + + +class TestDHCPDiscoveryClean(CiTestCase): + with_logs = True + + @mock.patch('cloudinit.net.dhcp.find_fallback_nic') + def test_no_fallback_nic_found(self, m_fallback_nic): + """Log and do nothing when nic is absent and no fallback is found.""" + m_fallback_nic.return_value = None # No fallback nic found + self.assertEqual({}, maybe_perform_dhcp_discovery()) + self.assertIn( + 'Skip dhcp_discovery: Unable to find fallback nic.', + self.logs.getvalue()) + + def test_provided_nic_does_not_exist(self): + """When the provided nic doesn't exist, log a message and no-op.""" + self.assertEqual({}, maybe_perform_dhcp_discovery('idontexist')) + self.assertIn( + 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.', + self.logs.getvalue()) + + @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.find_fallback_nic') + def test_absent_dhclient_command(self, m_fallback, m_which): + """When dhclient doesn't exist in the OS, log the issue and no-op.""" + m_fallback.return_value = 'eth9' + m_which.return_value = None # dhclient isn't found + self.assertEqual({}, maybe_perform_dhcp_discovery()) + self.assertIn( + 'Skip dhclient configuration: No dhclient command found.', + self.logs.getvalue()) + + @mock.patch('cloudinit.net.dhcp.dhcp_discovery') + @mock.patch('cloudinit.net.dhcp.util.which') + @mock.patch('cloudinit.net.dhcp.find_fallback_nic') + def test_dhclient_run_with_tmpdir(self, m_fallback, m_which, m_dhcp): + """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" + m_fallback.return_value = 'eth9' + m_which.return_value = '/sbin/dhclient' + m_dhcp.return_value = {'address': '192.168.2.2'} + self.assertEqual( + {'address': '192.168.2.2'}, maybe_perform_dhcp_discovery()) + m_dhcp.assert_called_once() + call = m_dhcp.call_args_list[0] + self.assertEqual('/sbin/dhclient', call[0][0]) + self.assertEqual('eth9', call[0][1]) + self.assertIn('/tmp/cloud-init-dhcp-', call[0][2]) + + @mock.patch('cloudinit.net.dhcp.util.subp') + def test_dhcp_discovery_run_in_sandbox(self, m_subp): + """dhcp_discovery brings up the interface and runs dhclient. + + It also returns the parsed dhcp.leases file generated in the sandbox. + """ + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + lease_content = dedent(""" + lease { + interface "eth9"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + lease_file = os.path.join(tmpdir, 'dhcp.leases') + write_file(lease_file, lease_content) + self.assertItemsEqual( + [{'interface': 'eth9', 'fixed-address': '192.168.2.74', + 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], + dhcp_discovery(dhclient_script, 'eth9', tmpdir)) + # dhclient script got copied + with open(os.path.join(tmpdir, 'dhclient')) as stream: + self.assertEqual(script_content, stream.read()) + # Interface was brought up before dhclient called from sandbox + m_subp.assert_has_calls([ + mock.call( + ['ip', 'link', 'set', 'dev', 'eth9', 'up'], capture=True), + mock.call( + [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf', + lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), + 'eth9', '-sf', '/bin/true'], capture=True)]) diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index 272a6ebd..cc052a7d 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -414,7 +414,7 @@ class TestEphemeralIPV4Network(CiTestCase): self.assertIn('Cannot init network on', str(error)) self.assertEqual(0, m_subp.call_count) - def test_ephemeral_ipv4_network_errors_invalid_mask(self, m_subp): + def test_ephemeral_ipv4_network_errors_invalid_mask_prefix(self, m_subp): """Raise an error when prefix_or_mask is not a netmask or prefix.""" params = { 'interface': 'eth0', 'ip': '192.168.2.2', diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index 380e27cb..43a7e42c 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -6,17 +6,20 @@ from cloudinit import sources from cloudinit.sources import DataSourceEc2 as EC2 from cloudinit import util -DEF_MD_VERSION = "2016-01-01" ALIYUN_PRODUCT = "Alibaba Cloud ECS" class DataSourceAliYun(EC2.DataSourceEc2): - metadata_urls = ["http://100.100.100.200"] + + metadata_urls = ['http://100.100.100.200'] + + # The minimum supported metadata_version from the ec2 metadata apis + min_metadata_version = '2016-01-01' + extended_metadata_versions = [] def __init__(self, sys_cfg, distro, paths): super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, "AliYun") - self.api_ver = DEF_MD_VERSION def get_hostname(self, fqdn=False, _resolve_ip=False): return self.metadata.get('hostname', 'localhost.localdomain') diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 4ec9592f..8e5f8ee4 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -13,6 +13,8 @@ import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit import net +from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util @@ -20,8 +22,7 @@ from cloudinit import warnings LOG = logging.getLogger(__name__) -# Which version we are requesting of the ec2 metadata apis -DEF_MD_VERSION = '2009-04-04' +SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND]) STRICT_ID_PATH = ("datasource", "Ec2", "strict_id") STRICT_ID_DEFAULT = "warn" @@ -41,17 +42,28 @@ class Platforms(object): class DataSourceEc2(sources.DataSource): + # Default metadata urls that will be used if none are provided # They will be checked for 'resolveability' and some of the # following may be discarded if they do not resolve metadata_urls = ["http://169.254.169.254", "http://instance-data.:8773"] + + # The minimum supported metadata_version from the ec2 metadata apis + min_metadata_version = '2009-04-04' + + # Priority ordered list of additional metadata versions which will be tried + # for extended metadata content. IPv6 support comes in 2016-09-02 + extended_metadata_versions = ['2016-09-02'] + _cloud_platform = None + # Whether we want to get network configuration from the metadata service. + get_network_metadata = False + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.metadata_address = None self.seed_dir = os.path.join(paths.seed_dir, "ec2") - self.api_ver = DEF_MD_VERSION def get_data(self): seed_ret = {} @@ -73,21 +85,27 @@ class DataSourceEc2(sources.DataSource): elif self.cloud_platform == Platforms.NO_EC2_METADATA: return False - try: - if not self.wait_for_metadata_service(): + if self.get_network_metadata: # Setup networking in init-local stage. + if util.is_FreeBSD(): + LOG.debug("FreeBSD doesn't support running dhclient with -sf") return False - start_time = time.time() - self.userdata_raw = \ - ec2.get_instance_userdata(self.api_ver, self.metadata_address) - self.metadata = ec2.get_instance_metadata(self.api_ver, - self.metadata_address) - LOG.debug("Crawl of metadata service took %.3f seconds", - time.time() - start_time) - return True - except Exception: - util.logexc(LOG, "Failed reading from metadata address %s", - self.metadata_address) - return False + dhcp_leases = dhcp.maybe_perform_dhcp_discovery() + if not dhcp_leases: + # DataSourceEc2Local failed in init-local stage. DataSourceEc2 + # will still run in init-network stage. + return False + dhcp_opts = dhcp_leases[-1] + net_params = {'interface': dhcp_opts.get('interface'), + 'ip': dhcp_opts.get('fixed-address'), + 'prefix_or_mask': dhcp_opts.get('subnet-mask'), + 'broadcast': dhcp_opts.get('broadcast-address'), + 'router': dhcp_opts.get('routers')} + with net.EphemeralIPv4Network(**net_params): + return util.log_time( + logfunc=LOG.debug, msg='Crawl of metadata service', + func=self._crawl_metadata) + else: + return self._crawl_metadata() @property def launch_index(self): @@ -95,6 +113,32 @@ class DataSourceEc2(sources.DataSource): return None return self.metadata.get('ami-launch-index') + def get_metadata_api_version(self): + """Get the best supported api version from the metadata service. + + Loop through all extended support metadata versions in order and + return the most-fully featured metadata api version discovered. + + If extended_metadata_versions aren't present, return the datasource's + min_metadata_version. + """ + # Assumes metadata service is already up + for api_ver in self.extended_metadata_versions: + url = '{0}/{1}/meta-data/instance-id'.format( + self.metadata_address, api_ver) + try: + resp = uhelp.readurl(url=url) + except uhelp.UrlError as e: + LOG.debug('url %s raised exception %s', url, e) + else: + if resp.code == 200: + LOG.debug('Found preferred metadata version %s', api_ver) + return api_ver + elif resp.code == 404: + msg = 'Metadata api version %s not present. Headers: %s' + LOG.debug(msg, api_ver, resp.headers) + return self.min_metadata_version + def get_instance_id(self): return self.metadata['instance-id'] @@ -138,21 +182,22 @@ class DataSourceEc2(sources.DataSource): urls = [] url2base = {} for url in mdurls: - cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) + cur = '{0}/{1}/meta-data/instance-id'.format( + url, self.min_metadata_version) urls.append(cur) url2base[cur] = url start_time = time.time() - url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + url = uhelp.wait_for_url( + urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn) if url: - LOG.debug("Using metadata source: '%s'", url2base[url]) + self.metadata_address = url2base[url] + LOG.debug("Using metadata source: '%s'", self.metadata_address) else: LOG.critical("Giving up on md from %s after %s seconds", urls, int(time.time() - start_time)) - self.metadata_address = url2base.get(url) return bool(url) def device_name_to_device(self, name): @@ -234,6 +279,37 @@ class DataSourceEc2(sources.DataSource): util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT), cfg) + def _crawl_metadata(self): + """Crawl metadata service when available. + + @returns: True on success, False otherwise. + """ + if not self.wait_for_metadata_service(): + return False + api_version = self.get_metadata_api_version() + try: + self.userdata_raw = ec2.get_instance_userdata( + api_version, self.metadata_address) + self.metadata = ec2.get_instance_metadata( + api_version, self.metadata_address) + except Exception: + util.logexc( + LOG, "Failed reading from metadata address %s", + self.metadata_address) + return False + return True + + +class DataSourceEc2Local(DataSourceEc2): + """Datasource run at init-local which sets up network to query metadata. + + In init-local, no network is available. This subclass sets up minimal + networking with dhclient on a viable nic so that it can talk to the + metadata service. If the metadata service provides network configuration + then render the network configuration for that instance based on metadata. + """ + get_network_metadata = True # Get metadata network config if present + def read_strict_mode(cfgval, default): try: @@ -349,6 +425,7 @@ def _collect_platform_data(): # Used to match classes to dependencies datasources = [ + (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local (DataSourceEc2, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 08c5c469..bf1dc5df 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -278,7 +278,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): return root -class HttprettyTestCase(TestCase): +class HttprettyTestCase(CiTestCase): # necessary as http_proxy gets in the way of httpretty # https://github.com/gabrielfalcao/HTTPretty/issues/122 def setUp(self): diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 990bff2c..996560e4 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -70,7 +70,6 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase): paths = helpers.Paths({}) self.ds = ay.DataSourceAliYun(cfg, distro, paths) self.metadata_address = self.ds.metadata_urls[0] - self.api_ver = self.ds.api_ver @property def default_metadata(self): @@ -82,13 +81,15 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase): @property def metadata_url(self): - return os.path.join(self.metadata_address, - self.api_ver, 'meta-data') + '/' + return os.path.join( + self.metadata_address, + self.ds.min_metadata_version, 'meta-data') + '/' @property def userdata_url(self): - return os.path.join(self.metadata_address, - self.api_ver, 'user-data') + return os.path.join( + self.metadata_address, + self.ds.min_metadata_version, 'user-data') def regist_default_server(self): register_mock_metaserver(self.metadata_url, self.default_metadata) diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 413e87ac..4802f105 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -35,6 +35,7 @@ DEFAULT_LOCAL = [ OpenNebula.DataSourceOpenNebula, OVF.DataSourceOVF, SmartOS.DataSourceSmartOS, + Ec2.DataSourceEc2Local, ] DEFAULT_NETWORK = [ diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 12230ae2..33d02619 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -8,35 +8,67 @@ from cloudinit import helpers from cloudinit.sources import DataSourceEc2 as ec2 -# collected from api version 2009-04-04/ with +# collected from api version 2016-09-02/ with # python3 -c 'import json # from cloudinit.ec2_utils import get_instance_metadata as gm -# print(json.dumps(gm("2009-04-04"), indent=1, sort_keys=True))' +# print(json.dumps(gm("2016-09-02"), indent=1, sort_keys=True))' DEFAULT_METADATA = { - "ami-id": "ami-80861296", + "ami-id": "ami-8b92b4ee", "ami-launch-index": "0", "ami-manifest-path": "(unknown)", "block-device-mapping": {"ami": "/dev/sda1", "root": "/dev/sda1"}, - "hostname": "ip-10-0-0-149", + "hostname": "ip-172-31-31-158.us-east-2.compute.internal", "instance-action": "none", - "instance-id": "i-0052913950685138c", - "instance-type": "t2.micro", - "local-hostname": "ip-10-0-0-149", - "local-ipv4": "10.0.0.149", - "placement": {"availability-zone": "us-east-1b"}, + "instance-id": "i-0a33f80f09c96477f", + "instance-type": "t2.small", + "local-hostname": "ip-172-3-3-15.us-east-2.compute.internal", + "local-ipv4": "172.3.3.15", + "mac": "06:17:04:d7:26:09", + "metrics": {"vhostmd": ""}, + "network": { + "interfaces": { + "macs": { + "06:17:04:d7:26:09": { + "device-number": "0", + "interface-id": "eni-e44ef49e", + "ipv4-associations": {"13.59.77.202": "172.3.3.15"}, + "ipv6s": "2600:1f16:aeb:b20b:9d87:a4af:5cc9:73dc", + "local-hostname": ("ip-172-3-3-15.us-east-2." + "compute.internal"), + "local-ipv4s": "172.3.3.15", + "mac": "06:17:04:d7:26:09", + "owner-id": "950047163771", + "public-hostname": ("ec2-13-59-77-202.us-east-2." + "compute.amazonaws.com"), + "public-ipv4s": "13.59.77.202", + "security-group-ids": "sg-5a61d333", + "security-groups": "wide-open", + "subnet-id": "subnet-20b8565b", + "subnet-ipv4-cidr-block": "172.31.16.0/20", + "subnet-ipv6-cidr-blocks": "2600:1f16:aeb:b20b::/64", + "vpc-id": "vpc-87e72bee", + "vpc-ipv4-cidr-block": "172.31.0.0/16", + "vpc-ipv4-cidr-blocks": "172.31.0.0/16", + "vpc-ipv6-cidr-blocks": "2600:1f16:aeb:b200::/56" + } + } + } + }, + "placement": {"availability-zone": "us-east-2b"}, "profile": "default-hvm", - "public-hostname": "", - "public-ipv4": "107.23.188.247", + "public-hostname": "ec2-13-59-77-202.us-east-2.compute.amazonaws.com", + "public-ipv4": "13.59.77.202", "public-keys": {"brickies": ["ssh-rsa AAAAB3Nz....w== brickies"]}, - "reservation-id": "r-00a2c173fb5782a08", - "security-groups": "wide-open" + "reservation-id": "r-01efbc9996bac1bd6", + "security-groups": "my-wide-open", + "services": {"domain": "amazonaws.com", "partition": "aws"} } def _register_ssh_keys(rfunc, base_url, keys_data): """handle ssh key inconsistencies. - public-keys in the ec2 metadata is inconsistently formatted compared + public-keys in the ec2 metadata is inconsistently formated compared to other entries. Given keys_data of {name1: pubkey1, name2: pubkey2} @@ -115,6 +147,8 @@ def register_mock_metaserver(base_url, data): class TestEc2(test_helpers.HttprettyTestCase): + with_logs = True + valid_platform_data = { 'uuid': 'ec212f79-87d1-2f1d-588f-d86dc0fd5412', 'uuid_source': 'dmi', @@ -123,16 +157,20 @@ class TestEc2(test_helpers.HttprettyTestCase): def setUp(self): super(TestEc2, self).setUp() - self.metadata_addr = ec2.DataSourceEc2.metadata_urls[0] - self.api_ver = '2009-04-04' + self.datasource = ec2.DataSourceEc2 + self.metadata_addr = self.datasource.metadata_urls[0] @property def metadata_url(self): - return '/'.join([self.metadata_addr, self.api_ver, 'meta-data', '']) + return '/'.join([ + self.metadata_addr, + self.datasource.min_metadata_version, 'meta-data', '']) @property def userdata_url(self): - return '/'.join([self.metadata_addr, self.api_ver, 'user-data']) + return '/'.join([ + self.metadata_addr, + self.datasource.min_metadata_version, 'user-data']) def _patch_add_cleanup(self, mpath, *args, **kwargs): p = mock.patch(mpath, *args, **kwargs) @@ -144,7 +182,7 @@ class TestEc2(test_helpers.HttprettyTestCase): paths = helpers.Paths({}) if sys_cfg is None: sys_cfg = {} - ds = ec2.DataSourceEc2(sys_cfg=sys_cfg, distro=distro, paths=paths) + ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths) if platform_data is not None: self._patch_add_cleanup( "cloudinit.sources.DataSourceEc2._collect_platform_data", @@ -157,14 +195,16 @@ class TestEc2(test_helpers.HttprettyTestCase): return ds @httpretty.activate - def test_valid_platform_with_strict_true(self): + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + def test_valid_platform_with_strict_true(self, m_dhcp): """Valid platform data should return true with strict_id true.""" ds = self._setup_ds( platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(True, ret) + self.assertTrue(ret) + self.assertEqual(0, m_dhcp.call_count) @httpretty.activate def test_valid_platform_with_strict_false(self): @@ -174,7 +214,7 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(True, ret) + self.assertTrue(ret) @httpretty.activate def test_unknown_platform_with_strict_true(self): @@ -185,7 +225,7 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(False, ret) + self.assertFalse(ret) @httpretty.activate def test_unknown_platform_with_strict_false(self): @@ -196,7 +236,55 @@ class TestEc2(test_helpers.HttprettyTestCase): sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) ret = ds.get_data() - self.assertEqual(True, ret) + self.assertTrue(ret) + + @httpretty.activate + @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') + def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd): + """DataSourceEc2Local returns False on BSD. + + FreeBSD dhclient doesn't support dhclient -sf to run in a sandbox. + """ + m_is_freebsd.return_value = True + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertFalse(ret) + self.assertIn( + "FreeBSD doesn't support running dhclient with -sf", + self.logs.getvalue()) + + @httpretty.activate + @mock.patch('cloudinit.net.EphemeralIPv4Network') + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') + def test_ec2_local_performs_dhcp_on_non_bsd(self, m_is_bsd, m_dhcp, m_net): + """Ec2Local returns True for valid platform data on non-BSD with dhcp. + + DataSourceEc2Local will setup initial IPv4 network via dhcp discovery. + Then the metadata services is crawled for more network config info. + When the platform data is valid, return True. + """ + m_is_bsd.return_value = False + m_dhcp.return_value = [{ + 'interface': 'eth9', 'fixed-address': '192.168.2.9', + 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', + 'broadcast-address': '192.168.2.255'}] + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + ret = ds.get_data() + self.assertTrue(ret) + m_dhcp.assert_called_once_with() + m_net.assert_called_once_with( + broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9', + prefix_or_mask='255.255.255.0', router='192.168.2.1') + self.assertIn('Crawl of metadata service took', self.logs.getvalue()) # vi: ts=4 expandtab diff --git a/tox.ini b/tox.ini index ef768847..1e7ca2d3 100644 --- a/tox.ini +++ b/tox.ini @@ -21,10 +21,10 @@ setenv = LC_ALL = en_US.utf-8 [testenv:pylint] -deps = +deps = # requirements pylint==1.7.1 - # test-requirements because unit tests are now present in cloudinit tree + # test-requirements because unit tests are now present in cloudinit tree -r{toxinidir}/test-requirements.txt commands = {envpython} -m pylint {posargs:cloudinit} -- cgit v1.2.3 From 1f8183ff4750cc7f8798749987ef10912719544d Mon Sep 17 00:00:00 2001 From: Maitreyee Saikia Date: Tue, 15 Aug 2017 09:33:50 -0600 Subject: vcloud directory: Guest Customization support for passwords This feature enables the following VMware VCloud Director functionality: 1. Setting admin password 2. Expire password. 3. Set admin password and expire. Password configuration is triggered only as part of a full recustomization, that happens either on first power on or when "poweron and full recustomization" is selected. Full customization flow is determined by marker files. Unique marker ids are generated when full recustomization is requested. And marker file based on these marker ids help to determine if we need to execute the above configuration. --- cloudinit/sources/DataSourceOVF.py | 63 +++++++++++++++++++- cloudinit/sources/helpers/vmware/imc/config.py | 24 +++++++- .../sources/helpers/vmware/imc/config_passwd.py | 67 ++++++++++++++++++++++ tests/unittests/test_vmware_config_file.py | 32 ++++++++++- 4 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/config_passwd.py diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index f20c9a65..73d38771 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -25,6 +25,8 @@ from cloudinit.sources.helpers.vmware.imc.config_file \ import ConfigFile from cloudinit.sources.helpers.vmware.imc.config_nic \ import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.config_passwd \ + import PasswordConfigurator from cloudinit.sources.helpers.vmware.imc.guestcust_error \ import GuestCustErrorEnum from cloudinit.sources.helpers.vmware.imc.guestcust_event \ @@ -117,6 +119,8 @@ class DataSourceOVF(sources.DataSource): (md, ud, cfg) = read_vmware_imc(conf) dirpath = os.path.dirname(vmwareImcConfigFilePath) nics = get_nics_to_enable(dirpath) + markerid = conf.marker_id + markerexists = check_marker_exists(markerid) except Exception as e: LOG.debug("Error parsing the customization Config File") LOG.exception(e) @@ -127,7 +131,6 @@ class DataSourceOVF(sources.DataSource): return False finally: util.del_dir(os.path.dirname(vmwareImcConfigFilePath)) - try: LOG.debug("Applying the Network customization") nicConfigurator = NicConfigurator(conf.nics) @@ -140,6 +143,35 @@ class DataSourceOVF(sources.DataSource): GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) enable_nics(nics) return False + if markerid and not markerexists: + LOG.debug("Applying password customization") + pwdConfigurator = PasswordConfigurator() + adminpwd = conf.admin_password + try: + resetpwd = conf.reset_password + if adminpwd or resetpwd: + pwdConfigurator.configure(adminpwd, resetpwd, + self.distro) + else: + LOG.debug("Changing password is not needed") + except Exception as e: + LOG.debug("Error applying Password Configuration: %s", e) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) + enable_nics(nics) + return False + if markerid: + LOG.debug("Handle marker creation") + try: + setup_marker_files(markerid) + except Exception as e: + LOG.debug("Error creating marker files: %s", e) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) + enable_nics(nics) + return False vmwarePlatformFound = True set_customization_status( @@ -445,4 +477,33 @@ datasources = ( def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + +# To check if marker file exists +def check_marker_exists(markerid): + """ + Check the existence of a marker file. + Presence of marker file determines whether a certain code path is to be + executed. It is needed for partial guest customization in VMware. + """ + if not markerid: + return False + markerfile = "/.markerfile-" + markerid + if os.path.exists(markerfile): + return True + return False + + +# Create a marker file +def setup_marker_files(markerid): + """ + Create a new marker file. + Marker files are unique to a full customization workflow in VMware + environment. + """ + if not markerid: + return + markerfile = "/.markerfile-" + markerid + util.del_file("/.markerfile-*.txt") + open(markerfile, 'w').close() + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 9a5e3a8a..49d441db 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -5,6 +5,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. + from .nic import Nic @@ -14,13 +15,16 @@ class Config(object): Specification file. """ + CUSTOM_SCRIPT = 'CUSTOM-SCRIPT|SCRIPT-NAME' DNS = 'DNS|NAMESERVER|' - SUFFIX = 'DNS|SUFFIX|' + DOMAINNAME = 'NETWORK|DOMAINNAME' + HOSTNAME = 'NETWORK|HOSTNAME' + MARKERID = 'MISC|MARKER-ID' PASS = 'PASSWORD|-PASS' + RESETPASS = 'PASSWORD|RESET' + SUFFIX = 'DNS|SUFFIX|' TIMEZONE = 'DATETIME|TIMEZONE' UTC = 'DATETIME|UTC' - HOSTNAME = 'NETWORK|HOSTNAME' - DOMAINNAME = 'NETWORK|DOMAINNAME' def __init__(self, configFile): self._configFile = configFile @@ -82,4 +86,18 @@ class Config(object): return res + @property + def reset_password(self): + """Retreives if the root password needs to be reset.""" + resetPass = self._configFile.get(Config.RESETPASS, 'no') + resetPass = resetPass.lower() + if resetPass not in ('yes', 'no'): + raise ValueError('ResetPassword value should be yes/no') + return resetPass == 'yes' + + @property + def marker_id(self): + """Returns marker id.""" + return self._configFile.get(Config.MARKERID, None) + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py new file mode 100644 index 00000000..75cfbaaf --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py @@ -0,0 +1,67 @@ +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware INC. +# +# Author: Maitreyee Saikia +# +# This file is part of cloud-init. See LICENSE file for license information. + + +import logging +import os + +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class PasswordConfigurator(object): + """ + Class for changing configurations related to passwords in a VM. Includes + setting and expiring passwords. + """ + def configure(self, passwd, resetPasswd, distro): + """ + Main method to perform all functionalities based on configuration file + inputs. + @param passwd: encoded admin password. + @param resetPasswd: boolean to determine if password needs to be reset. + @return cfg: dict to be used by cloud-init set_passwd code. + """ + LOG.info('Starting password configuration') + if passwd: + passwd = util.b64d(passwd) + allRootUsers = [] + for line in open('/etc/passwd', 'r'): + if line.split(':')[2] == '0': + allRootUsers.append(line.split(':')[0]) + # read shadow file and check for each user, if its uid0 or root. + uidUsersList = [] + for line in open('/etc/shadow', 'r'): + user = line.split(':')[0] + if user in allRootUsers: + uidUsersList.append(user) + if passwd: + LOG.info('Setting admin password') + distro.set_passwd('root', passwd) + if resetPasswd: + self.reset_password(uidUsersList) + LOG.info('Configure Password completed!') + + def reset_password(self, uidUserList): + """ + Method to reset password. Use passwd --expire command. Use chage if + not succeeded using passwd command. Log failure message otherwise. + @param: list of users for which to expire password. + """ + LOG.info('Expiring password.') + for user in uidUserList: + try: + out, err = util.subp(['passwd', '--expire', user]) + except util.ProcessExecutionError as e: + if os.path.exists('/usr/bin/chage'): + out, e = util.subp(['chage', '-d', '0', user]) + else: + LOG.warning('Failed to expire password for %s with error: ' + '%s', user, e) + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 18475f10..03b36d31 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -7,8 +7,8 @@ import logging import sys -import unittest +from .helpers import CiTestCase from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile @@ -17,7 +17,7 @@ logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) logger = logging.getLogger(__name__) -class TestVmwareConfigFile(unittest.TestCase): +class TestVmwareConfigFile(CiTestCase): def test_utility_methods(self): cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") @@ -90,4 +90,32 @@ class TestVmwareConfigFile(unittest.TestCase): self.assertEqual('00:50:56:a6:8c:08', nics[0].mac, "mac0") self.assertEqual(BootProtoEnum.DHCP, nics[0].bootProto, "bootproto0") + def test_config_password(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + cf._insertKey("PASSWORD|-PASS", "test-password") + cf._insertKey("PASSWORD|RESET", "no") + + conf = Config(cf) + self.assertEqual('test-password', conf.admin_password, "password") + self.assertFalse(conf.reset_password, "do not reset password") + + def test_config_reset_passwd(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + cf._insertKey("PASSWORD|-PASS", "test-password") + cf._insertKey("PASSWORD|RESET", "random") + + conf = Config(cf) + with self.assertRaises(ValueError): + conf.reset_password() + + cf.clear() + cf._insertKey("PASSWORD|RESET", "yes") + self.assertEqual(1, len(cf), "insert size") + + conf = Config(cf) + self.assertTrue(conf.reset_password, "reset password") + + # vi: ts=4 expandtab -- cgit v1.2.3 From 385d1cae1023ed89c3830a148aea02807240a07d Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 14 Aug 2017 11:40:54 -0500 Subject: doc: update capabilities with features available, link doc reference, cli example --- doc/rtd/topics/capabilities.rst | 50 ++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/doc/rtd/topics/capabilities.rst b/doc/rtd/topics/capabilities.rst index 2c8770bd..b8034b07 100644 --- a/doc/rtd/topics/capabilities.rst +++ b/doc/rtd/topics/capabilities.rst @@ -31,19 +31,49 @@ support. This allows other applications to detect what features the installed cloud-init supports without having to parse its version number. If present, this list of features will be located at ``cloudinit.version.FEATURES``. -When checking if cloud-init supports a feature, in order to not break the -detection script on older versions of cloud-init without the features list, a -script similar to the following should be used. Note that this will exit 0 if -the feature is supported and 1 otherwise:: +Currently defined feature names include: - import sys - from cloudinit import version - sys.exit('' not in getattr(version, 'FEATURES', [])) + - ``NETWORK_CONFIG_V1`` support for v1 networking configuration, + see :ref:`network_config_v1` documentation for examples. + - ``NETWORK_CONFIG_V2`` support for v2 networking configuration, + see :ref:`network_config_v2` documentation for examples. -Currently defined feature names include: - - ``NETWORK_CONFIG_V1`` support for v1 networking configuration, see curtin - documentation for examples. +CLI Interface : + +``cloud-init features`` will print out each feature supported. If cloud-init +does not have the features subcommand, it also does not support any features +described in this document. + +.. code-block:: bash + + % cloud-init --help + usage: cloud-init [-h] [--version] [--file FILES] [--debug] [--force] + {init,modules,query,single,dhclient-hook,features} ... + + positional arguments: + {init,modules,query,single,dhclient-hook,features} + init initializes cloud-init and performs initial modules + modules activates modules using a given configuration key + query query information stored in cloud-init + single run a single module + dhclient-hook run the dhclient hookto record network info + features list defined features + + optional arguments: + -h, --help show this help message and exit + --version, -v show program's version number and exit + --file FILES, -f FILES + additional yaml configuration files to use + --debug, -d show additional pre-action logging (default: False) + --force force running even if no datasource is found (use at + your own risk) + + + % cloud-init features + NETWORK_CONFIG_V1 + NETWORK_CONFIG_V2 + .. _Cloud-init: https://launchpad.net/cloud-init .. vi: textwidth=78 -- cgit v1.2.3 From dc2bd79949492bccdc1d7df0132f98c354d51943 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 9 Aug 2017 14:44:20 -0500 Subject: network: add v2 passthrough and fix parsing v2 config with bonds/bridge params If the network-config sent to cloud-init is in version: 2 format then when rendering netplan, we can pass the content through and avoid consuming network_state elements. This removes the need for trying to map many v2 features onto network state where other renderers won't be able to use anyhow (for example match parameters for multi-interface configuration and wifi configuration support). Additionally ensure we retain bond/bridge v2 configuration in network state so when rendering to eni or sysconfig we don't lose the configuration - Drop the NotImplemented wifi exception, log a warning that it works for netplan only - Adjust unittests to new code path and output - Fix issue with v2 macaddress values getting dropped - Add unittests for consuming/validating v2 configurations LP: #1709180 --- cloudinit/net/netplan.py | 35 ++------ cloudinit/net/network_state.py | 85 ++++++++++++++---- tests/unittests/test_distros/test_netconfig.py | 4 +- tests/unittests/test_net.py | 115 +++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 44 deletions(-) diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 9f35b72b..3b06fbf0 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -4,7 +4,7 @@ import copy import os from . import renderer -from .network_state import subnet_is_ipv6 +from .network_state import subnet_is_ipv6, NET_CONFIG_TO_V2 from cloudinit import log as logging from cloudinit import util @@ -27,31 +27,6 @@ network: """ LOG = logging.getLogger(__name__) -NET_CONFIG_TO_V2 = { - 'bond': {'bond-ad-select': 'ad-select', - 'bond-arp-interval': 'arp-interval', - 'bond-arp-ip-target': 'arp-ip-target', - 'bond-arp-validate': 'arp-validate', - 'bond-downdelay': 'down-delay', - 'bond-fail-over-mac': 'fail-over-mac-policy', - 'bond-lacp-rate': 'lacp-rate', - 'bond-miimon': 'mii-monitor-interval', - 'bond-min-links': 'min-links', - 'bond-mode': 'mode', - 'bond-num-grat-arp': 'gratuitious-arp', - 'bond-primary-reselect': 'primary-reselect-policy', - 'bond-updelay': 'up-delay', - 'bond-xmit-hash-policy': 'transmit-hash-policy'}, - 'bridge': {'bridge_ageing': 'ageing-time', - 'bridge_bridgeprio': 'priority', - 'bridge_fd': 'forward-delay', - 'bridge_gcint': None, - 'bridge_hello': 'hello-time', - 'bridge_maxage': 'max-age', - 'bridge_maxwait': None, - 'bridge_pathcost': 'path-cost', - 'bridge_portprio': None, - 'bridge_waitport': None}} def _get_params_dict_by_match(config, match): @@ -247,6 +222,14 @@ class Renderer(renderer.Renderer): util.subp(cmd, capture=True) def _render_content(self, network_state): + + # if content already in netplan format, pass it back + if network_state.version == 2: + LOG.debug('V2 to V2 passthrough') + return util.yaml_dumps({'network': network_state.config}, + explicit_start=False, + explicit_end=False) + ethernets = {} wifis = {} bridges = {} diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 87a7222d..6faf01b7 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -23,6 +23,33 @@ NETWORK_V2_KEY_FILTER = [ 'match', 'mtu', 'nameservers', 'renderer', 'set-name', 'wakeonlan' ] +NET_CONFIG_TO_V2 = { + 'bond': {'bond-ad-select': 'ad-select', + 'bond-arp-interval': 'arp-interval', + 'bond-arp-ip-target': 'arp-ip-target', + 'bond-arp-validate': 'arp-validate', + 'bond-downdelay': 'down-delay', + 'bond-fail-over-mac': 'fail-over-mac-policy', + 'bond-lacp-rate': 'lacp-rate', + 'bond-miimon': 'mii-monitor-interval', + 'bond-min-links': 'min-links', + 'bond-mode': 'mode', + 'bond-num-grat-arp': 'gratuitious-arp', + 'bond-primary': 'primary', + 'bond-primary-reselect': 'primary-reselect-policy', + 'bond-updelay': 'up-delay', + 'bond-xmit-hash-policy': 'transmit-hash-policy'}, + 'bridge': {'bridge_ageing': 'ageing-time', + 'bridge_bridgeprio': 'priority', + 'bridge_fd': 'forward-delay', + 'bridge_gcint': None, + 'bridge_hello': 'hello-time', + 'bridge_maxage': 'max-age', + 'bridge_maxwait': None, + 'bridge_pathcost': 'path-cost', + 'bridge_portprio': None, + 'bridge_waitport': None}} + def parse_net_config_data(net_config, skip_broken=True): """Parses the config, returns NetworkState object @@ -119,6 +146,10 @@ class NetworkState(object): self._version = version self.use_ipv6 = network_state.get('use_ipv6', False) + @property + def config(self): + return self._network_state['config'] + @property def version(self): return self._version @@ -166,12 +197,14 @@ class NetworkStateInterpreter(object): 'search': [], }, 'use_ipv6': False, + 'config': None, } def __init__(self, version=NETWORK_STATE_VERSION, config=None): self._version = version self._config = config self._network_state = copy.deepcopy(self.initial_network_state) + self._network_state['config'] = config self._parsed = False @property @@ -460,12 +493,15 @@ class NetworkStateInterpreter(object): v2_command = { bond0: { 'interfaces': ['interface0', 'interface1'], - 'miimon': 100, - 'mode': '802.3ad', - 'xmit_hash_policy': 'layer3+4'}, + 'parameters': { + 'mii-monitor-interval': 100, + 'mode': '802.3ad', + 'xmit_hash_policy': 'layer3+4'}}, bond1: { 'bond-slaves': ['interface2', 'interface7'], - 'mode': 1 + 'parameters': { + 'mode': 1, + } } } @@ -554,6 +590,7 @@ class NetworkStateInterpreter(object): if not mac_address: LOG.debug('NetworkState Version2: missing "macaddress" info ' 'in config entry: %s: %s', eth, str(cfg)) + phy_cmd.update({'mac_address': mac_address}) for key in ['mtu', 'match', 'wakeonlan']: if key in cfg: @@ -598,8 +635,8 @@ class NetworkStateInterpreter(object): self.handle_vlan(vlan_cmd) def handle_wifis(self, command): - raise NotImplementedError("NetworkState V2: " - "Skipping wifi configuration") + LOG.warning('Wifi configuration is only available to distros with' + 'netplan rendering support.') def _v2_common(self, cfg): LOG.debug('v2_common: handling config:\n%s', cfg) @@ -616,6 +653,11 @@ class NetworkStateInterpreter(object): def _handle_bond_bridge(self, command, cmd_type=None): """Common handler for bond and bridge types""" + + # inverse mapping for v2 keynames to v1 keynames + v2key_to_v1 = dict((v, k) for k, v in + NET_CONFIG_TO_V2.get(cmd_type).items()) + for item_name, item_cfg in command.items(): item_params = dict((key, value) for (key, value) in item_cfg.items() if key not in @@ -624,14 +666,20 @@ class NetworkStateInterpreter(object): 'type': cmd_type, 'name': item_name, cmd_type + '_interfaces': item_cfg.get('interfaces'), - 'params': item_params, + 'params': dict((v2key_to_v1[k], v) for k, v in + item_params.get('parameters', {}).items()) } subnets = self._v2_to_v1_ipcfg(item_cfg) if len(subnets) > 0: v1_cmd.update({'subnets': subnets}) - LOG.debug('v2(%ss) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) - self.handle_bridge(v1_cmd) + LOG.debug('v2(%s) -> v1(%s):\n%s', cmd_type, cmd_type, v1_cmd) + if cmd_type == "bridge": + self.handle_bridge(v1_cmd) + elif cmd_type == "bond": + self.handle_bond(v1_cmd) + else: + raise ValueError('Unknown command type: %s', cmd_type) def _v2_to_v1_ipcfg(self, cfg): """Common ipconfig extraction from v2 to v1 subnets array.""" @@ -651,12 +699,6 @@ class NetworkStateInterpreter(object): 'address': address, } - routes = [] - for route in cfg.get('routes', []): - routes.append(_normalize_route( - {'address': route.get('to'), 'gateway': route.get('via')})) - subnet['routes'] = routes - if ":" in address: if 'gateway6' in cfg and gateway6 is None: gateway6 = cfg.get('gateway6') @@ -667,6 +709,17 @@ class NetworkStateInterpreter(object): subnet.update({'gateway': gateway4}) subnets.append(subnet) + + routes = [] + for route in cfg.get('routes', []): + routes.append(_normalize_route( + {'destination': route.get('to'), 'gateway': route.get('via')})) + + # v2 routes are bound to the interface, in v1 we add them under + # the first subnet since there isn't an equivalent interface level. + if len(subnets) and len(routes): + subnets[0]['routes'] = routes + return subnets @@ -721,7 +774,7 @@ def _normalize_net_keys(network, address_keys=()): elif netmask: prefix = mask_to_net_prefix(netmask) elif 'prefix' in net: - prefix = int(prefix) + prefix = int(net['prefix']) else: prefix = 64 if ipv6 else 24 diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 2f505d93..6d89dba8 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -135,7 +135,7 @@ network: V2_NET_CFG = { 'ethernets': { 'eth7': { - 'addresses': ['192.168.1.5/255.255.255.0'], + 'addresses': ['192.168.1.5/24'], 'gateway4': '192.168.1.254'}, 'eth9': { 'dhcp4': True} @@ -151,7 +151,6 @@ V2_TO_V2_NET_CFG_OUTPUT = """ # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} network: - version: 2 ethernets: eth7: addresses: @@ -159,6 +158,7 @@ network: gateway4: 192.168.1.254 eth9: dhcp4: true + version: 2 """ diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 4653be1a..f251024b 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1059,6 +1059,100 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true - type: static address: 2001:1::1/92 """), + 'expected_netplan': textwrap.dedent(""" + network: + version: 2 + ethernets: + bond0s0: + match: + macaddress: aa:bb:cc:dd:e8:00 + set-name: bond0s0 + bond0s1: + match: + macaddress: aa:bb:cc:dd:e8:01 + set-name: bond0s1 + bonds: + bond0: + addresses: + - 192.168.0.2/24 + - 192.168.1.2/24 + - 2001:1::1/92 + gateway4: 192.168.0.1 + interfaces: + - bond0s0 + - bond0s1 + parameters: + mii-monitor-interval: 100 + mode: active-backup + transmit-hash-policy: layer3+4 + routes: + - to: 10.1.3.0/24 + via: 192.168.0.3 + """), + 'yaml-v2': textwrap.dedent(""" + version: 2 + ethernets: + eth0: + match: + driver: "virtio_net" + macaddress: "aa:bb:cc:dd:e8:00" + vf0: + set-name: vf0 + match: + driver: "e1000" + macaddress: "aa:bb:cc:dd:e8:01" + bonds: + bond0: + addresses: + - 192.168.0.2/24 + - 192.168.1.2/24 + - 2001:1::1/92 + gateway4: 192.168.0.1 + interfaces: + - eth0 + - vf0 + parameters: + mii-monitor-interval: 100 + mode: active-backup + primary: vf0 + transmit-hash-policy: "layer3+4" + routes: + - to: 10.1.3.0/24 + via: 192.168.0.3 + """), + 'expected_netplan-v2': textwrap.dedent(""" + network: + bonds: + bond0: + addresses: + - 192.168.0.2/24 + - 192.168.1.2/24 + - 2001:1::1/92 + gateway4: 192.168.0.1 + interfaces: + - eth0 + - vf0 + parameters: + mii-monitor-interval: 100 + mode: active-backup + primary: vf0 + transmit-hash-policy: layer3+4 + routes: + - to: 10.1.3.0/24 + via: 192.168.0.3 + ethernets: + eth0: + match: + driver: virtio_net + macaddress: aa:bb:cc:dd:e8:00 + vf0: + match: + driver: e1000 + macaddress: aa:bb:cc:dd:e8:01 + set-name: vf0 + version: 2 + """), + 'expected_sysconfig': { 'ifcfg-bond0': textwrap.dedent("""\ BONDING_MASTER=yes @@ -2159,6 +2253,27 @@ class TestNetplanRoundTrip(CiTestCase): renderer.render_network_state(ns, target) return dir2dict(target) + def testsimple_render_bond_netplan(self): + entry = NETWORK_CONFIGS['bond'] + files = self._render_and_read(network_config=yaml.load(entry['yaml'])) + print(entry['expected_netplan']) + print('-- expected ^ | v rendered --') + print(files['/etc/netplan/50-cloud-init.yaml']) + self.assertEqual( + entry['expected_netplan'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + + def testsimple_render_bond_v2_input_netplan(self): + entry = NETWORK_CONFIGS['bond'] + files = self._render_and_read( + network_config=yaml.load(entry['yaml-v2'])) + print(entry['expected_netplan-v2']) + print('-- expected ^ | v rendered --') + print(files['/etc/netplan/50-cloud-init.yaml']) + self.assertEqual( + entry['expected_netplan-v2'].splitlines(), + files['/etc/netplan/50-cloud-init.yaml'].splitlines()) + def testsimple_render_small_netplan(self): entry = NETWORK_CONFIGS['small'] files = self._render_and_read(network_config=yaml.load(entry['yaml'])) -- cgit v1.2.3 From e74d7752f1761c3a8d3c19877de4707d00c49d08 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 21 Aug 2017 13:46:23 -0600 Subject: tools: Add tooling for basic cloud-init performance analysis. This branch adds cloudinit-analyze into cloud-init proper. It adds an "analyze" subcommand to the cloud-init command line utility for quick performance assessment of cloud-init stages and events. On a cloud-init configured instance, running "cloud-init analyze blame" will now report which cloud-init events cost the most wall time. This allows for quick assessment of the most costly stages of cloud-init. This functionality is pulled from Ryan Harper's analyze work. The cloudinit-analyze main script itself has been refactored a bit for inclusion as a subcommand of cloud-init CLI. There will be a followup branch at some point which will optionally instrument detailed strace profiling, but that approach needs a bit more discussion first. This branch also adds: * additional debugging topic to the sphinx-generated docs describing cloud-init analyze, dump and show as well as cloud-init single usage. * Updates the Makefile unittests target to include cloudinit directory because we now have unittests within that package. LP: #1709761 --- Makefile | 2 +- cloudinit/analyze/__init__.py | 0 cloudinit/analyze/__main__.py | 155 ++++++++++++++++++++++++++ cloudinit/analyze/dump.py | 176 +++++++++++++++++++++++++++++ cloudinit/analyze/show.py | 207 ++++++++++++++++++++++++++++++++++ cloudinit/analyze/tests/test_dump.py | 210 +++++++++++++++++++++++++++++++++++ cloudinit/cmd/main.py | 44 +++----- doc/rtd/index.rst | 1 + doc/rtd/topics/debugging.rst | 146 ++++++++++++++++++++++++ tests/unittests/test_cli.py | 87 ++++++++++++++- 10 files changed, 995 insertions(+), 33 deletions(-) create mode 100644 cloudinit/analyze/__init__.py create mode 100644 cloudinit/analyze/__main__.py create mode 100644 cloudinit/analyze/dump.py create mode 100644 cloudinit/analyze/show.py create mode 100644 cloudinit/analyze/tests/test_dump.py create mode 100644 doc/rtd/topics/debugging.rst diff --git a/Makefile b/Makefile index f280911f..9e7f4ee7 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ pyflakes3: @$(CWD)/tools/run-pyflakes3 unittest: clean_pyc - nosetests $(noseopts) tests/unittests + nosetests $(noseopts) tests/unittests cloudinit unittest3: clean_pyc nosetests3 $(noseopts) tests/unittests diff --git a/cloudinit/analyze/__init__.py b/cloudinit/analyze/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/analyze/__main__.py b/cloudinit/analyze/__main__.py new file mode 100644 index 00000000..71cba4f2 --- /dev/null +++ b/cloudinit/analyze/__main__.py @@ -0,0 +1,155 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +import argparse +import re +import sys + +from . import dump +from . import show + + +def get_parser(parser=None): + if not parser: + parser = argparse.ArgumentParser( + prog='cloudinit-analyze', + description='Devel tool: Analyze cloud-init logs and data') + subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand') + subparsers.required = True + + parser_blame = subparsers.add_parser( + 'blame', help='Print list of executed stages ordered by time to init') + parser_blame.add_argument( + '-i', '--infile', action='store', dest='infile', + default='/var/log/cloud-init.log', + help='specify where to read input.') + parser_blame.add_argument( + '-o', '--outfile', action='store', dest='outfile', default='-', + help='specify where to write output. ') + parser_blame.set_defaults(action=('blame', analyze_blame)) + + parser_show = subparsers.add_parser( + 'show', help='Print list of in-order events during execution') + parser_show.add_argument('-f', '--format', action='store', + dest='print_format', default='%I%D @%Es +%ds', + help='specify formatting of output.') + parser_show.add_argument('-i', '--infile', action='store', + dest='infile', default='/var/log/cloud-init.log', + help='specify where to read input.') + parser_show.add_argument('-o', '--outfile', action='store', + dest='outfile', default='-', + help='specify where to write output.') + parser_show.set_defaults(action=('show', analyze_show)) + parser_dump = subparsers.add_parser( + 'dump', help='Dump cloud-init events in JSON format') + parser_dump.add_argument('-i', '--infile', action='store', + dest='infile', default='/var/log/cloud-init.log', + help='specify where to read input. ') + parser_dump.add_argument('-o', '--outfile', action='store', + dest='outfile', default='-', + help='specify where to write output. ') + parser_dump.set_defaults(action=('dump', analyze_dump)) + return parser + + +def analyze_blame(name, args): + """Report a list of records sorted by largest time delta. + + For example: + 30.210s (init-local) searching for datasource + 8.706s (init-network) reading and applying user-data + 166ms (modules-config) .... + 807us (modules-final) ... + + We generate event records parsing cloud-init logs, formatting the output + and sorting by record data ('delta') + """ + (infh, outfh) = configure_io(args) + blame_format = ' %ds (%n)' + r = re.compile('(^\s+\d+\.\d+)', re.MULTILINE) + for idx, record in enumerate(show.show_events(_get_events(infh), + blame_format)): + srecs = sorted(filter(r.match, record), reverse=True) + outfh.write('-- Boot Record %02d --\n' % (idx + 1)) + outfh.write('\n'.join(srecs) + '\n') + outfh.write('\n') + outfh.write('%d boot records analyzed\n' % (idx + 1)) + + +def analyze_show(name, args): + """Generate output records using the 'standard' format to printing events. + + Example output follows: + Starting stage: (init-local) + ... + Finished stage: (init-local) 0.105195 seconds + + Starting stage: (init-network) + ... + Finished stage: (init-network) 0.339024 seconds + + Starting stage: (modules-config) + ... + Finished stage: (modules-config) 0.NNN seconds + + Starting stage: (modules-final) + ... + Finished stage: (modules-final) 0.NNN seconds + """ + (infh, outfh) = configure_io(args) + for idx, record in enumerate(show.show_events(_get_events(infh), + args.print_format)): + outfh.write('-- Boot Record %02d --\n' % (idx + 1)) + outfh.write('The total time elapsed since completing an event is' + ' printed after the "@" character.\n') + outfh.write('The time the event takes is printed after the "+" ' + 'character.\n\n') + outfh.write('\n'.join(record) + '\n') + outfh.write('%d boot records analyzed\n' % (idx + 1)) + + +def analyze_dump(name, args): + """Dump cloud-init events in json format""" + (infh, outfh) = configure_io(args) + outfh.write(dump.json_dumps(_get_events(infh)) + '\n') + + +def _get_events(infile): + rawdata = None + events, rawdata = show.load_events(infile, None) + if not events: + events, _ = dump.dump_events(rawdata=rawdata) + return events + + +def configure_io(args): + """Common parsing and setup of input/output files""" + if args.infile == '-': + infh = sys.stdin + else: + try: + infh = open(args.infile, 'r') + except (FileNotFoundError, PermissionError): + sys.stderr.write('Cannot open file %s\n' % args.infile) + sys.exit(1) + + if args.outfile == '-': + outfh = sys.stdout + else: + try: + outfh = open(args.outfile, 'w') + except PermissionError: + sys.stderr.write('Cannot open file %s\n' % args.outfile) + sys.exit(1) + + return (infh, outfh) + + +if __name__ == '__main__': + parser = get_parser() + args = parser.parse_args() + (name, action_functor) = args.action + action_functor(name, args) + +# vi: ts=4 expandtab diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py new file mode 100644 index 00000000..ca4da496 --- /dev/null +++ b/cloudinit/analyze/dump.py @@ -0,0 +1,176 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import calendar +from datetime import datetime +import json +import sys + +from cloudinit import util + +stage_to_description = { + 'finished': 'finished running cloud-init', + 'init-local': 'starting search for local datasources', + 'init-network': 'searching for network datasources', + 'init': 'searching for network datasources', + 'modules-config': 'running config modules', + 'modules-final': 'finalizing modules', + 'modules': 'running modules for', + 'single': 'running single module ', +} + +# logger's asctime format +CLOUD_INIT_ASCTIME_FMT = "%Y-%m-%d %H:%M:%S,%f" + +# journctl -o short-precise +CLOUD_INIT_JOURNALCTL_FMT = "%b %d %H:%M:%S.%f %Y" + +# other +DEFAULT_FMT = "%b %d %H:%M:%S %Y" + + +def parse_timestamp(timestampstr): + # default syslog time does not include the current year + months = [calendar.month_abbr[m] for m in range(1, 13)] + if timestampstr.split()[0] in months: + # Aug 29 22:55:26 + FMT = DEFAULT_FMT + if '.' in timestampstr: + FMT = CLOUD_INIT_JOURNALCTL_FMT + dt = datetime.strptime(timestampstr + " " + + str(datetime.now().year), + FMT) + timestamp = dt.strftime("%s.%f") + elif "," in timestampstr: + # 2016-09-12 14:39:20,839 + dt = datetime.strptime(timestampstr, CLOUD_INIT_ASCTIME_FMT) + timestamp = dt.strftime("%s.%f") + else: + # allow date(1) to handle other formats we don't expect + timestamp = parse_timestamp_from_date(timestampstr) + + return float(timestamp) + + +def parse_timestamp_from_date(timestampstr): + out, _ = util.subp(['date', '+%s.%3N', '-d', timestampstr]) + timestamp = out.strip() + return float(timestamp) + + +def parse_ci_logline(line): + # Stage Starts: + # Cloud-init v. 0.7.7 running 'init-local' at \ + # Fri, 02 Sep 2016 19:28:07 +0000. Up 1.0 seconds. + # Cloud-init v. 0.7.7 running 'init' at \ + # Fri, 02 Sep 2016 19:28:08 +0000. Up 2.0 seconds. + # Cloud-init v. 0.7.7 finished at + # Aug 29 22:55:26 test1 [CLOUDINIT] handlers.py[DEBUG]: \ + # finish: modules-final: SUCCESS: running modules for final + # 2016-08-30T21:53:25.972325+00:00 y1 [CLOUDINIT] handlers.py[DEBUG]: \ + # finish: modules-final: SUCCESS: running modules for final + # + # Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]: \ + # Cloud-init v. 0.7.8 running 'init-local' at \ + # Thu, 03 Nov 2016 06:51:06 +0000. Up 1.0 seconds. + # + # 2017-05-22 18:02:01,088 - util.py[DEBUG]: Cloud-init v. 0.7.9 running \ + # 'init-local' at Mon, 22 May 2017 18:02:01 +0000. Up 2.0 seconds. + + separators = [' - ', ' [CLOUDINIT] '] + found = False + for sep in separators: + if sep in line: + found = True + break + + if not found: + return None + + (timehost, eventstr) = line.split(sep) + + # journalctl -o short-precise + if timehost.endswith(":"): + timehost = " ".join(timehost.split()[0:-1]) + + if "," in timehost: + timestampstr, extra = timehost.split(",") + timestampstr += ",%s" % extra.split()[0] + if ' ' in extra: + hostname = extra.split()[-1] + else: + hostname = timehost.split()[-1] + timestampstr = timehost.split(hostname)[0].strip() + if 'Cloud-init v.' in eventstr: + event_type = 'start' + if 'running' in eventstr: + stage_and_timestamp = eventstr.split('running')[1].lstrip() + event_name, _ = stage_and_timestamp.split(' at ') + event_name = event_name.replace("'", "").replace(":", "-") + if event_name == "init": + event_name = "init-network" + else: + # don't generate a start for the 'finished at' banner + return None + event_description = stage_to_description[event_name] + else: + (pymodloglvl, event_type, event_name) = eventstr.split()[0:3] + event_description = eventstr.split(event_name)[1].strip() + + event = { + 'name': event_name.rstrip(":"), + 'description': event_description, + 'timestamp': parse_timestamp(timestampstr), + 'origin': 'cloudinit', + 'event_type': event_type.rstrip(":"), + } + if event['event_type'] == "finish": + result = event_description.split(":")[0] + desc = event_description.split(result)[1].lstrip(':').strip() + event['result'] = result + event['description'] = desc.strip() + + return event + + +def json_dumps(data): + return json.dumps(data, indent=1, sort_keys=True, + separators=(',', ': ')) + + +def dump_events(cisource=None, rawdata=None): + events = [] + event = None + CI_EVENT_MATCHES = ['start:', 'finish:', 'Cloud-init v.'] + + if not any([cisource, rawdata]): + raise ValueError('Either cisource or rawdata parameters are required') + + if rawdata: + data = rawdata.splitlines() + else: + data = cisource.readlines() + + for line in data: + for match in CI_EVENT_MATCHES: + if match in line: + try: + event = parse_ci_logline(line) + except ValueError: + sys.stderr.write('Skipping invalid entry\n') + if event: + events.append(event) + + return events, data + + +def main(): + if len(sys.argv) > 1: + cisource = open(sys.argv[1]) + else: + cisource = sys.stdin + + return json_dumps(dump_events(cisource)) + + +if __name__ == "__main__": + print(main()) diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py new file mode 100644 index 00000000..3b356bb8 --- /dev/null +++ b/cloudinit/analyze/show.py @@ -0,0 +1,207 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Ryan Harper +# +# This file is part of cloud-init. See LICENSE file for license information. + +import base64 +import datetime +import json +import os + +from cloudinit import util + +# An event: +''' +{ + "description": "executing late commands", + "event_type": "start", + "level": "INFO", + "name": "cmd-install/stage-late" + "origin": "cloudinit", + "timestamp": 1461164249.1590767, +}, + + { + "description": "executing late commands", + "event_type": "finish", + "level": "INFO", + "name": "cmd-install/stage-late", + "origin": "cloudinit", + "result": "SUCCESS", + "timestamp": 1461164249.1590767 + } + +''' +format_key = { + '%d': 'delta', + '%D': 'description', + '%E': 'elapsed', + '%e': 'event_type', + '%I': 'indent', + '%l': 'level', + '%n': 'name', + '%o': 'origin', + '%r': 'result', + '%t': 'timestamp', + '%T': 'total_time', +} + +formatting_help = " ".join(["{0}: {1}".format(k.replace('%', '%%'), v) + for k, v in format_key.items()]) + + +def format_record(msg, event): + for i, j in format_key.items(): + if i in msg: + # ensure consistent formatting of time values + if j in ['delta', 'elapsed', 'timestamp']: + msg = msg.replace(i, "{%s:08.5f}" % j) + else: + msg = msg.replace(i, "{%s}" % j) + return msg.format(**event) + + +def dump_event_files(event): + content = dict((k, v) for k, v in event.items() if k not in ['content']) + files = content['files'] + saved = [] + for f in files: + fname = f['path'] + fn_local = os.path.basename(fname) + fcontent = base64.b64decode(f['content']).decode('ascii') + util.write_file(fn_local, fcontent) + saved.append(fn_local) + + return saved + + +def event_name(event): + if event: + return event.get('name') + return None + + +def event_type(event): + if event: + return event.get('event_type') + return None + + +def event_parent(event): + if event: + return event_name(event).split("/")[0] + return None + + +def event_timestamp(event): + return float(event.get('timestamp')) + + +def event_datetime(event): + return datetime.datetime.utcfromtimestamp(event_timestamp(event)) + + +def delta_seconds(t1, t2): + return (t2 - t1).total_seconds() + + +def event_duration(start, finish): + return delta_seconds(event_datetime(start), event_datetime(finish)) + + +def event_record(start_time, start, finish): + record = finish.copy() + record.update({ + 'delta': event_duration(start, finish), + 'elapsed': delta_seconds(start_time, event_datetime(start)), + 'indent': '|' + ' ' * (event_name(start).count('/') - 1) + '`->', + }) + + return record + + +def total_time_record(total_time): + return 'Total Time: %3.5f seconds\n' % total_time + + +def generate_records(events, blame_sort=False, + print_format="(%n) %d seconds in %I%D", + dump_files=False, log_datafiles=False): + + sorted_events = sorted(events, key=lambda x: x['timestamp']) + records = [] + start_time = None + total_time = 0.0 + stage_start_time = {} + stages_seen = [] + boot_records = [] + + unprocessed = [] + for e in range(0, len(sorted_events)): + event = events[e] + try: + next_evt = events[e + 1] + except IndexError: + next_evt = None + + if event_type(event) == 'start': + if event.get('name') in stages_seen: + records.append(total_time_record(total_time)) + boot_records.append(records) + records = [] + start_time = None + total_time = 0.0 + + if start_time is None: + stages_seen = [] + start_time = event_datetime(event) + stage_start_time[event_parent(event)] = start_time + + # see if we have a pair + if event_name(event) == event_name(next_evt): + if event_type(next_evt) == 'finish': + records.append(format_record(print_format, + event_record(start_time, + event, + next_evt))) + else: + # This is a parent event + records.append("Starting stage: %s" % event.get('name')) + unprocessed.append(event) + stages_seen.append(event.get('name')) + continue + else: + prev_evt = unprocessed.pop() + if event_name(event) == event_name(prev_evt): + record = event_record(start_time, prev_evt, event) + records.append(format_record("Finished stage: " + "(%n) %d seconds ", + record) + "\n") + total_time += record.get('delta') + else: + # not a match, put it back + unprocessed.append(prev_evt) + + records.append(total_time_record(total_time)) + boot_records.append(records) + return boot_records + + +def show_events(events, print_format): + return generate_records(events, print_format=print_format) + + +def load_events(infile, rawdata=None): + if rawdata: + data = rawdata.read() + else: + data = infile.read() + + j = None + try: + j = json.loads(data) + except json.JSONDecodeError: + pass + + return j, data diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py new file mode 100644 index 00000000..2c0885d0 --- /dev/null +++ b/cloudinit/analyze/tests/test_dump.py @@ -0,0 +1,210 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from datetime import datetime +from textwrap import dedent + +from cloudinit.analyze.dump import ( + dump_events, parse_ci_logline, parse_timestamp) +from cloudinit.util import subp, write_file +from tests.unittests.helpers import CiTestCase + + +class TestParseTimestamp(CiTestCase): + + def test_parse_timestamp_handles_cloud_init_default_format(self): + """Logs with cloud-init detailed formats will be properly parsed.""" + trusty_fmt = '%Y-%m-%d %H:%M:%S,%f' + trusty_stamp = '2016-09-12 14:39:20,839' + + parsed = parse_timestamp(trusty_stamp) + + # convert ourselves + dt = datetime.strptime(trusty_stamp, trusty_fmt) + expected = float(dt.strftime('%s.%f')) + + # use date(1) + out, _err = subp(['date', '+%s.%3N', '-d', trusty_stamp]) + timestamp = out.strip() + date_ts = float(timestamp) + + self.assertEqual(expected, parsed) + self.assertEqual(expected, date_ts) + self.assertEqual(date_ts, parsed) + + def test_parse_timestamp_handles_syslog_adding_year(self): + """Syslog timestamps lack a year. Add year and properly parse.""" + syslog_fmt = '%b %d %H:%M:%S %Y' + syslog_stamp = 'Aug 08 15:12:51' + + # convert stamp ourselves by adding the missing year value + year = datetime.now().year + dt = datetime.strptime(syslog_stamp + " " + str(year), syslog_fmt) + expected = float(dt.strftime('%s.%f')) + parsed = parse_timestamp(syslog_stamp) + + # use date(1) + out, _ = subp(['date', '+%s.%3N', '-d', syslog_stamp]) + timestamp = out.strip() + date_ts = float(timestamp) + + self.assertEqual(expected, parsed) + self.assertEqual(expected, date_ts) + self.assertEqual(date_ts, parsed) + + def test_parse_timestamp_handles_journalctl_format_adding_year(self): + """Journalctl precise timestamps lack a year. Add year and parse.""" + journal_fmt = '%b %d %H:%M:%S.%f %Y' + journal_stamp = 'Aug 08 17:15:50.606811' + + # convert stamp ourselves by adding the missing year value + year = datetime.now().year + dt = datetime.strptime(journal_stamp + " " + str(year), journal_fmt) + expected = float(dt.strftime('%s.%f')) + parsed = parse_timestamp(journal_stamp) + + # use date(1) + out, _ = subp(['date', '+%s.%6N', '-d', journal_stamp]) + timestamp = out.strip() + date_ts = float(timestamp) + + self.assertEqual(expected, parsed) + self.assertEqual(expected, date_ts) + self.assertEqual(date_ts, parsed) + + def test_parse_unexpected_timestamp_format_with_date_command(self): + """Dump sends unexpected timestamp formats to data for processing.""" + new_fmt = '%H:%M %m/%d %Y' + new_stamp = '17:15 08/08' + + # convert stamp ourselves by adding the missing year value + year = datetime.now().year + dt = datetime.strptime(new_stamp + " " + str(year), new_fmt) + expected = float(dt.strftime('%s.%f')) + parsed = parse_timestamp(new_stamp) + + # use date(1) + out, _ = subp(['date', '+%s.%6N', '-d', new_stamp]) + timestamp = out.strip() + date_ts = float(timestamp) + + self.assertEqual(expected, parsed) + self.assertEqual(expected, date_ts) + self.assertEqual(date_ts, parsed) + + +class TestParseCILogLine(CiTestCase): + + def test_parse_logline_returns_none_without_separators(self): + """When no separators are found, parse_ci_logline returns None.""" + expected_parse_ignores = [ + '', '-', 'adsf-asdf', '2017-05-22 18:02:01,088', 'CLOUDINIT'] + for parse_ignores in expected_parse_ignores: + self.assertIsNone(parse_ci_logline(parse_ignores)) + + def test_parse_logline_returns_event_for_cloud_init_logs(self): + """parse_ci_logline returns an event parse from cloud-init format.""" + line = ( + "2017-08-08 20:05:07,147 - util.py[DEBUG]: Cloud-init v. 0.7.9" + " running 'init-local' at Tue, 08 Aug 2017 20:05:07 +0000. Up" + " 6.26 seconds.") + dt = datetime.strptime( + '2017-08-08 20:05:07,147', '%Y-%m-%d %H:%M:%S,%f') + timestamp = float(dt.strftime('%s.%f')) + expected = { + 'description': 'starting search for local datasources', + 'event_type': 'start', + 'name': 'init-local', + 'origin': 'cloudinit', + 'timestamp': timestamp} + self.assertEqual(expected, parse_ci_logline(line)) + + def test_parse_logline_returns_event_for_journalctl_logs(self): + """parse_ci_logline returns an event parse from journalctl format.""" + line = ("Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT]" + " util.py[DEBUG]: Cloud-init v. 0.7.8 running 'init-local' at" + " Thu, 03 Nov 2016 06:51:06 +0000. Up 1.0 seconds.") + year = datetime.now().year + dt = datetime.strptime( + 'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y') + timestamp = float(dt.strftime('%s.%f')) + expected = { + 'description': 'starting search for local datasources', + 'event_type': 'start', + 'name': 'init-local', + 'origin': 'cloudinit', + 'timestamp': timestamp} + self.assertEqual(expected, parse_ci_logline(line)) + + def test_parse_logline_returns_event_for_finish_events(self): + """parse_ci_logline returns a finish event for a parsed log line.""" + line = ('2016-08-30 21:53:25.972325+00:00 y1 [CLOUDINIT]' + ' handlers.py[DEBUG]: finish: modules-final: SUCCESS: running' + ' modules for final') + expected = { + 'description': 'running modules for final', + 'event_type': 'finish', + 'name': 'modules-final', + 'origin': 'cloudinit', + 'result': 'SUCCESS', + 'timestamp': 1472594005.972} + self.assertEqual(expected, parse_ci_logline(line)) + + +SAMPLE_LOGS = dedent("""\ +Nov 03 06:51:06.074410 x2 cloud-init[106]: [CLOUDINIT] util.py[DEBUG]:\ + Cloud-init v. 0.7.8 running 'init-local' at Thu, 03 Nov 2016\ + 06:51:06 +0000. Up 1.0 seconds. +2016-08-30 21:53:25.972325+00:00 y1 [CLOUDINIT] handlers.py[DEBUG]: finish:\ + modules-final: SUCCESS: running modules for final +""") + + +class TestDumpEvents(CiTestCase): + maxDiff = None + + def test_dump_events_with_rawdata(self): + """Rawdata is split and parsed into a tuple of events and data""" + events, data = dump_events(rawdata=SAMPLE_LOGS) + expected_data = SAMPLE_LOGS.splitlines() + year = datetime.now().year + dt1 = datetime.strptime( + 'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y') + timestamp1 = float(dt1.strftime('%s.%f')) + expected_events = [{ + 'description': 'starting search for local datasources', + 'event_type': 'start', + 'name': 'init-local', + 'origin': 'cloudinit', + 'timestamp': timestamp1}, { + 'description': 'running modules for final', + 'event_type': 'finish', + 'name': 'modules-final', + 'origin': 'cloudinit', + 'result': 'SUCCESS', + 'timestamp': 1472594005.972}] + self.assertEqual(expected_events, events) + self.assertEqual(expected_data, data) + + def test_dump_events_with_cisource(self): + """Cisource file is read and parsed into a tuple of events and data.""" + tmpfile = self.tmp_path('logfile') + write_file(tmpfile, SAMPLE_LOGS) + events, data = dump_events(cisource=open(tmpfile)) + year = datetime.now().year + dt1 = datetime.strptime( + 'Nov 03 06:51:06.074410 %d' % year, '%b %d %H:%M:%S.%f %Y') + timestamp1 = float(dt1.strftime('%s.%f')) + expected_events = [{ + 'description': 'starting search for local datasources', + 'event_type': 'start', + 'name': 'init-local', + 'origin': 'cloudinit', + 'timestamp': timestamp1}, { + 'description': 'running modules for final', + 'event_type': 'finish', + 'name': 'modules-final', + 'origin': 'cloudinit', + 'result': 'SUCCESS', + 'timestamp': 1472594005.972}] + self.assertEqual(expected_events, events) + self.assertEqual(SAMPLE_LOGS.splitlines(), [d.strip() for d in data]) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 139e03b3..9c0ac864 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -50,13 +50,6 @@ WELCOME_MSG_TPL = ("Cloud-init v. {version} running '{action}' at " # Module section template MOD_SECTION_TPL = "cloud_%s_modules" -# Things u can query on -QUERY_DATA_TYPES = [ - 'data', - 'data_raw', - 'instance_id', -] - # Frequency shortname to full name # (so users don't have to remember the full name...) FREQ_SHORT_NAMES = { @@ -510,11 +503,6 @@ def main_modules(action_name, args): return run_module_section(mods, name, name) -def main_query(name, _args): - raise NotImplementedError(("Action '%s' is not" - " currently implemented") % (name)) - - def main_single(name, args): # Cloud-init single stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors @@ -713,9 +701,11 @@ def main(sysv_args=None): default=False) parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers() + subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand') + subparsers.required = True # Each action and its sub-options (if any) + parser_init = subparsers.add_parser('init', help=('initializes cloud-init and' ' performs initial modules')) @@ -737,17 +727,6 @@ def main(sysv_args=None): choices=('init', 'config', 'final')) parser_mod.set_defaults(action=('modules', main_modules)) - # These settings are used when you want to query information - # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', - help=('query information stored ' - 'in cloud-init')) - parser_query.add_argument("--name", '-n', action="store", - help="item name to query on", - required=True, - choices=QUERY_DATA_TYPES) - parser_query.set_defaults(action=('query', main_query)) - # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', help=('run a single module ')) @@ -781,15 +760,22 @@ def main(sysv_args=None): help=('list defined features')) parser_features.set_defaults(action=('features', main_features)) + parser_analyze = subparsers.add_parser( + 'analyze', help='Devel tool: Analyze cloud-init logs and data') + if sysv_args and sysv_args[0] == 'analyze': + # Only load this parser if analyze is specified to avoid file load cost + # FIXME put this under 'devel' subcommand (coming in next branch) + from cloudinit.analyze.__main__ import get_parser as analyze_parser + # Construct analyze subcommand parser + analyze_parser(parser_analyze) + args = parser.parse_args(args=sysv_args) - try: - (name, functor) = args.action - except AttributeError: - parser.error('too few arguments') + # Subparsers.required = True and each subparser sets action=(name, functor) + (name, functor) = args.action # Setup basic logging to start (until reinitialized) - # iff in debug mode... + # iff in debug mode. if args.debug: logging.setupBasicLogging() diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index a691103e..de67f361 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -40,6 +40,7 @@ initialization of a cloud instance. topics/merging.rst topics/network-config.rst topics/vendordata.rst + topics/debugging.rst topics/moreinfo.rst topics/hacking.rst topics/tests.rst diff --git a/doc/rtd/topics/debugging.rst b/doc/rtd/topics/debugging.rst new file mode 100644 index 00000000..4e43dd57 --- /dev/null +++ b/doc/rtd/topics/debugging.rst @@ -0,0 +1,146 @@ +********************** +Testing and debugging cloud-init +********************** + +Overview +======== +This topic will discuss general approaches for test and debug of cloud-init on +deployed instances. + + +Boot Time Analysis - cloud-init analyze +====================================== +Occasionally instances don't appear as performant as we would like and +cloud-init packages a simple facility to inspect what operations took +cloud-init the longest during boot and setup. + +The script **/usr/bin/cloud-init** has an analyze sub-command **analyze** +which parses any cloud-init.log file into formatted and sorted events. It +allows for detailed analysis of the most costly cloud-init operations are to +determine the long-pole in cloud-init configuration and setup. These +subcommands default to reading /var/log/cloud-init.log. + +* ``analyze show`` Parse and organize cloud-init.log events by stage and +include each sub-stage granularity with time delta reports. + +.. code-block:: bash + + $ cloud-init analyze show -i my-cloud-init.log + -- Boot Record 01 -- + The total time elapsed since completing an event is printed after the "@" + character. + The time the event takes is printed after the "+" character. + + Starting stage: modules-config + |`->config-emit_upstart ran successfully @05.47600s +00.00100s + |`->config-snap_config ran successfully @05.47700s +00.00100s + |`->config-ssh-import-id ran successfully @05.47800s +00.00200s + |`->config-locale ran successfully @05.48000s +00.00100s + ... + + +* ``analyze dump`` Parse cloud-init.log into event records and return a list of +dictionaries that can be consumed for other reporting needs. + +.. code-block:: bash + + $ cloud-init analyze blame -i my-cloud-init.log + [ + { + "description": "running config modules", + "event_type": "start", + "name": "modules-config", + "origin": "cloudinit", + "timestamp": 1510807493.0 + },... + +* ``analyze blame`` Parse cloud-init.log into event records and sort them based +on highest time cost for quick assessment of areas of cloud-init that may need +improvement. + +.. code-block:: bash + + $ cloud-init analyze blame -i my-cloud-init.log + -- Boot Record 11 -- + 00.01300s (modules-final/config-scripts-per-boot) + 00.00400s (modules-final/config-final-message) + 00.00100s (modules-final/config-rightscale_userdata) + ... + + +Analyze quickstart - LXC +--------------------------- +To quickly obtain a cloud-init log try using lxc on any ubuntu system: + +.. code-block:: bash + + $ lxc init ubuntu-daily:xenial x1 + $ lxc start x1 + # Take lxc's cloud-init.log and pipe it to the analyzer + $ lxc file pull x1/var/log/cloud-init.log - | cloud-init analyze dump -i - + $ lxc file pull x1/var/log/cloud-init.log - | \ + python3 -m cloudinit.analyze dump -i - + +Analyze quickstart - KVM +--------------------------- +To quickly analyze a KVM a cloud-init log: + +1. Download the current cloud image + wget https://cloud-images.ubuntu.com/daily/server/xenial/current/xenial-server-cloudimg-amd64.img +2. Create a snapshot image to preserve the original cloud-image + +.. code-block:: bash + + $ qemu-img create -b xenial-server-cloudimg-amd64.img -f qcow2 \ + test-cloudinit.qcow2 + +3. Create a seed image with metadata using `cloud-localds` + +.. code-block:: bash + + $ cat > user-data <.`` which marks +when the module last successfully ran. Presence of this semaphore file +prevents a module from running again if it has already been run. To ensure that +a module is run again, the desired frequency can be overridden on the +commandline: + +.. code-block:: bash + + $ sudo cloud-init single --name cc_ssh --frequency always + ... + Generating public/private ed25519 key pair + ... + +Inspect cloud-init.log for output of what operations were performed as a +result. diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 06f366b2..7780f164 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -31,9 +31,90 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): def test_no_arguments_shows_error_message(self): exit_code = self._call_main() - self.assertIn('cloud-init: error: too few arguments', - self.stderr.getvalue()) + missing_subcommand_message = [ + 'too few arguments', # python2.7 msg + 'the following arguments are required: subcommand' # python3 msg + ] + error = self.stderr.getvalue() + matches = ([msg in error for msg in missing_subcommand_message]) + self.assertTrue( + any(matches), 'Did not find error message for missing subcommand') self.assertEqual(2, exit_code) + def test_all_subcommands_represented_in_help(self): + """All known subparsers are represented in the cloud-int help doc.""" + self._call_main() + error = self.stderr.getvalue() + expected_subcommands = ['analyze', 'init', 'modules', 'single', + 'dhclient-hook', 'features'] + for subcommand in expected_subcommands: + self.assertIn(subcommand, error) -# vi: ts=4 expandtab + @mock.patch('cloudinit.cmd.main.status_wrapper') + def test_init_subcommand_parser(self, m_status_wrapper): + """The subcommand 'init' calls status_wrapper passing init.""" + self._call_main(['cloud-init', 'init']) + (name, parseargs) = m_status_wrapper.call_args_list[0][0] + self.assertEqual('init', name) + self.assertEqual('init', parseargs.subcommand) + self.assertEqual('init', parseargs.action[0]) + self.assertEqual('main_init', parseargs.action[1].__name__) + + @mock.patch('cloudinit.cmd.main.status_wrapper') + def test_modules_subcommand_parser(self, m_status_wrapper): + """The subcommand 'modules' calls status_wrapper passing modules.""" + self._call_main(['cloud-init', 'modules']) + (name, parseargs) = m_status_wrapper.call_args_list[0][0] + self.assertEqual('modules', name) + self.assertEqual('modules', parseargs.subcommand) + self.assertEqual('modules', parseargs.action[0]) + self.assertEqual('main_modules', parseargs.action[1].__name__) + + def test_analyze_subcommand_parser(self): + """The subcommand cloud-init analyze calls the correct subparser.""" + self._call_main(['cloud-init', 'analyze']) + # These subcommands only valid for cloud-init analyze script + expected_subcommands = ['blame', 'show', 'dump'] + error = self.stderr.getvalue() + for subcommand in expected_subcommands: + self.assertIn(subcommand, error) + + @mock.patch('cloudinit.cmd.main.main_single') + def test_single_subcommand(self, m_main_single): + """The subcommand 'single' calls main_single with valid args.""" + self._call_main(['cloud-init', 'single', '--name', 'cc_ntp']) + (name, parseargs) = m_main_single.call_args_list[0][0] + self.assertEqual('single', name) + self.assertEqual('single', parseargs.subcommand) + self.assertEqual('single', parseargs.action[0]) + self.assertFalse(parseargs.debug) + self.assertFalse(parseargs.force) + self.assertIsNone(parseargs.frequency) + self.assertEqual('cc_ntp', parseargs.name) + self.assertFalse(parseargs.report) + + @mock.patch('cloudinit.cmd.main.dhclient_hook') + def test_dhclient_hook_subcommand(self, m_dhclient_hook): + """The subcommand 'dhclient-hook' calls dhclient_hook with args.""" + self._call_main(['cloud-init', 'dhclient-hook', 'net_action', 'eth0']) + (name, parseargs) = m_dhclient_hook.call_args_list[0][0] + self.assertEqual('dhclient_hook', name) + self.assertEqual('dhclient-hook', parseargs.subcommand) + self.assertEqual('dhclient_hook', parseargs.action[0]) + self.assertFalse(parseargs.debug) + self.assertFalse(parseargs.force) + self.assertEqual('net_action', parseargs.net_action) + self.assertEqual('eth0', parseargs.net_interface) + + @mock.patch('cloudinit.cmd.main.main_features') + def test_features_hook_subcommand(self, m_features): + """The subcommand 'features' calls main_features with args.""" + self._call_main(['cloud-init', 'features']) + (name, parseargs) = m_features.call_args_list[0][0] + self.assertEqual('features', name) + self.assertEqual('features', parseargs.subcommand) + self.assertEqual('features', parseargs.action[0]) + self.assertFalse(parseargs.debug) + self.assertFalse(parseargs.force) + +# : ts=4 expandtab -- cgit v1.2.3 From 3395a331c014dd7b83e93a1e2b66bb55b1966d83 Mon Sep 17 00:00:00 2001 From: Joonas Kylmälä Date: Sun, 20 Aug 2017 13:53:20 +0300 Subject: Debian: Remove non-free repositories from apt sources template. The Debian GNU/Linux distribution doesn't come offically with the non-free repositories enabled. Therefore, we want to disable those in the cloud-init template. LP: #1700091 --- templates/sources.list.debian.tmpl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/sources.list.debian.tmpl b/templates/sources.list.debian.tmpl index d64ace4d..e7ef9ed1 100644 --- a/templates/sources.list.debian.tmpl +++ b/templates/sources.list.debian.tmpl @@ -10,15 +10,15 @@ # See http://www.debian.org/releases/stable/i386/release-notes/ch-upgrading.html # for how to upgrade to newer versions of the distribution. -deb {{mirror}} {{codename}} main contrib non-free -deb-src {{mirror}} {{codename}} main contrib non-free +deb {{mirror}} {{codename}} main +deb-src {{mirror}} {{codename}} main ## Major bug fix updates produced after the final release of the ## distribution. -deb {{security}} {{codename}}/updates main contrib non-free -deb-src {{security}} {{codename}}/updates main contrib non-free -deb {{mirror}} {{codename}}-updates main contrib non-free -deb-src {{mirror}} {{codename}}-updates main contrib non-free +deb {{security}} {{codename}}/updates main +deb-src {{security}} {{codename}}/updates main +deb {{mirror}} {{codename}}-updates main +deb-src {{mirror}} {{codename}}-updates main ## Uncomment the following two lines to add software from the 'backports' ## repository. @@ -26,5 +26,5 @@ deb-src {{mirror}} {{codename}}-updates main contrib non-free ## N.B. software from this repository may not have been tested as ## extensively as that contained in the main release, although it includes ## newer versions of some applications which may provide useful features. -deb {{mirror}} {{codename}}-backports main contrib non-free -deb-src {{mirror}} {{codename}}-backports main contrib non-free +deb {{mirror}} {{codename}}-backports main +deb-src {{mirror}} {{codename}}-backports main -- cgit v1.2.3 From cc9762a2d737ead386ffb9f067adc5e543224560 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 22 Aug 2017 20:06:20 -0600 Subject: schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema This branch does a few things: - Add 'schema' subcommand to cloud-init CLI for validating cloud-config files against strict module jsonschema definitions - Add --annotate parameter to 'cloud-init schema' to annotate existing cloud-config file content with validation errors - Add jsonschema definition to cc_runcmd - Add unit test coverage for cc_runcmd - Update CLI capabilities documentation This branch only imports development (and analyze) subparsers when the specific subcommand is provided on the CLI to avoid adding costly unused file imports during cloud-init system boot. The schema command allows a person to quickly validate a cloud-config text file against cloud-init's known module schemas to avoid costly roundtrips deploying instances in their cloud of choice. As of this branch, only cc_ntp and cc_runcmd cloud-config modules define schemas. Schema validation will ignore all undefined config keys until all modules define a strict schema. To perform validation of runcmd and ntp sections of a cloud-config file: $ cat > cloud.cfg < Date: Wed, 23 Aug 2017 13:24:38 -0600 Subject: cc_landscape & cc_puppet: Fix six.StringIO use in writing configs Both landscape and puppet modules had issues with the way they wrote /etc/landscape/client.conf or /etc/puppet/puppet.conf in either python3 or python2. This branch adds initial unit tests for both modules which will get better exercise under both python2 and python3. The unit tests shed light on a few issues: - In the cc_landscape module py3 can't provide six.StringIO content to ConfigParser.write, so we need to use six.BytesIO instead - In the cc_puppet module, python <= 2.7 doesn't support using six.StringIO as a context manager, so we drop the context manager fanciness and directly set outputstream = StringIO(). - The docstring in cc_puppet is fixed to document the 'conf' sub-key requiring valid puppet section names for each key-value list. LP: #1699282 LP: #1710932 --- cloudinit/config/cc_landscape.py | 4 +- cloudinit/config/cc_puppet.py | 33 ++--- cloudinit/helpers.py | 14 +- .../test_handler/test_handler_landscape.py | 129 +++++++++++++++++++ .../unittests/test_handler/test_handler_puppet.py | 142 +++++++++++++++++++++ 5 files changed, 298 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_landscape.py create mode 100644 tests/unittests/test_handler/test_handler_puppet.py diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 86b71383..8f9f1abd 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -57,7 +57,7 @@ The following default client config is provided, but can be overridden:: import os -from six import StringIO +from six import BytesIO from configobj import ConfigObj @@ -109,7 +109,7 @@ def handle(_name, cfg, cloud, log, _args): ls_cloudcfg, ] merged = merge_together(merge_data) - contents = StringIO() + contents = BytesIO() merged.write(contents) util.ensure_dir(os.path.dirname(LSC_CLIENT_CFG_FILE)) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index dc11561b..28b1d568 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -15,21 +15,23 @@ This module handles puppet installation and configuration. If the ``puppet`` key does not exist in global configuration, no action will be taken. If a config entry for ``puppet`` is present, then by default the latest version of puppet will be installed. If ``install`` is set to ``false``, puppet will not -be installed. However, this may result in an error if puppet is not already +be installed. However, this will result in an error if puppet is not already present on the system. The version of puppet to be installed can be specified under ``version``, and defaults to ``none``, which selects the latest version in the repos. If the ``puppet`` config key exists in the config archive, this module will attempt to start puppet even if no installation was performed. -Puppet configuration can be specified under the ``conf`` key. The configuration -is specified as a dictionary which is converted into ``=`` format -and appended to ``puppet.conf`` under the ``[puppetd]`` section. The +Puppet configuration can be specified under the ``conf`` key. The +configuration is specified as a dictionary containing high-level ``
`` +keys and lists of ``=`` pairs within each section. Each section +name and ``=`` pair is written directly to ``puppet.conf``. As +such, section names should be one of: ``main``, ``master``, ``agent`` or +``user`` and keys should be valid puppet configuration options. The ``certname`` key supports string substitutions for ``%i`` and ``%f``, corresponding to the instance id and fqdn of the machine respectively. -If ``ca_cert`` is present under ``conf``, it will not be written to -``puppet.conf``, but instead will be used as the puppermaster certificate. -It should be specified in pem format as a multi-line string (using the ``|`` -yaml notation). +If ``ca_cert`` is present, it will not be written to ``puppet.conf``, but +instead will be used as the puppermaster certificate. It should be specified +in pem format as a multi-line string (using the ``|`` yaml notation). **Internal name:** ``cc_puppet`` @@ -43,12 +45,13 @@ yaml notation). install: version: conf: - server: "puppetmaster.example.org" - certname: "%i.%f" - ca_cert: | - -------BEGIN CERTIFICATE------- - - -------END CERTIFICATE------- + agent: + server: "puppetmaster.example.org" + certname: "%i.%f" + ca_cert: | + -------BEGIN CERTIFICATE------- + + -------END CERTIFICATE------- """ from six import StringIO @@ -127,7 +130,7 @@ def handle(name, cfg, cloud, log, _args): util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: - # Iterate throug the config items, we'll use ConfigParser.set + # Iterate through the config items, we'll use ConfigParser.set # to overwrite or create new items as needed for (o, v) in cfg.items(): if o == 'certname': diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index f01021aa..1979cd96 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -13,7 +13,7 @@ from time import time import contextlib import os -import six +from six import StringIO from six.moves.configparser import ( NoSectionError, NoOptionError, RawConfigParser) @@ -441,12 +441,12 @@ class DefaultingConfigParser(RawConfigParser): def stringify(self, header=None): contents = '' - with six.StringIO() as outputstream: - self.write(outputstream) - outputstream.flush() - contents = outputstream.getvalue() - if header: - contents = "\n".join([header, contents]) + outputstream = StringIO() + self.write(outputstream) + outputstream.flush() + contents = outputstream.getvalue() + if header: + contents = '\n'.join([header, contents, '']) return contents # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py new file mode 100644 index 00000000..7c247fa9 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -0,0 +1,129 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_landscape +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call + +from configobj import ConfigObj +import logging + + +LOG = logging.getLogger(__name__) + + +class TestLandscape(FilesystemMockingTestCase): + + with_logs = True + + def setUp(self): + super(TestLandscape, self).setUp() + self.new_root = self.tmp_dir() + self.conf = self.tmp_path('client.conf', self.new_root) + self.default_file = self.tmp_path('default_landscape', self.new_root) + + def _get_cloud(self, distro): + self.patchUtils(self.new_root) + paths = helpers.Paths({'templates_dir': self.new_root}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skips_empty_landscape_cloudconfig(self): + """Empty landscape cloud-config section does no work.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'landscape': {}} + cc_landscape.handle('notimportant', cfg, mycloud, LOG, None) + self.assertFalse(mycloud.distro.install_packages.called) + + def test_handler_error_on_invalid_landscape_type(self): + """Raise an error when landscape configuraiton option is invalid.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': 'wrongtype'} + with self.assertRaises(RuntimeError) as context_manager: + cc_landscape.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "'landscape' key existed in config, but not a dict", + str(context_manager.exception)) + + @mock.patch('cloudinit.config.cc_landscape.util') + def test_handler_restarts_landscape_client(self, m_util): + """handler restarts lansdscape-client after install.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(['service', 'landscape-client', 'restart'])], + m_util.subp.call_args_list) + + def test_handler_installs_client_and_creates_config_file(self): + """Write landscape client.conf and install landscape-client.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client'}} + mycloud.distro = mock.MagicMock() + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}, + 'LS_DEFAULT_FILE': {'new': self.default_file}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call('landscape-client')], + mycloud.distro.install_packages.call_args) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) + default_content = util.load_file(self.default_file) + self.assertEqual('RUN=1\n', default_content) + + def test_handler_writes_merged_client_config_file_with_defaults(self): + """Merge and write options from LSC_CLIENT_CFG_FILE with defaults.""" + # Write existing sparse client.conf file + util.write_file(self.conf, '[client]\ncomputer_title = My PC\n') + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client', + 'computer_title': 'My PC'}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) + + def test_handler_writes_merged_provided_cloudconfig_with_defaults(self): + """Merge and write options from cloud-config options with defaults.""" + # Write empty sparse client.conf file + util.write_file(self.conf, '') + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {'computer_title': 'My PC'}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client', + 'computer_title': 'My PC'}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py new file mode 100644 index 00000000..805c76ba --- /dev/null +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -0,0 +1,142 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_puppet +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from ..helpers import CiTestCase, mock + +import logging + + +LOG = logging.getLogger(__name__) + + +@mock.patch('cloudinit.config.cc_puppet.util') +@mock.patch('cloudinit.config.cc_puppet.os') +class TestAutostartPuppet(CiTestCase): + + with_logs = True + + def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): + """Update /etc/default/puppet to autostart if it exists.""" + + def _fake_exists(path): + return path == '/etc/default/puppet' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + self.assertEqual( + [mock.call(['sed', '-i', '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False)], + m_util.subp.call_args_list) + + def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_util): + """If systemctl is present, enable puppet via systemctl.""" + + def _fake_exists(path): + return path == '/bin/systemctl' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + expected_calls = [mock.call( + ['/bin/systemctl', 'enable', 'puppet.service'], capture=False)] + self.assertEqual(expected_calls, m_util.subp.call_args_list) + + def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_util): + """If chkconfig is present, enable puppet via checkcfg.""" + + def _fake_exists(path): + return path == '/sbin/chkconfig' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + expected_calls = [mock.call( + ['/sbin/chkconfig', 'puppet', 'on'], capture=False)] + self.assertEqual(expected_calls, m_util.subp.call_args_list) + + +@mock.patch('cloudinit.config.cc_puppet._autostart_puppet') +class TestPuppetHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestPuppetHandle, self).setUp() + self.new_root = self.tmp_dir() + self.conf = self.tmp_path('puppet.conf') + + def _get_cloud(self, distro): + paths = helpers.Paths({'templates_dir': self.new_root}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skips_missing_puppet_key_in_cloudconfig(self, m_auto): + """Cloud-config containing no 'puppet' key is skipped.""" + mycloud = self._get_cloud('ubuntu') + cfg = {} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "no 'puppet' configuration found", self.logs.getvalue()) + self.assertEqual(0, m_auto.call_count) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_starts_puppet_service(self, m_subp, m_auto): + """Cloud-config 'puppet' configuration starts puppet.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'puppet': {'install': False}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual(1, m_auto.call_count) + self.assertEqual( + [mock.call(['service', 'puppet', 'start'], capture=False)], + m_subp.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_empty_puppet_config_installs_puppet(self, m_subp, m_auto): + """Cloud-config empty 'puppet' configuration installs latest puppet.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', None))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_installs_puppet_on_true(self, m_subp, _): + """Cloud-config with 'puppet' key installs when 'install' is True.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {'install': True}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', None))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_installs_puppet_version(self, m_subp, _): + """Cloud-config 'puppet' configuration can specify a version.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {'version': '3.8'}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', '3.8'))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_updates_puppet_conf(self, m_subp, m_auto): + """When 'conf' is provided update values in PUPPET_CONF_PATH.""" + mycloud = self._get_cloud('ubuntu') + cfg = { + 'puppet': { + 'conf': {'agent': {'server': 'puppetmaster.example.org'}}}} + util.write_file(self.conf, '[agent]\nserver = origpuppet\nother = 3') + puppet_conf_path = 'cloudinit.config.cc_puppet.PUPPET_CONF_PATH' + mycloud.distro = mock.MagicMock() + with mock.patch(puppet_conf_path, self.conf): + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + content = util.load_file(self.conf) + expected = '[agent]\nserver = puppetmaster.example.org\nother = 3\n\n' + self.assertEqual(expected, content) -- cgit v1.2.3 From 44773a480d8fe32e97da44afd01e5882a480d136 Mon Sep 17 00:00:00 2001 From: Jason Butz Date: Fri, 25 Aug 2017 07:16:21 -0400 Subject: doc: Explain error behavior in user data include file format. Update user data 'include file' format documentation to explain the behavior that occurs when an error occurs while reading a file. --- doc/rtd/topics/format.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/rtd/topics/format.rst b/doc/rtd/topics/format.rst index 436eb00f..e25289ad 100644 --- a/doc/rtd/topics/format.rst +++ b/doc/rtd/topics/format.rst @@ -85,6 +85,7 @@ This content is a ``include`` file. The file contains a list of urls, one per line. Each of the URLs will be read, and their content will be passed through this same set of rules. Ie, the content read from the URL can be gzipped, mime-multi-part, or plain text. +If an error occurs reading a file the remaining files will not be read. Begins with: ``#include`` or ``Content-Type: text/x-include-url`` when using a MIME archive. -- cgit v1.2.3 From 89579a68d9f51e51b24f96b933da656afd83edfb Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 23 Aug 2017 18:54:01 -0600 Subject: cli: Fix command line parsing of coniditionally loaded subcommands. In an effort to save file load cost during system boot, certain subcommands, analyze and devel, do not get loaded unless the subcommand is specified on the commandline. Because setup.py entrypoint for cloud-init script doesn't specify sysv_args parameter when calling the CLI's main() we need main to read sys.argv into sysv_args so our subparser loading continues to work. LP: #1712676 --- cloudinit/cmd/main.py | 9 ++++----- tests/unittests/test_cli.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 5b467979..68563e0c 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -676,11 +676,10 @@ def main_features(name, args): def main(sysv_args=None): - if sysv_args is not None: - parser = argparse.ArgumentParser(prog=sysv_args[0]) - sysv_args = sysv_args[1:] - else: - parser = argparse.ArgumentParser() + if not sysv_args: + sysv_args = sys.argv + parser = argparse.ArgumentParser(prog=sysv_args[0]) + sysv_args = sysv_args[1:] # Top level args parser.add_argument('--version', '-v', action='version', diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 24498802..12f01852 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -70,6 +70,21 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self.assertEqual('modules', parseargs.action[0]) self.assertEqual('main_modules', parseargs.action[1].__name__) + def test_conditional_subcommands_from_entry_point_sys_argv(self): + """Subcommands from entry-point are properly parsed from sys.argv.""" + expected_errors = [ + 'usage: cloud-init analyze', 'usage: cloud-init devel'] + conditional_subcommands = ['analyze', 'devel'] + # The cloud-init entrypoint calls main without passing sys_argv + for subcommand in conditional_subcommands: + with mock.patch('sys.argv', ['cloud-init', subcommand]): + try: + cli.main() + except SystemExit as e: + self.assertEqual(2, e.code) # exit 2 on proper usage docs + for error_message in expected_errors: + self.assertIn(error_message, self.stderr.getvalue()) + def test_analyze_subcommand_parser(self): """The subcommand cloud-init analyze calls the correct subparser.""" self._call_main(['cloud-init', 'analyze']) -- cgit v1.2.3 From 20ca23cab0bdfdffa567f8fb4b49f3727bac6444 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Mon, 14 Aug 2017 17:08:37 +0000 Subject: Log a helpful message if a user script does not include shebang. A patch to allow scripts missing a #! to run by using shell=True was proposed but rejected. Instead we emit a log message to help the user understand what went wrong. --- cloudinit/util.py | 20 ++++++++++++-------- tests/unittests/test_util.py | 13 ++++++++++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index ce2c6034..609e94c8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -12,7 +12,6 @@ import contextlib import copy as obj_copy import ctypes import email -import errno import glob import grp import gzip @@ -34,6 +33,8 @@ import sys import tempfile import time +from errno import ENOENT, ENOEXEC + from base64 import b64decode, b64encode from six.moves.urllib import parse as urlparse @@ -239,7 +240,10 @@ class ProcessExecutionError(IOError): self.cmd = cmd if not description: - self.description = 'Unexpected error while running command.' + if not exit_code and errno == ENOEXEC: + self.description = 'Exec format error. Missing #! in script?' + else: + self.description = 'Unexpected error while running command.' else: self.description = description @@ -433,7 +437,7 @@ def read_conf(fname): try: return load_yaml(load_file(fname), default={}) except IOError as e: - if e.errno == errno.ENOENT: + if e.errno == ENOENT: return {} else: raise @@ -901,7 +905,7 @@ def read_file_or_url(url, timeout=5, retries=10, contents = load_file(file_path, decode=False) except IOError as e: code = e.errno - if e.errno == errno.ENOENT: + if e.errno == ENOENT: code = url_helper.NOT_FOUND raise url_helper.UrlError(cause=e, code=code, headers=None, url=url) @@ -1247,7 +1251,7 @@ def find_devs_with(criteria=None, oformat='device', try: (out, _err) = subp(cmd, rcs=[0, 2]) except ProcessExecutionError as e: - if e.errno == errno.ENOENT: + if e.errno == ENOENT: # blkid not found... out = "" else: @@ -1285,7 +1289,7 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): except IOError as e: if not quiet: raise - if e.errno != errno.ENOENT: + if e.errno != ENOENT: raise contents = ofh.getvalue() LOG.debug("Read %s bytes from %s", len(contents), fname) @@ -1653,7 +1657,7 @@ def del_file(path): try: os.unlink(path) except OSError as e: - if e.errno != errno.ENOENT: + if e.errno != ENOENT: raise e @@ -2281,7 +2285,7 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): try: ret[f] = load_file(base + delim + f, quiet=False, decode=False) except IOError as e: - if e.errno != errno.ENOENT: + if e.errno != ENOENT: raise if f in required: missing.append(f) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index f38a664c..5f11c88f 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -568,7 +568,8 @@ class TestReadSeeded(helpers.TestCase): self.assertEqual(found_ud, ud) -class TestSubp(helpers.TestCase): +class TestSubp(helpers.CiTestCase): + with_logs = True stdin2err = [BASH, '-c', 'cat >&2'] stdin2out = ['cat'] @@ -650,6 +651,16 @@ class TestSubp(helpers.TestCase): self.assertEqual( ['FOO=BAR', 'HOME=/myhome', 'K1=V1', 'K2=V2'], out.splitlines()) + def test_subp_warn_missing_shebang(self): + """Warn on no #! in script""" + noshebang = self.tmp_path('noshebang') + util.write_file(noshebang, 'true\n') + + os.chmod(noshebang, os.stat(noshebang).st_mode | stat.S_IEXEC) + self.assertRaisesRegexp(util.ProcessExecutionError, + 'Missing #! in script\?', + util.subp, (noshebang,)) + def test_returns_none_if_no_capture(self): (out, err) = util.subp(self.stdin2out, data=b'', capture=False) self.assertIsNone(err) -- cgit v1.2.3 From 556a0220734097aa4e9fbfd93c8f263684232b3b Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Mon, 7 Aug 2017 13:38:56 -0500 Subject: Configure logging module to always use UTC time. Currently the python logging module will default to a local time which may contain an TZ offset in the values it produces, but the logged time format does not contain the offset. Switching to UTC time for logging produces consistent values in the cloud-init.log file and avoids issues when the timezone is changed during boot. LP: #1713158 --- cloudinit/log.py | 5 ++++ tests/unittests/test_log.py | 58 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/unittests/test_log.py diff --git a/cloudinit/log.py b/cloudinit/log.py index 3861709e..1d75c9ff 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -19,6 +19,8 @@ import sys import six from six import StringIO +import time + # Logging levels for easy access CRITICAL = logging.CRITICAL FATAL = logging.FATAL @@ -32,6 +34,9 @@ NOTSET = logging.NOTSET # Default basic format DEF_CON_FORMAT = '%(asctime)s - %(filename)s[%(levelname)s]: %(message)s' +# Always format logging timestamps as UTC time +logging.Formatter.converter = time.gmtime + def setupBasicLogging(level=DEBUG): root = logging.getLogger() diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py new file mode 100644 index 00000000..68fb4b8d --- /dev/null +++ b/tests/unittests/test_log.py @@ -0,0 +1,58 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Tests for cloudinit.log """ + +from .helpers import CiTestCase +from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT +from cloudinit import log as ci_logging +import datetime +import logging +import six +import time + + +class TestCloudInitLogger(CiTestCase): + + def setUp(self): + # set up a logger like cloud-init does in setupLogging, but instead + # of sys.stderr, we'll plug in a StringIO() object so we can see + # what gets logged + logging.Formatter.converter = time.gmtime + self.ci_logs = six.StringIO() + self.ci_root = logging.getLogger() + console = logging.StreamHandler(self.ci_logs) + console.setFormatter(logging.Formatter(ci_logging.DEF_CON_FORMAT)) + console.setLevel(ci_logging.DEBUG) + self.ci_root.addHandler(console) + self.ci_root.setLevel(ci_logging.DEBUG) + self.LOG = logging.getLogger('test_cloudinit_logger') + + def test_logger_uses_gmtime(self): + """Test that log message have timestamp in UTC (gmtime)""" + + # Log a message, extract the timestamp from the log entry + # convert to datetime, and compare to a utc timestamp before + # and after the logged message. + + # Due to loss of precision in the LOG timestamp, subtract and add + # time to the utc stamps for comparison + # + # utc_before: 2017-08-23 14:19:42.569299 + # parsed dt : 2017-08-23 14:19:43.069000 + # utc_after : 2017-08-23 14:19:43.570064 + + utc_before = datetime.datetime.utcnow() - datetime.timedelta(0, 0.5) + self.LOG.error('Test message') + utc_after = datetime.datetime.utcnow() + datetime.timedelta(0, 0.5) + + # extract timestamp from log: + # 2017-08-23 14:19:43,069 - test_log.py[ERROR]: Test message + logstr = self.ci_logs.getvalue().splitlines()[0] + timestampstr = logstr.split(' - ')[0] + parsed_dt = datetime.datetime.strptime(timestampstr, + CLOUD_INIT_ASCTIME_FMT) + + self.assertLess(utc_before, parsed_dt) + self.assertLess(parsed_dt, utc_after) + self.assertLess(utc_before, utc_after) + self.assertGreater(utc_after, parsed_dt) -- cgit v1.2.3 From 0ab9859168eb0ba4fc348843e866751cfc67181f Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Fri, 25 Aug 2017 22:17:34 +0000 Subject: cloud-init analyze: fix issues running under python 2. Some Python 3 exception names crept into the cloud-init analyze code. This patches those back out at a cost of catching less specific parents of the desired exceptions. --- cloudinit/analyze/__main__.py | 4 ++-- cloudinit/analyze/show.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/analyze/__main__.py b/cloudinit/analyze/__main__.py index 71cba4f2..69b9e43e 100644 --- a/cloudinit/analyze/__main__.py +++ b/cloudinit/analyze/__main__.py @@ -130,7 +130,7 @@ def configure_io(args): else: try: infh = open(args.infile, 'r') - except (FileNotFoundError, PermissionError): + except OSError: sys.stderr.write('Cannot open file %s\n' % args.infile) sys.exit(1) @@ -139,7 +139,7 @@ def configure_io(args): else: try: outfh = open(args.outfile, 'w') - except PermissionError: + except OSError: sys.stderr.write('Cannot open file %s\n' % args.outfile) sys.exit(1) diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index 3b356bb8..3e778b8b 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -201,7 +201,7 @@ def load_events(infile, rawdata=None): j = None try: j = json.loads(data) - except json.JSONDecodeError: + except ValueError: pass return j, data -- cgit v1.2.3 From af4630c9846fe979152320035e9cc6c411506503 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 29 Aug 2017 09:55:48 -0600 Subject: url_helper: fail gracefully if oauthlib is not available We are unable to ship python-oauthlib in RHEL. This commit allows imports of url_helper to succeed even when oauthlib is unavailable and OauthUrlHelper.oauth_headers to raise a NotImplementedException when called. LP: #1713760 --- cloudinit/url_helper.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 7cf76aae..c83061a9 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -17,7 +17,11 @@ import time from email.utils import parsedate from functools import partial -import oauthlib.oauth1 as oauth1 +try: + import oauthlib.oauth1 as oauth1 +except ImportError: + oauth1 = None + from requests import exceptions from six.moves.urllib.parse import ( @@ -488,6 +492,10 @@ class OauthUrlHelper(object): def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): + + if oauth1 is None: + raise NotImplementedError('oauth support is not available') + if timestamp: timestamp = str(timestamp) else: -- cgit v1.2.3 From 3c45330af2a301f2bf219da556844d01cef6778e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 29 Aug 2017 10:34:19 -0600 Subject: ec2: Add IPv6 dhcp support to Ec2DataSource. DataSourceEc2 now parses the metadata for each nic to determine if configured for ipv6 and/or ipv4 addresses. In AWS for metadata version 2016-09-02, nics configured for ipv4 or ipv6 addresses will have non-zero values stored in metadata at network/interfaces/macs//public-ipv4 or ipv6s respectively. Those metadata files are only non-zero when an ipv4 or ipv6 ip is associated to the specific nic. A new DataSourceEc2.network_config property is added which parses the metadata and renders a network version 1 dictionary representing both dhcp4 and dhcp6 configuration for associated nics. The network configuration returned from the datasource will also 'pin' the nic name to the name presented on the instance for each nic. LP: #1639030 --- cloudinit/sources/DataSourceEc2.py | 38 +++++++++++ tests/unittests/test_datasource/test_ec2.py | 97 +++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 8e5f8ee4..07c12bb4 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -57,6 +57,8 @@ class DataSourceEc2(sources.DataSource): _cloud_platform = None + _network_config = None # Used for caching calculated network config v1 + # Whether we want to get network configuration from the metadata service. get_network_metadata = False @@ -279,6 +281,15 @@ class DataSourceEc2(sources.DataSource): util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT), cfg) + @property + def network_config(self): + """Return a network config dict for rendering ENI or netplan files.""" + if self._network_config is None: + if self.metadata is not None: + self._network_config = convert_ec2_metadata_network_config( + self.metadata) + return self._network_config + def _crawl_metadata(self): """Crawl metadata service when available. @@ -423,6 +434,33 @@ def _collect_platform_data(): return data +def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None): + """Convert ec2 metadata to network config version 1 data dict. + + @param: metadata: Dictionary of metadata crawled from EC2 metadata url. + @param: macs_to_name: Optional dict mac addresses and the nic name. If + not provided, get_interfaces_by_mac is called to get it from the OS. + + @return A dict of network config version 1 based on the metadata and macs. + """ + netcfg = {'version': 1, 'config': []} + if not macs_to_nics: + macs_to_nics = net.get_interfaces_by_mac() + macs_metadata = metadata['network']['interfaces']['macs'] + for mac, nic_name in macs_to_nics.items(): + nic_metadata = macs_metadata.get(mac) + if not nic_metadata: + continue # Not a physical nic represented in metadata + nic_cfg = {'type': 'physical', 'name': nic_name, 'subnets': []} + nic_cfg['mac_address'] = mac + if nic_metadata.get('public-ipv4s'): + nic_cfg['subnets'].append({'type': 'dhcp4'}) + if nic_metadata.get('ipv6s'): + nic_cfg['subnets'].append({'type': 'dhcp6'}) + netcfg['config'].append(nic_cfg) + return netcfg + + # Used to match classes to dependencies datasources = [ (DataSourceEc2Local, (sources.DEP_FILESYSTEM,)), # Run at init-local diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 33d02619..e1ce6446 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -1,5 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. +import copy import httpretty import mock @@ -194,6 +195,34 @@ class TestEc2(test_helpers.HttprettyTestCase): return ds + @httpretty.activate + def test_network_config_property_returns_version_1_network_data(self): + """network_config property returns network version 1 for metadata.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=DEFAULT_METADATA) + ds.get_data() + mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA + expected = {'version': 1, 'config': [ + {'mac_address': '06:17:04:d7:26:09', 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}], + 'type': 'physical'}]} + patch_path = ( + 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') + with mock.patch(patch_path) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.return_value = {mac1: 'eth9'} + self.assertEqual(expected, ds.network_config) + + def test_network_config_property_is_cached_in_datasource(self): + """network_config property is cached in DataSourceEc2.""" + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=DEFAULT_METADATA) + ds._network_config = {'cached': 'data'} + self.assertEqual({'cached': 'data'}, ds.network_config) + @httpretty.activate @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_valid_platform_with_strict_true(self, m_dhcp): @@ -287,4 +316,72 @@ class TestEc2(test_helpers.HttprettyTestCase): self.assertIn('Crawl of metadata service took', self.logs.getvalue()) +class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): + + def setUp(self): + super(TestConvertEc2MetadataNetworkConfig, self).setUp() + self.mac1 = '06:17:04:d7:26:09' + self.network_metadata = { + 'network': {'interfaces': {'macs': { + self.mac1: {'public-ipv4s': '172.31.2.16'}}}}} + + def test_convert_ec2_metadata_network_config_skips_absent_macs(self): + """Any mac absent from metadata is skipped by network config.""" + macs_to_nics = {self.mac1: 'eth9', 'DE:AD:BE:EF:FF:FF': 'vitualnic2'} + + # DE:AD:BE:EF:FF:FF represented by OS but not in metadata + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', 'subnets': [{'type': 'dhcp4'}]}]} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + self.network_metadata, macs_to_nics)) + + def test_convert_ec2_metadata_network_config_handles_only_dhcp6(self): + """Config dhcp6 when ipv6s is in metadata for a mac.""" + macs_to_nics = {self.mac1: 'eth9'} + network_metadata_ipv6 = copy.deepcopy(self.network_metadata) + nic1_metadata = ( + network_metadata_ipv6['network']['interfaces']['macs'][self.mac1]) + nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' + nic1_metadata.pop('public-ipv4s') + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', 'subnets': [{'type': 'dhcp6'}]}]} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + network_metadata_ipv6, macs_to_nics)) + + def test_convert_ec2_metadata_network_config_handles_dhcp4_and_dhcp6(self): + """Config both dhcp4 and dhcp6 when both vpc-ipv6 and ipv4 exists.""" + macs_to_nics = {self.mac1: 'eth9'} + network_metadata_both = copy.deepcopy(self.network_metadata) + nic1_metadata = ( + network_metadata_both['network']['interfaces']['macs'][self.mac1]) + nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}]}]} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config( + network_metadata_both, macs_to_nics)) + + def test_convert_ec2_metadata_gets_macs_from_get_interfaces_by_mac(self): + """Convert Ec2 Metadata calls get_interfaces_by_mac by default.""" + expected = {'version': 1, 'config': [ + {'mac_address': self.mac1, 'type': 'physical', + 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}]}]} + patch_path = ( + 'cloudinit.sources.DataSourceEc2.net.get_interfaces_by_mac') + with mock.patch(patch_path) as m_get_interfaces_by_mac: + m_get_interfaces_by_mac.return_value = {self.mac1: 'eth9'} + self.assertEqual( + expected, + ec2.convert_ec2_metadata_network_config(self.network_metadata)) + # vi: ts=4 expandtab -- cgit v1.2.3 From 44529c1de0098ccd684b46b0bc18d48312c4097c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 22 Aug 2017 15:31:48 -0400 Subject: GCE: Add a main to the GCE Datasource. This just adds a main to the GCE datasource so that it is easily callable: python3 -m cloudinit.sources.DataSourceGCE It also adds a log of the time it took to crawl. --- cloudinit/sources/DataSourceGCE.py | 182 +++++++++++++++++++++++-------------- 1 file changed, 112 insertions(+), 70 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 684eac86..94484d60 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -11,9 +11,8 @@ from cloudinit import util LOG = logging.getLogger(__name__) -BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' -} +MD_V1_URL = 'http://metadata.google.internal/computeMetadata/v1/' +BUILTIN_DS_CONFIG = {'metadata_url': MD_V1_URL} REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -51,75 +50,20 @@ class DataSourceGCE(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata_address = self.ds_cfg['metadata_url'] - # GCE takes sshKeys attribute in the format of ':' - # so we have to trim each key to remove the username part - def _trim_key(self, public_key): - try: - index = public_key.index(':') - if index > 0: - return public_key[(index + 1):] - except Exception: - return public_key - def get_data(self): - if not platform_reports_gce(): - return False + ret = util.log_time( + LOG.debug, 'Crawl of GCE metadata service', + read_md, kwargs={'address': self.metadata_address}) - # url_map: (our-key, path, required, is_text) - url_map = [ - ('instance-id', ('instance/id',), True, True), - ('availability-zone', ('instance/zone',), True, True), - ('local-hostname', ('instance/hostname',), True, True), - ('public-keys', ('project/attributes/sshKeys', - 'instance/attributes/ssh-keys'), False, True), - ('user-data', ('instance/attributes/user-data',), False, False), - ('user-data-encoding', ('instance/attributes/user-data-encoding',), - False, True), - ] - - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable_url(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) - return False - - metadata_fetcher = GoogleMetadataFetcher(self.metadata_address) - # iterate over url_map keys to get metadata items - running_on_gce = False - for (mkey, paths, required, is_text) in url_map: - value = None - for path in paths: - new_value = metadata_fetcher.get_value(path, is_text) - if new_value is not None: - value = new_value - if value: - running_on_gce = True - if required and value is None: - msg = "required key %s returned nothing. not GCE" - if not running_on_gce: - LOG.debug(msg, mkey) - else: - LOG.warning(msg, mkey) - return False - self.metadata[mkey] = value - - if self.metadata['public-keys']: - lines = self.metadata['public-keys'].splitlines() - self.metadata['public-keys'] = [self._trim_key(k) for k in lines] - - if self.metadata['availability-zone']: - self.metadata['availability-zone'] = self.metadata[ - 'availability-zone'].split('/')[-1] - - encoding = self.metadata.get('user-data-encoding') - if encoding: - if encoding == 'base64': - self.metadata['user-data'] = b64decode( - self.metadata['user-data']) + if not ret['success']: + if ret['platform_reports_gce']: + LOG.warning(ret['reason']) else: - LOG.warning('unknown user-data-encoding: %s, ignoring', - encoding) - - return running_on_gce + LOG.debug(ret['reason']) + return False + self.metadata = ret['meta-data'] + self.userdata = ret['user-data'] + return True @property def launch_index(self): @@ -137,7 +81,7 @@ class DataSourceGCE(sources.DataSource): return self.metadata['local-hostname'].split('.')[0] def get_userdata_raw(self): - return self.metadata['user-data'] + return self.userdata @property def availability_zone(self): @@ -148,6 +92,87 @@ class DataSourceGCE(sources.DataSource): return self.availability_zone.rsplit('-', 1)[0] +def _trim_key(public_key): + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except Exception: + return public_key + + +def read_md(address=None, platform_check=True): + + if address is None: + address = MD_V1_URL + + ret = {'meta-data': None, 'user-data': None, + 'success': False, 'reason': None} + ret['platform_reports_gce'] = platform_reports_gce() + + if platform_check and not ret['platform_reports_gce']: + ret['reason'] = "Not running on GCE." + return ret + + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable_url(address): + LOG.debug("%s is not resolvable", address) + ret['reason'] = 'address "%s" is not resolvable' % address + return ret + + # url_map: (our-key, path, required, is_text) + url_map = [ + ('instance-id', ('instance/id',), True, True), + ('availability-zone', ('instance/zone',), True, True), + ('local-hostname', ('instance/hostname',), True, True), + ('public-keys', ('project/attributes/sshKeys', + 'instance/attributes/ssh-keys'), False, True), + ('user-data', ('instance/attributes/user-data',), False, False), + ('user-data-encoding', ('instance/attributes/user-data-encoding',), + False, True), + ] + + metadata_fetcher = GoogleMetadataFetcher(address) + md = {} + # iterate over url_map keys to get metadata items + for (mkey, paths, required, is_text) in url_map: + value = None + for path in paths: + new_value = metadata_fetcher.get_value(path, is_text) + if new_value is not None: + value = new_value + if required and value is None: + msg = "required key %s returned nothing. not GCE" + ret['reason'] = msg % mkey + return ret + md[mkey] = value + + if md['public-keys']: + lines = md['public-keys'].splitlines() + md['public-keys'] = [_trim_key(k) for k in lines] + + if md['availability-zone']: + md['availability-zone'] = md['availability-zone'].split('/')[-1] + + encoding = md.get('user-data-encoding') + if encoding: + if encoding == 'base64': + md['user-data'] = b64decode(md['user-data']) + else: + LOG.warning('unknown user-data-encoding: %s, ignoring', encoding) + + if 'user-data' in md: + ret['user-data'] = md['user-data'] + del md['user-data'] + + ret['meta-data'] = md + ret['success'] = True + + return ret + + def platform_reports_gce(): pname = util.read_dmi_data('system-product-name') or "N/A" if pname == "Google Compute Engine": @@ -173,4 +198,21 @@ datasources = [ def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + +if __name__ == "__main__": + import argparse + import json + + parser = argparse.ArgumentParser(description='Query GCE Metadata Service') + parser.add_argument("--endpoint", metavar="URL", + help="The url of the metadata service.", + default=MD_V1_URL) + parser.add_argument("--no-platform-check", dest="platform_check", + help="Ignore smbios platform check", + action='store_false', default=True) + args = parser.parse_args() + print(json.dumps( + read_md(address=args.endpoint, platform_check=args.platform_check), + indent=1, sort_keys=True, separators=(',', ': '))) + # vi: ts=4 expandtab -- cgit v1.2.3 From cbda576a7bbf846710ad55940bf8ca1f2d2194b9 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Fri, 25 Aug 2017 11:13:58 -0400 Subject: suse: Add support for openSUSE and return SLES to a working state. This gets initial opensuse and SLES support back to a working state. Still missing is more complete network file writing and unit tests. --- cloudinit/config/cc_resolv_conf.py | 2 +- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/opensuse.py | 212 +++++++++++++++++++++ cloudinit/distros/sles.py | 160 +--------------- templates/hosts.opensuse.tmpl | 26 +++ templates/hosts.suse.tmpl | 3 - tests/unittests/test_distros/test_opensuse.py | 12 ++ tests/unittests/test_distros/test_sles.py | 12 ++ .../unittests/test_handler/test_handler_locale.py | 12 +- .../test_handler/test_handler_set_hostname.py | 5 +- tox.ini | 16 ++ 11 files changed, 297 insertions(+), 165 deletions(-) create mode 100644 cloudinit/distros/opensuse.py create mode 100644 templates/hosts.opensuse.tmpl create mode 100644 tests/unittests/test_distros/test_opensuse.py create mode 100644 tests/unittests/test_distros/test_sles.py diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 2548d1f1..9812562a 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -55,7 +55,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -distros = ['fedora', 'rhel', 'sles'] +distros = ['fedora', 'opensuse', 'rhel', 'sles'] def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 1fd48a7b..807b3ea2 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -35,7 +35,7 @@ OSFAMILIES = { 'redhat': ['centos', 'fedora', 'rhel'], 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], - 'suse': ['sles'], + 'suse': ['opensuse', 'sles'], 'arch': ['arch'], } diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py new file mode 100644 index 00000000..a219e9fb --- /dev/null +++ b/cloudinit/distros/opensuse.py @@ -0,0 +1,212 @@ +# Copyright (C) 2017 SUSE LLC +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Author: Robert Schweikert +# Author: Juerg Haefliger +# +# Leaning very heavily on the RHEL and Debian implementation +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import distros + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.distros import net_util +from cloudinit.distros import rhel_util as rhutil +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + clock_conf_fn = '/etc/sysconfig/clock' + hostname_conf_fn = '/etc/HOSTNAME' + init_cmd = ['service'] + locale_conf_fn = '/etc/sysconfig/language' + network_conf_fn = '/etc/sysconfig/network' + network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' + resolve_conf_fn = '/etc/resolv.conf' + route_conf_tpl = '/etc/sysconfig/network/ifroute-%s' + systemd_hostname_conf_fn = '/etc/hostname' + systemd_locale_conf_fn = '/etc/locale.conf' + tz_local_fn = '/etc/localtime' + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + self._runner = helpers.Runners(paths) + self.osfamily = 'suse' + cfg['ssh_svcname'] = 'sshd' + if self.uses_systemd(): + self.init_cmd = ['systemctl'] + cfg['ssh_svcname'] = 'sshd.service' + + def apply_locale(self, locale, out_fn=None): + if self.uses_systemd(): + if not out_fn: + out_fn = self.systemd_locale_conf_fn + locale_cfg = {'LANG': locale} + else: + if not out_fn: + out_fn = self.locale_conf_fn + locale_cfg = {'RC_LANG': locale} + rhutil.update_sysconfig_file(out_fn, locale_cfg) + + def install_packages(self, pkglist): + self.package_command( + 'install', + args='--auto-agree-with-licenses', + pkgs=pkglist + ) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['zypper'] + # No user interaction possible, enable non-interactive mode + cmd.append('--non-interactive') + + # Comand is the operation, such as install + if command == 'upgrade': + command = 'update' + cmd.append(command) + + # args are the arguments to the command, not global options + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, capture=False) + + def set_timezone(self, tz): + tz_file = self._find_tz_file(tz) + if self.uses_systemd(): + # Currently, timedatectl complains if invoked during startup + # so for compatibility, create the link manually. + util.del_file(self.tz_local_fn) + util.sym_link(tz_file, self.tz_local_fn) + else: + # Adjust the sysconfig clock zone setting + clock_cfg = { + 'TIMEZONE': str(tz), + } + rhutil.update_sysconfig_file(self.clock_conf_fn, clock_cfg) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ['refresh'], freq=PER_INSTANCE) + + def _bring_up_interfaces(self, device_names): + if device_names and 'all' in device_names: + raise RuntimeError(('Distro %s can not translate ' + 'the device name "all"') % (self.name)) + return distros.Distro._bring_up_interfaces(self, device_names) + + def _read_hostname(self, filename, default=None): + if self.uses_systemd() and filename.endswith('/previous-hostname'): + return util.load_file(filename).strip() + elif self.uses_systemd(): + (out, _err) = util.subp(['hostname']) + if len(out): + return out + else: + return default + else: + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_system_hostname(self): + if self.uses_systemd(): + host_fn = self.systemd_hostname_conf_fn + else: + host_fn = self.hostname_conf_fn + return (host_fn, self._read_hostname(host_fn)) + + def _write_hostname(self, hostname, out_fn): + if self.uses_systemd() and out_fn.endswith('/previous-hostname'): + util.write_file(out_fn, hostname) + elif self.uses_systemd(): + util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + else: + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(hostname) + util.write_file(out_fn, str(conf), 0o644) + + def _write_network(self, settings): + # Convert debian settings to ifcfg format + entries = net_util.translate_network(settings) + LOG.debug("Translated ubuntu style network settings %s into %s", + settings, entries) + # Make the intermediate format as the suse format... + nameservers = [] + searchservers = [] + dev_names = entries.keys() + for (dev, info) in entries.items(): + net_fn = self.network_script_tpl % (dev) + route_fn = self.route_conf_tpl % (dev) + mode = None + if info.get('auto', None): + mode = 'auto' + else: + mode = 'manual' + bootproto = info.get('bootproto', None) + gateway = info.get('gateway', None) + net_cfg = { + 'BOOTPROTO': bootproto, + 'BROADCAST': info.get('broadcast'), + 'GATEWAY': gateway, + 'IPADDR': info.get('address'), + 'LLADDR': info.get('hwaddress'), + 'NETMASK': info.get('netmask'), + 'STARTMODE': mode, + 'USERCONTROL': 'no' + } + if dev != 'lo': + net_cfg['ETHTOOL_OPTIONS'] = '' + else: + net_cfg['FIREWALL'] = 'no' + rhutil.update_sysconfig_file(net_fn, net_cfg, True) + if gateway and bootproto == 'static': + default_route = 'default %s' % gateway + util.write_file(route_fn, default_route, 0o644) + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + if 'dns-search' in info: + searchservers.extend(info['dns-search']) + if nameservers or searchservers: + rhutil.update_resolve_conf_file(self.resolve_conf_fn, + nameservers, searchservers) + return dev_names + +# vi: ts=4 expandtab diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index dbec2edf..6e336cbf 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -1,167 +1,17 @@ -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2017 SUSE LLC # -# Author: Juerg Haefliger +# Author: Robert Schweikert # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import distros +from cloudinit.distros import opensuse -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros import rhel_util -from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -class Distro(distros.Distro): - clock_conf_fn = '/etc/sysconfig/clock' - locale_conf_fn = '/etc/sysconfig/language' - network_conf_fn = '/etc/sysconfig/network' - hostname_conf_fn = '/etc/HOSTNAME' - network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' - resolve_conf_fn = '/etc/resolv.conf' - tz_local_fn = '/etc/localtime' - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'suse' - - def install_packages(self, pkglist): - self.package_command('install', args='-l', pkgs=pkglist) - - def _write_network(self, settings): - # Convert debian settings to ifcfg format - entries = net_util.translate_network(settings) - LOG.debug("Translated ubuntu style network settings %s into %s", - settings, entries) - # Make the intermediate format as the suse format... - nameservers = [] - searchservers = [] - dev_names = entries.keys() - for (dev, info) in entries.items(): - net_fn = self.network_script_tpl % (dev) - mode = info.get('auto') - if mode and mode.lower() == 'true': - mode = 'auto' - else: - mode = 'manual' - net_cfg = { - 'BOOTPROTO': info.get('bootproto'), - 'BROADCAST': info.get('broadcast'), - 'GATEWAY': info.get('gateway'), - 'IPADDR': info.get('address'), - 'LLADDR': info.get('hwaddress'), - 'NETMASK': info.get('netmask'), - 'STARTMODE': mode, - 'USERCONTROL': 'no' - } - if dev != 'lo': - net_cfg['ETHERDEVICE'] = dev - net_cfg['ETHTOOL_OPTIONS'] = '' - else: - net_cfg['FIREWALL'] = 'no' - rhel_util.update_sysconfig_file(net_fn, net_cfg, True) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - if 'dns-search' in info: - searchservers.extend(info['dns-search']) - if nameservers or searchservers: - rhel_util.update_resolve_conf_file(self.resolve_conf_fn, - nameservers, searchservers) - return dev_names - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - locale_cfg = { - 'RC_LANG': locale, - } - rhel_util.update_sysconfig_file(out_fn, locale_cfg) - - def _write_hostname(self, hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0o644) - - def _read_system_hostname(self): - host_fn = self.hostname_conf_fn - return (host_fn, self._read_hostname(host_fn)) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def _bring_up_interfaces(self, device_names): - if device_names and 'all' in device_names: - raise RuntimeError(('Distro %s can not translate ' - 'the device name "all"') % (self.name)) - return distros.Distro._bring_up_interfaces(self, device_names) - - def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Adjust the sysconfig clock zone setting - clock_cfg = { - 'TIMEZONE': str(tz), - } - rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['zypper'] - # No user interaction possible, enable non-interactive mode - cmd.append('--non-interactive') - - # Comand is the operation, such as install - cmd.append(command) - - # args are the arguments to the command, not global options - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ['refresh'], freq=PER_INSTANCE) +class Distro(opensuse.Distro): + pass # vi: ts=4 expandtab diff --git a/templates/hosts.opensuse.tmpl b/templates/hosts.opensuse.tmpl new file mode 100644 index 00000000..655da3f7 --- /dev/null +++ b/templates/hosts.opensuse.tmpl @@ -0,0 +1,26 @@ +* + This file /etc/cloud/templates/hosts.opensuse.tmpl is only utilized + if enabled in cloud-config. Specifically, in order to enable it + you need to add the following to config: + manage_etc_hosts: True +*# +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in +# /etc/cloud/templates/hosts.opensuse.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.0.1 localhost + +# The following lines are desirable for IPv6 capable hosts +::1 localhost ipv6-localhost ipv6-loopback +fe00::0 ipv6-localnet + +ff00::0 ipv6-mcastprefix +ff02::1 ipv6-allnodes +ff02::2 ipv6-allrouters +ff02::3 ipv6-allhosts + diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl index 399ec9b4..b6082692 100644 --- a/templates/hosts.suse.tmpl +++ b/templates/hosts.suse.tmpl @@ -14,12 +14,9 @@ you need to add the following to config: # # The following lines are desirable for IPv4 capable hosts 127.0.0.1 localhost -127.0.0.1 {{fqdn}} {{hostname}} - # The following lines are desirable for IPv6 capable hosts ::1 localhost ipv6-localhost ipv6-loopback -::1 {{fqdn}} {{hostname}} fe00::0 ipv6-localnet ff00::0 ipv6-mcastprefix diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py new file mode 100644 index 00000000..bdb1d633 --- /dev/null +++ b/tests/unittests/test_distros/test_opensuse.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import CiTestCase + +from . import _get_distro + + +class TestopenSUSE(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("opensuse") + self.assertEqual(distro.osfamily, 'suse') diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py new file mode 100644 index 00000000..c656aacc --- /dev/null +++ b/tests/unittests/test_distros/test_sles.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import CiTestCase + +from . import _get_distro + + +class TestSLES(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("sles") + self.assertEqual(distro.osfamily, 'suse') diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index e9a810c5..aaf6c762 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -49,9 +49,15 @@ class TestLocale(t_help.FilesystemMockingTestCase): } cc = self._get_cloud('sles') cc_locale.handle('cc_locale', cfg, cc, LOG, []) - - contents = util.load_file('/etc/sysconfig/language', decode=False) + if cc.distro.uses_systemd: + locale_conf = cc.distro.systemd_locale_conf_fn + else: + locale_conf = cc.distro.locale_conf_fn + contents = util.load_file(locale_conf, decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) + if cc.distro.uses_systemd(): + self.assertEqual({'LANG': cfg['locale']}, dict(n_cfg)) + else: + self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 4b18de75..8165bf9a 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -70,7 +70,8 @@ class TestHostname(t_help.FilesystemMockingTestCase): cc = cloud.Cloud(ds, paths, {}, distro, None) self.patchUtils(self.tmp) cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) - contents = util.load_file("/etc/HOSTNAME") - self.assertEqual('blah', contents.strip()) + if not distro.uses_systemd(): + contents = util.load_file(distro.hostname_conf_fn) + self.assertEqual('blah', contents.strip()) # vi: ts=4 expandtab diff --git a/tox.ini b/tox.ini index 1e7ca2d3..a17156ce 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,22 @@ deps = six==1.9.0 -r{toxinidir}/test-requirements.txt +[testenv:opensusel42] +basepython = python2.7 +commands = nosetests {posargs:tests/unittests} +deps = + # requirements + argparse==1.3.0 + jinja2==2.8 + PyYAML==3.11 + PrettyTable==0.7.2 + oauthlib==0.7.2 + configobj==5.0.6 + requests==2.11.1 + jsonpatch==1.11 + six==1.9.0 + -r{toxinidir}/test-requirements.txt + [testenv:tip-pycodestyle] commands = {envpython} -m pycodestyle {posargs:cloudinit/ tests/ tools/} deps = pycodestyle -- cgit v1.2.3 From 502082f6f21fb7592a798087a4c49f90d886ad14 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 30 Aug 2017 10:35:30 -0400 Subject: tox: make xenial environment run with python3.6 The pinned versions of python packages in xenial do not work with python3.6. Currently, the failure can be seen with: $ tox -e xenial tests/unittests/test_merging.py which ends up failing with in /usr/lib/python3.6/inspect.py with: ValueError: Function has keyword-only parameters or annotations, use getfullargspec() API which can support them Instead of setting 'basepython' to 3.5 for the 'xenial', we just update the one package that does not run correctly with python3.6. That allows the developer to have either python3.5 or python3.6 installed and have tox work as expected. --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index a17156ce..ec96e859 100644 --- a/tox.ini +++ b/tox.ini @@ -66,8 +66,10 @@ deps = pyserial==3.0.1 configobj==5.0.6 requests==2.9.1 - # jsonpatch ubuntu is 1.10, not 1.19 (#839779) - jsonpatch==1.10 + # jsonpatch in xenial is 1.10, not 1.19 (#839779). The oldest version + # to work with python3.6 is 1.16 as found in Artful. To keep default + # invocation of 'tox' happy, accept the difference in version here. + jsonpatch==1.16 six==1.10.0 # test-requirements httpretty==0.8.6 -- cgit v1.2.3 From b931a6473ee929193c0048640bf34876ce831a15 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 29 Aug 2017 10:32:38 -0600 Subject: url_helper: dynamically import oauthlib import from inside oauth_headers oauth_headers is the only function which requires oauthlib, move the import and ImportError handling inside this function to only attempt loading at runtime if called. This will allow us to build on platforms that don't have python-oauthlib installed by default. Add simple unittests around the missing oauthlib dependencies to make sure the function performs as intended and raises and NotImplementedError if oauthlib can't be imported. --- cloudinit/tests/__init__.py | 0 cloudinit/tests/test_url_helper.py | 40 ++++++++++++++++++++++++++++++++++++++ cloudinit/url_helper.py | 10 +++------- 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 cloudinit/tests/__init__.py create mode 100644 cloudinit/tests/test_url_helper.py diff --git a/cloudinit/tests/__init__.py b/cloudinit/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py new file mode 100644 index 00000000..349110d9 --- /dev/null +++ b/cloudinit/tests/test_url_helper.py @@ -0,0 +1,40 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.url_helper import oauth_headers +from tests.unittests.helpers import CiTestCase, mock, skipIf + + +try: + import oauthlib + assert oauthlib # avoid pyflakes error F401: import unused + _missing_oauthlib_dep = False +except ImportError: + _missing_oauthlib_dep = True + + +class TestOAuthHeaders(CiTestCase): + + def test_oauth_headers_raises_not_implemented_when_oathlib_missing(self): + """oauth_headers raises a NotImplemented error when oauth absent.""" + with mock.patch.dict('sys.modules', {'oauthlib': None}): + with self.assertRaises(NotImplementedError) as context_manager: + oauth_headers(1, 2, 3, 4, 5) + self.assertEqual( + 'oauth support is not available', + str(context_manager.exception)) + + @skipIf(_missing_oauthlib_dep, "No python-oauthlib dependency") + @mock.patch('oauthlib.oauth1.Client') + def test_oauth_headers_calls_oathlibclient_when_available(self, m_client): + """oauth_headers calls oaut1.hClient.sign with the provided url.""" + class fakeclient(object): + def sign(self, url): + # The first and 3rd item of the client.sign tuple are ignored + return ('junk', url, 'junk2') + + m_client.return_value = fakeclient() + + return_value = oauth_headers( + 'url', 'consumer_key', 'token_key', 'token_secret', + 'consumer_secret') + self.assertEqual('url', return_value) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index c83061a9..0e0f5b4c 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -17,11 +17,6 @@ import time from email.utils import parsedate from functools import partial -try: - import oauthlib.oauth1 as oauth1 -except ImportError: - oauth1 = None - from requests import exceptions from six.moves.urllib.parse import ( @@ -492,8 +487,9 @@ class OauthUrlHelper(object): def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, timestamp=None): - - if oauth1 is None: + try: + import oauthlib.oauth1 as oauth1 + except ImportError: raise NotImplementedError('oauth support is not available') if timestamp: -- cgit v1.2.3 From 300e4516f78dbb0a9533749aa84f7e366b023d04 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 30 Aug 2017 20:58:27 -0400 Subject: tests: fix two recently added tests for sles distro. test_set_locale_sles and test_set_locale_sles_default were incorrectly testing for truth of .uses_systemd rather than calling that function and checking its result. The error was only seen if the system running the tests was not using systemd. --- tests/unittests/test_handler/test_handler_locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index aaf6c762..cba5cae8 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -49,7 +49,7 @@ class TestLocale(t_help.FilesystemMockingTestCase): } cc = self._get_cloud('sles') cc_locale.handle('cc_locale', cfg, cc, LOG, []) - if cc.distro.uses_systemd: + if cc.distro.uses_systemd(): locale_conf = cc.distro.systemd_locale_conf_fn else: locale_conf = cc.distro.locale_conf_fn -- cgit v1.2.3 From 7e76c57b590c7c2c13f7b1a2a8b5b7d4f2d18396 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 16 Aug 2017 16:50:07 -0500 Subject: distro: allow distro to specify a default locale Currently the cloud-init default locale (en_US.UTF-8) is set by the base datasource class. This patch allows a distro to overide the fallback value with one that's available in the distro but continues to respect an image which has preconfigured a locale. - Distro object now has a get_locale method which will return a preconfigure locale setting by checking the distros locale system configuration file. If not set or not present, return the default locale of en_US.UTF-8 which retains behavior of all previous cloud-init releases. - Apply locale now handles regenerating locales or system configuration files as needed. - Adjust apply_locale logic to skip locale-regen if the specified LANG value is C.UTF-8,C, or POSIX; they do not require regeneration. - Further add unittests to exercise the default paths for Ubuntu and non-ubuntu paths to validate they get the LANG expected. --- cloudinit/distros/__init__.py | 3 + cloudinit/distros/debian.py | 94 ++++++++++++++++------ cloudinit/sources/__init__.py | 9 ++- tests/unittests/test_distros/test_debian.py | 66 +++++++++------ tests/unittests/test_distros/test_generic.py | 16 ++++ tests/unittests/test_handler/test_handler_debug.py | 11 ++- .../unittests/test_handler/test_handler_locale.py | 48 +++++++++++ 7 files changed, 195 insertions(+), 52 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 807b3ea2..b714b9ab 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -188,6 +188,9 @@ class Distro(object): def _get_localhost_ip(self): return "127.0.0.1" + def get_locale(self): + raise NotImplementedError() + @abc.abstractmethod def _read_hostname(self, filename, default=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index abfb81f4..33cc0bf1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -61,11 +61,49 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' + self.default_locale = 'en_US.UTF-8' + self.system_locale = None - def apply_locale(self, locale, out_fn=None): + def get_locale(self): + """Return the default locale if set, else use default locale""" + + # read system locale value + if not self.system_locale: + self.system_locale = read_system_locale() + + # Return system_locale setting if valid, else use default locale + return (self.system_locale if self.system_locale else + self.default_locale) + + def apply_locale(self, locale, out_fn=None, keyname='LANG'): + """Apply specified locale to system, regenerate if specified locale + differs from system default.""" if not out_fn: out_fn = LOCALE_CONF_FN - apply_locale(locale, out_fn) + + if not locale: + raise ValueError('Failed to provide locale value.') + + # Only call locale regeneration if needed + # Update system locale config with specified locale if needed + distro_locale = self.get_locale() + conf_fn_exists = os.path.exists(out_fn) + sys_locale_unset = False if self.system_locale else True + need_regen = (locale.lower() != distro_locale.lower() or + not conf_fn_exists or sys_locale_unset) + need_conf = not conf_fn_exists or need_regen or sys_locale_unset + + if need_regen: + regenerate_locale(locale, out_fn, keyname=keyname) + else: + LOG.debug( + "System has '%s=%s' requested '%s', skipping regeneration.", + keyname, self.system_locale, locale) + + if need_conf: + update_locale_conf(locale, out_fn, keyname=keyname) + # once we've updated the system config, invalidate cache + self.system_locale = None def install_packages(self, pkglist): self.update_package_sources() @@ -218,37 +256,47 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): LOG.warning(msg) -def apply_locale(locale, sys_path=LOCALE_CONF_FN, keyname='LANG'): - """Apply the locale. - - Run locale-gen for the provided locale and set the default - system variable `keyname` appropriately in the provided `sys_path`. - - If sys_path indicates that `keyname` is already set to `locale` - then no changes will be made and locale-gen not called. - This allows images built with a locale already generated to not re-run - locale-gen which can be very heavy. - """ - if not locale: - raise ValueError('Failed to provide locale value.') - +def read_system_locale(sys_path=LOCALE_CONF_FN, keyname='LANG'): + """Read system default locale setting, if present""" + sys_val = "" if not sys_path: raise ValueError('Invalid path: %s' % sys_path) if os.path.exists(sys_path): locale_content = util.load_file(sys_path) - # if LANG isn't present, regen sys_defaults = util.load_shell_content(locale_content) sys_val = sys_defaults.get(keyname, "") - if sys_val.lower() == locale.lower(): - LOG.debug( - "System has '%s=%s' requested '%s', skipping regeneration.", - keyname, sys_val, locale) - return - util.subp(['locale-gen', locale], capture=False) + return sys_val + + +def update_locale_conf(locale, sys_path, keyname='LANG'): + """Update system locale config""" + LOG.debug('Updating %s with locale setting %s=%s', + sys_path, keyname, locale) util.subp( ['update-locale', '--locale-file=' + sys_path, '%s=%s' % (keyname, locale)], capture=False) + +def regenerate_locale(locale, sys_path, keyname='LANG'): + """ + Run locale-gen for the provided locale and set the default + system variable `keyname` appropriately in the provided `sys_path`. + + """ + # special case for locales which do not require regen + # % locale -a + # C + # C.UTF-8 + # POSIX + if locale.lower() in ['c', 'c.utf-8', 'posix']: + LOG.debug('%s=%s does not require rengeneration', keyname, locale) + return + + # finally, trigger regeneration + LOG.debug('Generating locales for %s', locale) + util.subp(['locale-gen', locale], capture=False) + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 952caf35..9a43fbee 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -44,6 +44,7 @@ class DataSourceNotFoundException(Exception): class DataSource(object): dsmode = DSMODE_NETWORK + default_locale = 'en_US.UTF-8' def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg @@ -150,7 +151,13 @@ class DataSource(object): return None def get_locale(self): - return 'en_US.UTF-8' + """Default locale is en_US.UTF-8, but allow distros to override""" + locale = self.default_locale + try: + locale = self.distro.get_locale() + except NotImplementedError: + pass + return locale @property def availability_zone(self): diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index 2330ad52..72d3aad6 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -1,67 +1,85 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import (CiTestCase, mock) - -from cloudinit.distros.debian import apply_locale +from cloudinit import distros from cloudinit import util +from ..helpers import (FilesystemMockingTestCase, mock) @mock.patch("cloudinit.distros.debian.util.subp") -class TestDebianApplyLocale(CiTestCase): +class TestDebianApplyLocale(FilesystemMockingTestCase): + + def setUp(self): + super(TestDebianApplyLocale, self).setUp() + self.new_root = self.tmp_dir() + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + self.spath = self.tmp_path('etc/default/locale', self.new_root) + cls = distros.fetch("debian") + self.distro = cls("debian", {}, None) + def test_no_rerun(self, m_subp): """If system has defined locale, no re-run is expected.""" - spath = self.tmp_path("default-locale") m_subp.return_value = (None, None) locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=%s\n' % locale, omode="w") - apply_locale(locale, sys_path=spath) + util.write_file(self.spath, 'LANG=%s\n' % locale, omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) m_subp.assert_not_called() + def test_no_regen_on_c_utf8(self, m_subp): + """If locale is set to C.UTF8, do not attempt to call locale-gen""" + m_subp.return_value = (None, None) + locale = 'C.UTF-8' + util.write_file(self.spath, 'LANG=%s\n' % 'en_US.UTF-8', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) + self.assertEqual( + [['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + def test_rerun_if_different(self, m_subp): """If system has different locale, locale-gen should be called.""" - spath = self.tmp_path("default-locale") m_subp.return_value = (None, None) locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=fr_FR.UTF-8', omode="w") - apply_locale(locale, sys_path=spath) + util.write_file(self.spath, 'LANG=fr_FR.UTF-8', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + ['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) def test_rerun_if_no_file(self, m_subp): """If system has no locale file, locale-gen should be called.""" - spath = self.tmp_path("default-locale") m_subp.return_value = (None, None) locale = 'en_US.UTF-8' - apply_locale(locale, sys_path=spath) + self.distro.apply_locale(locale, out_fn=self.spath) self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + ['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) def test_rerun_on_unset_system_locale(self, m_subp): """If system has unset locale, locale-gen should be called.""" m_subp.return_value = (None, None) - spath = self.tmp_path("default-locale") locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=', omode="w") - apply_locale(locale, sys_path=spath) + util.write_file(self.spath, 'LANG=', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + ['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) def test_rerun_on_mismatched_keys(self, m_subp): """If key is LC_ALL and system has only LANG, rerun is expected.""" m_subp.return_value = (None, None) - spath = self.tmp_path("default-locale") locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=', omode="w") - apply_locale(locale, sys_path=spath, keyname='LC_ALL') + util.write_file(self.spath, 'LANG=', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath, keyname='LC_ALL') self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, + ['update-locale', '--locale-file=' + self.spath, 'LC_ALL=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) @@ -69,14 +87,14 @@ class TestDebianApplyLocale(CiTestCase): """locale as None or "" is invalid and should raise ValueError.""" with self.assertRaises(ValueError) as ctext_m: - apply_locale(None) + self.distro.apply_locale(None) m_subp.assert_not_called() self.assertEqual( 'Failed to provide locale value.', str(ctext_m.exception)) with self.assertRaises(ValueError) as ctext_m: - apply_locale("") + self.distro.apply_locale("") m_subp.assert_not_called() self.assertEqual( 'Failed to provide locale value.', str(ctext_m.exception)) diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index c9be277e..b355a19e 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -228,5 +228,21 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): os.symlink('/', '/run/systemd/system') self.assertFalse(d.uses_systemd()) + @mock.patch('cloudinit.distros.debian.read_system_locale') + def test_get_locale_ubuntu(self, m_locale): + """Test ubuntu distro returns locale set to C.UTF-8""" + m_locale.return_value = 'C.UTF-8' + cls = distros.fetch("ubuntu") + d = cls("ubuntu", {}, None) + locale = d.get_locale() + self.assertEqual('C.UTF-8', locale) + + def test_get_locale_rhel(self): + """Test rhel distro returns NotImplementedError exception""" + cls = distros.fetch("rhel") + d = cls("rhel", {}, None) + with self.assertRaises(NotImplementedError): + d.get_locale() + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index 929f786e..1873c3e1 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -11,7 +11,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from ..helpers import (FilesystemMockingTestCase, mock) import logging import shutil @@ -20,7 +20,8 @@ import tempfile LOG = logging.getLogger(__name__) -class TestDebug(t_help.FilesystemMockingTestCase): +@mock.patch('cloudinit.distros.debian.read_system_locale') +class TestDebug(FilesystemMockingTestCase): def setUp(self): super(TestDebug, self).setUp() self.new_root = tempfile.mkdtemp() @@ -36,7 +37,8 @@ class TestDebug(t_help.FilesystemMockingTestCase): ds.metadata.update(metadata) return cloud.Cloud(ds, paths, {}, d, None) - def test_debug_write(self): + def test_debug_write(self, m_locale): + m_locale.return_value = 'en_US.UTF-8' cfg = { 'abc': '123', 'c': u'\u20a0', @@ -54,7 +56,8 @@ class TestDebug(t_help.FilesystemMockingTestCase): for k in cfg.keys(): self.assertIn(k, contents) - def test_debug_no_write(self): + def test_debug_no_write(self, m_locale): + m_locale.return_value = 'en_US.UTF-8' cfg = { 'abc': '123', 'debug': { diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index cba5cae8..a789db32 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -20,6 +20,8 @@ from configobj import ConfigObj from six import BytesIO import logging +import mock +import os import shutil import tempfile @@ -27,6 +29,9 @@ LOG = logging.getLogger(__name__) class TestLocale(t_help.FilesystemMockingTestCase): + + with_logs = True + def setUp(self): super(TestLocale, self).setUp() self.new_root = tempfile.mkdtemp() @@ -60,4 +65,47 @@ class TestLocale(t_help.FilesystemMockingTestCase): else: self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) + def test_set_locale_sles_default(self): + cfg = {} + cc = self._get_cloud('sles') + cc_locale.handle('cc_locale', cfg, cc, LOG, []) + + if cc.distro.uses_systemd(): + locale_conf = cc.distro.systemd_locale_conf_fn + keyname = 'LANG' + else: + locale_conf = cc.distro.locale_conf_fn + keyname = 'RC_LANG' + + contents = util.load_file(locale_conf, decode=False) + n_cfg = ConfigObj(BytesIO(contents)) + self.assertEqual({keyname: 'en_US.UTF-8'}, dict(n_cfg)) + + def test_locale_update_config_if_different_than_default(self): + """Test cc_locale writes updates conf if different than default""" + locale_conf = os.path.join(self.new_root, "etc/default/locale") + util.write_file(locale_conf, 'LANG="en_US.UTF-8"\n') + cfg = {'locale': 'C.UTF-8'} + cc = self._get_cloud('ubuntu') + with mock.patch('cloudinit.distros.debian.util.subp') as m_subp: + with mock.patch('cloudinit.distros.debian.LOCALE_CONF_FN', + locale_conf): + cc_locale.handle('cc_locale', cfg, cc, LOG, []) + m_subp.assert_called_with(['update-locale', + '--locale-file=%s' % locale_conf, + 'LANG=C.UTF-8'], capture=False) + + def test_locale_rhel_defaults_en_us_utf8(self): + """Test cc_locale gets en_US.UTF-8 from distro get_locale fallback""" + cfg = {} + cc = self._get_cloud('rhel') + update_sysconfig = 'cloudinit.distros.rhel_util.update_sysconfig_file' + with mock.patch.object(cc.distro, 'uses_systemd') as m_use_sd: + m_use_sd.return_value = True + with mock.patch(update_sysconfig) as m_update_syscfg: + cc_locale.handle('cc_locale', cfg, cc, LOG, []) + m_update_syscfg.assert_called_with('/etc/locale.conf', + {'LANG': 'en_US.UTF-8'}) + + # vi: ts=4 expandtab -- cgit v1.2.3 From 1770a1eb647d24e14732194e72210ea494986ad2 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 31 Aug 2017 16:24:35 -0600 Subject: tests: Stop leaking calls through unmocked metadata addresses DataSourceEc2 behavior changed to first check a minimum acceptable metadata version uri http://169.154.169.254//instance-id, retrying on 404, until the metadata service is available. After the metadata service is up, the datasource inspects preferred extended_metadata_versions for availability. Unit tests only mocked the preferred extended_metadata_version so all Ec2 tests were retrying attempts against http://169.254.169.254/meta-data//instance-id adding a lot of time cost to the unit test runs. This branch uses httpretty to properly mock the following: - 404s from metadata on undesired extended_metadata_version test routes - https://169.254.169.254/meta-data/2016-09-02/instance-id - full metadata dictionary represented on min_metadata_version - https://169.254.169.254/meta-data/2016-09-02/* The branch also tightens httpretty to raise a MockError for any URL which isn't mocked via httpretty.HTTPretty.allow_net_connect=False. LP: #1714117 --- tests/unittests/test_datasource/test_ec2.py | 46 +++++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index e1ce6446..b7a84e21 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -116,6 +116,9 @@ def register_mock_metaserver(base_url, data): In the index, references to lists or dictionaries have a trailing /. """ def register_helper(register, base_url, body): + if not isinstance(base_url, str): + register(base_url, body) + return base_url = base_url.rstrip("/") if isinstance(body, str): register(base_url, body) @@ -138,7 +141,7 @@ def register_mock_metaserver(base_url, data): register(base_url, '\n'.join(vals) + '\n') register(base_url + '/', '\n'.join(vals) + '\n') elif body is None: - register(base_url, 'not found', status_code=404) + register(base_url, 'not found', status=404) def myreg(*argc, **kwargs): # print("register_url(%s, %s)" % (argc, kwargs)) @@ -161,38 +164,47 @@ class TestEc2(test_helpers.HttprettyTestCase): self.datasource = ec2.DataSourceEc2 self.metadata_addr = self.datasource.metadata_urls[0] - @property - def metadata_url(self): - return '/'.join([ - self.metadata_addr, - self.datasource.min_metadata_version, 'meta-data', '']) - - @property - def userdata_url(self): - return '/'.join([ - self.metadata_addr, - self.datasource.min_metadata_version, 'user-data']) + def data_url(self, version): + """Return a metadata url based on the version provided.""" + return '/'.join([self.metadata_addr, version, 'meta-data', '']) def _patch_add_cleanup(self, mpath, *args, **kwargs): p = mock.patch(mpath, *args, **kwargs) p.start() self.addCleanup(p.stop) - def _setup_ds(self, sys_cfg, platform_data, md, ud=None): + def _setup_ds(self, sys_cfg, platform_data, md, md_version=None): + self.uris = [] distro = {} paths = helpers.Paths({}) if sys_cfg is None: sys_cfg = {} ds = self.datasource(sys_cfg=sys_cfg, distro=distro, paths=paths) + if not md_version: + md_version = ds.min_metadata_version if platform_data is not None: self._patch_add_cleanup( "cloudinit.sources.DataSourceEc2._collect_platform_data", return_value=platform_data) if md: - register_mock_metaserver(self.metadata_url, md) - register_mock_metaserver(self.userdata_url, ud) - + httpretty.HTTPretty.allow_net_connect = False + all_versions = ( + [ds.min_metadata_version] + ds.extended_metadata_versions) + for version in all_versions: + metadata_url = self.data_url(version) + if version == md_version: + # Register all metadata for desired version + register_mock_metaserver(metadata_url, md) + else: + instance_id_url = metadata_url + 'instance-id' + if version == ds.min_metadata_version: + # Add min_metadata_version service availability check + register_mock_metaserver( + instance_id_url, DEFAULT_METADATA['instance-id']) + else: + # Register 404s for all unrequested extended versions + register_mock_metaserver(instance_id_url, None) return ds @httpretty.activate @@ -297,6 +309,7 @@ class TestEc2(test_helpers.HttprettyTestCase): Then the metadata services is crawled for more network config info. When the platform data is valid, return True. """ + m_is_bsd.return_value = False m_dhcp.return_value = [{ 'interface': 'eth9', 'fixed-address': '192.168.2.9', @@ -307,6 +320,7 @@ class TestEc2(test_helpers.HttprettyTestCase): platform_data=self.valid_platform_data, sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, md=DEFAULT_METADATA) + ret = ds.get_data() self.assertTrue(ret) m_dhcp.assert_called_once_with() -- cgit v1.2.3 From fa266bf8818a08e37cd32a603d076ba2db300124 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 31 Aug 2017 20:01:57 -0600 Subject: upstart: do not package upstart jobs, drop ubuntu-init-switch module. The ubuntu-init-switch module allowed the use to launch an instance that was booted with upstart and have it switch its init system to systemd and then reboot itself. It was only useful for the time period when Ubuntu was transitioning to systemd but only produced images using upstart. Also, do not run setup with --init-system=upstart. This means that by default, debian packages built with packages/bddeb will not have upstart unit files included. No other removal is done here. --- cloudinit/config/cc_ubuntu_init_switch.py | 160 ------------------------------ config/cloud.cfg.tmpl | 3 - doc/rtd/topics/modules.rst | 1 - packages/bddeb | 3 +- packages/debian/dirs | 1 - packages/debian/rules.in | 2 +- setup.py | 2 + tests/cloud_tests/configs/modules/TODO.md | 2 - 8 files changed, 4 insertions(+), 170 deletions(-) delete mode 100644 cloudinit/config/cc_ubuntu_init_switch.py diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py deleted file mode 100644 index 5dd26901..00000000 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (C) 2014 Canonical Ltd. -# -# Author: Scott Moser -# -# This file is part of cloud-init. See LICENSE file for license information. - -""" -Ubuntu Init Switch ------------------- -**Summary:** reboot system into another init. - -This module provides a way for the user to boot with systemd even if the image -is set to boot with upstart. It should be run as one of the first -``cloud_init_modules``, and will switch the init system and then issue a -reboot. The next boot will come up in the target init system and no action -will be taken. This should be inert on non-ubuntu systems, and also -exit quickly. - -.. note:: - best effort is made, but it's possible this system will break, and probably - won't interact well with any other mechanism you've used to switch the init - system. - -**Internal name:** ``cc_ubuntu_init_switch`` - -**Module frequency:** once per instance - -**Supported distros:** ubuntu - -**Config keys**:: - - init_switch: - target: systemd (can be 'systemd' or 'upstart') - reboot: true (reboot if a change was made, or false to not reboot) -""" - -from cloudinit.distros import ubuntu -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -import os -import time - -frequency = PER_INSTANCE -REBOOT_CMD = ["/sbin/reboot", "--force"] - -DEFAULT_CONFIG = { - 'init_switch': {'target': None, 'reboot': True} -} - -SWITCH_INIT = """ -#!/bin/sh -# switch_init: [upstart | systemd] - -is_systemd() { - [ "$(dpkg-divert --listpackage /sbin/init)" = "systemd-sysv" ] -} -debug() { echo "$@" 1>&2; } -fail() { echo "$@" 1>&2; exit 1; } - -if [ "$1" = "systemd" ]; then - if is_systemd; then - debug "already systemd, nothing to do" - else - [ -f /lib/systemd/systemd ] || fail "no systemd available"; - dpkg-divert --package systemd-sysv --divert /sbin/init.diverted \\ - --rename /sbin/init - fi - [ -f /sbin/init ] || ln /lib/systemd/systemd /sbin/init -elif [ "$1" = "upstart" ]; then - if is_systemd; then - rm -f /sbin/init - dpkg-divert --package systemd-sysv --rename --remove /sbin/init - else - debug "already upstart, nothing to do." - fi -else - fail "Error. expect 'upstart' or 'systemd'" -fi -""" - -distros = ['ubuntu'] - - -def handle(name, cfg, cloud, log, args): - """Handler method activated by cloud-init.""" - - if not isinstance(cloud.distro, ubuntu.Distro): - log.debug("%s: distro is '%s', not ubuntu. returning", - name, cloud.distro.__class__) - return - - cfg = util.mergemanydict([cfg, DEFAULT_CONFIG]) - target = cfg['init_switch']['target'] - reboot = cfg['init_switch']['reboot'] - - if len(args) != 0: - target = args[0] - if len(args) > 1: - reboot = util.is_true(args[1]) - - if not target: - log.debug("%s: target=%s. nothing to do", name, target) - return - - if not util.which('dpkg'): - log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name) - return - - supported = ('upstart', 'systemd') - if target not in supported: - log.warn("%s: target set to %s, expected one of: %s", - name, target, str(supported)) - - if os.path.exists("/run/systemd/system"): - current = "systemd" - else: - current = "upstart" - - if current == target: - log.debug("%s: current = target = %s. nothing to do", name, target) - return - - try: - util.subp(['sh', '-s', target], data=SWITCH_INIT) - except util.ProcessExecutionError as e: - log.warn("%s: Failed to switch to init '%s'. %s", name, target, e) - return - - if util.is_false(reboot): - log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.", - name, current, target) - return - - try: - log.warn("%s: switched '%s' to '%s'. rebooting.", - name, current, target) - logging.flushLoggers(log) - _fire_reboot(log, wait_attempts=4, initial_sleep=4) - except Exception as e: - util.logexc(log, "Requested reboot did not happen!") - raise - - -def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) - start = time.time() - wait_time = initial_sleep - for _i in range(0, wait_attempts): - time.sleep(wait_time) - wait_time *= backoff - elapsed = time.time() - start - log.debug("Rebooted, but still running after %s seconds", int(elapsed)) - # If we got here, not good - elapsed = time.time() - start - raise RuntimeError(("Reboot did not happen" - " after %s seconds!") % (int(elapsed))) - -# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index f4b9069b..a537d65a 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -45,9 +45,6 @@ datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] # The modules that run in the 'init' stage cloud_init_modules: - migrator -{% if variant in ["ubuntu", "unknown", "debian"] %} - - ubuntu-init-switch -{% endif %} - seed_random - bootcmd - write-files diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index c963c09a..cdb0f419 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -50,7 +50,6 @@ Modules .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone -.. automodule:: cloudinit.config.cc_ubuntu_init_switch .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname .. automodule:: cloudinit.config.cc_users_groups diff --git a/packages/bddeb b/packages/bddeb index 609a94fb..7c123548 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -112,8 +112,7 @@ def get_parser(): parser.add_argument("--init-system", dest="init_system", help=("build deb with INIT_SYSTEM=xxx" " (default: %(default)s"), - default=os.environ.get("INIT_SYSTEM", - "upstart,systemd")) + default=os.environ.get("INIT_SYSTEM", "systemd")) parser.add_argument("--release", dest="release", help=("build with changelog referencing RELEASE"), diff --git a/packages/debian/dirs b/packages/debian/dirs index 9a633c60..1315cf8a 100644 --- a/packages/debian/dirs +++ b/packages/debian/dirs @@ -1,6 +1,5 @@ var/lib/cloud usr/bin -etc/init usr/share/doc/cloud etc/cloud lib/udev/rules.d diff --git a/packages/debian/rules.in b/packages/debian/rules.in index 053b7649..b87a5e84 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -1,6 +1,6 @@ ## template:basic #!/usr/bin/make -f -INIT_SYSTEM ?= upstart,systemd +INIT_SYSTEM ?= systemd export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) PYVER ?= python${pyver} diff --git a/setup.py b/setup.py index 5c65c7fe..7662bd8b 100755 --- a/setup.py +++ b/setup.py @@ -191,6 +191,8 @@ class InitsysInstallData(install): datakeys = [k for k in INITSYS_ROOTS if k.partition(".")[0] == system] for k in datakeys: + if not INITSYS_FILES[k]: + continue self.distribution.data_files.append( (INITSYS_ROOTS[k], INITSYS_FILES[k])) # Force that command to reinitalize (with new file list) diff --git a/tests/cloud_tests/configs/modules/TODO.md b/tests/cloud_tests/configs/modules/TODO.md index d496da95..0b933b3b 100644 --- a/tests/cloud_tests/configs/modules/TODO.md +++ b/tests/cloud_tests/configs/modules/TODO.md @@ -89,8 +89,6 @@ Not applicable to write a test for this as it specifies when something should be ## ssh authkey fingerprints The authkey_hash key does not appear to work. In fact the default claims to be md5, however syslog only shows sha256 -## ubuntu init switch - ## update etc hosts 2016-11-17: Issues with changing /etc/hosts and lxc backend. -- cgit v1.2.3 From 653c0b4cfc6325382a3fb93a2185ab74f9cee62a Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Fri, 1 Sep 2017 16:35:01 -0600 Subject: tox: add nose timer output This adds the output of the nose timer plugin to the py3 environment to tox. This will print out the 10 longest running tests and automatically turn tests longer than 1 second "red" after the coverage output. --- tox.ini | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index ec96e859..72de9830 100644 --- a/tox.ini +++ b/tox.ini @@ -30,10 +30,13 @@ commands = {envpython} -m pylint {posargs:cloudinit} [testenv:py3] basepython = python3 -deps = -r{toxinidir}/test-requirements.txt -commands = {envpython} -m nose {posargs:--with-coverage \ - --cover-erase --cover-branches --cover-inclusive \ - --cover-package=cloudinit tests/unittests cloudinit} +deps = + nose-timer + -r{toxinidir}/test-requirements.txt +commands = {envpython} -m nose --with-timer --timer-top-n 10 \ + {posargs:--with-coverage --cover-erase --cover-branches \ + --cover-inclusive --cover-package=cloudinit \ + tests/unittests cloudinit} [testenv:py27] basepython = python2.7 -- cgit v1.2.3 From a3649e03206a3596131413956ea7ecc18790ec73 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 5 Sep 2017 11:03:59 -0600 Subject: relocate tests/unittests/helpers.py to cloudinit/tests This moves the base test case classes into into cloudinit/tests and updates all the corresponding imports. --- cloudinit/analyze/tests/test_dump.py | 2 +- cloudinit/net/tests/test_dhcp.py | 2 +- cloudinit/net/tests/test_init.py | 2 +- cloudinit/tests/helpers.py | 395 +++++++++++++++++++++ cloudinit/tests/test_url_helper.py | 2 +- tests/unittests/helpers.py | 391 -------------------- tests/unittests/test__init__.py | 2 +- tests/unittests/test_atomic_helper.py | 2 +- tests/unittests/test_builtin_handlers.py | 2 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_cs_util.py | 2 +- tests/unittests/test_data.py | 2 +- tests/unittests/test_datasource/test_aliyun.py | 2 +- tests/unittests/test_datasource/test_altcloud.py | 2 +- tests/unittests/test_datasource/test_azure.py | 6 +- .../unittests/test_datasource/test_azure_helper.py | 2 +- tests/unittests/test_datasource/test_cloudsigma.py | 2 +- tests/unittests/test_datasource/test_cloudstack.py | 2 +- tests/unittests/test_datasource/test_common.py | 2 +- .../unittests/test_datasource/test_configdrive.py | 2 +- .../unittests/test_datasource/test_digitalocean.py | 2 +- tests/unittests/test_datasource/test_ec2.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 2 +- tests/unittests/test_datasource/test_ovf.py | 2 +- tests/unittests/test_datasource/test_scaleway.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_distros/test_arch.py | 2 +- tests/unittests/test_distros/test_create_users.py | 2 +- tests/unittests/test_distros/test_debian.py | 2 +- tests/unittests/test_distros/test_generic.py | 2 +- tests/unittests/test_distros/test_netconfig.py | 2 +- tests/unittests/test_distros/test_opensuse.py | 2 +- tests/unittests/test_distros/test_resolv.py | 2 +- tests/unittests/test_distros/test_sles.py | 2 +- tests/unittests/test_distros/test_sysconfig.py | 2 +- .../test_distros/test_user_data_normalize.py | 2 +- tests/unittests/test_ds_identify.py | 3 +- tests/unittests/test_ec2_util.py | 2 +- tests/unittests/test_filters/test_launch_index.py | 2 +- .../test_handler/test_handler_apt_conf_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v3.py | 2 +- .../test_handler/test_handler_apt_source_v1.py | 2 +- .../test_handler/test_handler_apt_source_v3.py | 2 +- .../test_handler/test_handler_ca_certs.py | 2 +- tests/unittests/test_handler/test_handler_chef.py | 2 +- tests/unittests/test_handler/test_handler_debug.py | 2 +- .../test_handler/test_handler_disk_setup.py | 2 +- .../test_handler/test_handler_growpart.py | 2 +- .../test_handler/test_handler_landscape.py | 5 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 2 +- .../test_handler/test_handler_mcollective.py | 2 +- .../unittests/test_handler/test_handler_mounts.py | 2 +- tests/unittests/test_handler/test_handler_ntp.py | 2 +- .../test_handler/test_handler_power_state.py | 4 +- .../unittests/test_handler/test_handler_puppet.py | 2 +- .../unittests/test_handler/test_handler_rsyslog.py | 2 +- .../unittests/test_handler/test_handler_runcmd.py | 2 +- .../test_handler/test_handler_seed_random.py | 2 +- .../test_handler/test_handler_set_hostname.py | 2 +- .../unittests/test_handler/test_handler_snappy.py | 4 +- .../test_handler/test_handler_spacewalk.py | 2 +- .../test_handler/test_handler_timezone.py | 2 +- .../test_handler/test_handler_write_files.py | 2 +- .../test_handler/test_handler_yum_add_repo.py | 2 +- tests/unittests/test_handler/test_schema.py | 2 +- tests/unittests/test_helpers.py | 2 +- tests/unittests/test_log.py | 2 +- tests/unittests/test_merging.py | 2 +- tests/unittests/test_net.py | 8 +- tests/unittests/test_pathprefix2dict.py | 2 +- tests/unittests/test_registry.py | 2 +- tests/unittests/test_reporting.py | 2 +- tests/unittests/test_rh_subscription.py | 2 +- tests/unittests/test_runs/test_merge_run.py | 2 +- tests/unittests/test_runs/test_simple_run.py | 2 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 2 +- tests/unittests/test_util.py | 2 +- tests/unittests/test_version.py | 2 +- tests/unittests/test_vmware_config_file.py | 2 +- 86 files changed, 491 insertions(+), 482 deletions(-) create mode 100644 cloudinit/tests/helpers.py delete mode 100644 tests/unittests/helpers.py diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index 2c0885d0..f4c42841 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -6,7 +6,7 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) from cloudinit.util import subp, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseTimestamp(CiTestCase): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 47d8d461..4a37e98a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -8,7 +8,7 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery) from cloudinit.util import ensure_file, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseDHCPLeasesFile(CiTestCase): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index cc052a7d..8cb4114e 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -7,7 +7,7 @@ import os import cloudinit.net as net from cloudinit.util import ensure_file, write_file, ProcessExecutionError -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestSysDevPath(CiTestCase): diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py new file mode 100644 index 00000000..28e26622 --- /dev/null +++ b/cloudinit/tests/helpers.py @@ -0,0 +1,395 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from __future__ import print_function + +import functools +import json +import logging +import os +import shutil +import sys +import tempfile +import unittest + +import mock +import six +import unittest2 + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + +from cloudinit import helpers as ch +from cloudinit import util + +# Used for skipping tests +SkipTest = unittest2.SkipTest + +# Used for detecting different python versions +PY2 = False +PY26 = False +PY27 = False +PY3 = False + +_PY_VER = sys.version_info +_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] +if (_PY_MAJOR, _PY_MINOR) <= (2, 6): + if (_PY_MAJOR, _PY_MINOR) == (2, 6): + PY26 = True + if (_PY_MAJOR, _PY_MINOR) >= (2, 0): + PY2 = True +else: + if (_PY_MAJOR, _PY_MINOR) == (2, 7): + PY27 = True + PY2 = True + if (_PY_MAJOR, _PY_MINOR) >= (3, 0): + PY3 = True + + +# Makes the old path start +# with new base instead of whatever +# it previously had +def rebase_path(old_path, new_base): + if old_path.startswith(new_base): + # Already handled... + return old_path + # Retarget the base of that path + # to the new base instead of the + # old one... + path = os.path.join(new_base, old_path.lstrip("/")) + path = os.path.abspath(path) + return path + + +# Can work on anything that takes a path as arguments +def retarget_many_wrapper(new_base, am, old_func): + def wrapper(*args, **kwds): + n_args = list(args) + nam = am + if am == -1: + nam = len(n_args) + for i in range(0, nam): + path = args[i] + # patchOS() wraps various os and os.path functions, however in + # Python 3 some of these now accept file-descriptors (integers). + # That breaks rebase_path() so in lieu of a better solution, just + # don't rebase if we get a fd. + if isinstance(path, six.string_types): + n_args[i] = rebase_path(path, new_base) + return old_func(*n_args, **kwds) + return wrapper + + +class TestCase(unittest2.TestCase): + + def reset_global_state(self): + """Reset any global state to its original settings. + + cloudinit caches some values in cloudinit.util. Unit tests that + involved those cached paths were then subject to failure if the order + of invocation changed (LP: #1703697). + + This function resets any of these global state variables to their + initial state. + + In the future this should really be done with some registry that + can then be cleaned in a more obvious way. + """ + util.PROC_CMDLINE = None + util._DNS_REDIRECT_IP = None + util._LSB_RELEASE = {} + + def setUp(self): + super(TestCase, self).setUp() + self.reset_global_state() + + +class CiTestCase(TestCase): + """This is the preferred test case base class unless user + needs other test case classes below.""" + + # Subclass overrides for specific test behavior + # Whether or not a unit test needs logfile setup + with_logs = False + + def setUp(self): + super(CiTestCase, self).setUp() + if self.with_logs: + # Create a log handler so unit tests can search expected logs. + self.logger = logging.getLogger() + self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') + handler = logging.StreamHandler(self.logs) + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] + + def tearDown(self): + if self.with_logs: + # Remove the handler we setup + logging.getLogger().handlers = self.old_handlers + super(CiTestCase, self).tearDown() + + def tmp_dir(self, dir=None, cleanup=True): + # return a full path to a temporary directory that will be cleaned up. + if dir is None: + tmpd = tempfile.mkdtemp( + prefix="ci-%s." % self.__class__.__name__) + else: + tmpd = tempfile.mkdtemp(dir=dir) + self.addCleanup(functools.partial(shutil.rmtree, tmpd)) + return tmpd + + def tmp_path(self, path, dir=None): + # return an absolute path to 'path' under dir. + # if dir is None, one will be created with tmp_dir() + # the file is not created or modified. + if dir is None: + dir = self.tmp_dir() + return os.path.normpath(os.path.abspath(os.path.join(dir, path))) + + +class ResourceUsingTestCase(CiTestCase): + + def setUp(self): + super(ResourceUsingTestCase, self).setUp() + self.resource_path = None + + def resourceLocation(self, subname=None): + if self.resource_path is None: + paths = [ + os.path.join('tests', 'data'), + os.path.join('data'), + os.path.join(os.pardir, 'tests', 'data'), + os.path.join(os.pardir, 'data'), + ] + for p in paths: + if os.path.isdir(p): + self.resource_path = p + break + self.assertTrue((self.resource_path and + os.path.isdir(self.resource_path)), + msg="Unable to locate test resource data path!") + if not subname: + return self.resource_path + return os.path.join(self.resource_path, subname) + + def readResource(self, name): + where = self.resourceLocation(name) + with open(where, 'r') as fh: + return fh.read() + + def getCloudPaths(self, ds=None): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + cp = ch.Paths({'cloud_dir': tmpdir, + 'templates_dir': self.resourceLocation()}, + ds=ds) + return cp + + +class FilesystemMockingTestCase(ResourceUsingTestCase): + + def setUp(self): + super(FilesystemMockingTestCase, self).setUp() + self.patched_funcs = ExitStack() + + def tearDown(self): + self.patched_funcs.close() + ResourceUsingTestCase.tearDown(self) + + def replicateTestRoot(self, example_root, target_root): + real_root = self.resourceLocation() + real_root = os.path.join(real_root, 'roots', example_root) + for (dir_path, _dirnames, filenames) in os.walk(real_root): + real_path = dir_path + make_path = rebase_path(real_path[len(real_root):], target_root) + util.ensure_dir(make_path) + for f in filenames: + real_path = util.abs_join(real_path, f) + make_path = util.abs_join(make_path, f) + shutil.copy(real_path, make_path) + + def patchUtils(self, new_root): + patch_funcs = { + util: [('write_file', 1), + ('append_file', 1), + ('load_file', 1), + ('ensure_dir', 1), + ('chmod', 1), + ('delete_dir_contents', 1), + ('del_file', 1), + ('sym_link', -1), + ('copy', -1)], + } + for (mod, funcs) in patch_funcs.items(): + for (f, am) in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, am, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + # Handle subprocess calls + func = getattr(util, 'subp') + + def nsubp(*_args, **_kwargs): + return ('', '') + + self.patched_funcs.enter_context( + mock.patch.object(util, 'subp', nsubp)) + + def null_func(*_args, **_kwargs): + return None + + for f in ['chownbyid', 'chownbyname']: + self.patched_funcs.enter_context( + mock.patch.object(util, f, null_func)) + + def patchOS(self, new_root): + patch_funcs = { + os.path: [('isfile', 1), ('exists', 1), + ('islink', 1), ('isdir', 1)], + os: [('listdir', 1), ('mkdir', 1), + ('lstat', 1), ('symlink', 2)], + } + for (mod, funcs) in patch_funcs.items(): + for f, nargs in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, nargs, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + def patchOpen(self, new_root): + trap_func = retarget_many_wrapper(new_root, 1, open) + name = 'builtins.open' if PY3 else '__builtin__.open' + self.patched_funcs.enter_context(mock.patch(name, trap_func)) + + def patchStdoutAndStderr(self, stdout=None, stderr=None): + if stdout is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stdout', stdout)) + if stderr is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stderr', stderr)) + + def reRoot(self, root=None): + if root is None: + root = self.tmp_dir() + self.patchUtils(root) + self.patchOS(root) + return root + + +class HttprettyTestCase(CiTestCase): + # necessary as http_proxy gets in the way of httpretty + # https://github.com/gabrielfalcao/HTTPretty/issues/122 + + def setUp(self): + self.restore_proxy = os.environ.get('http_proxy') + if self.restore_proxy is not None: + del os.environ['http_proxy'] + super(HttprettyTestCase, self).setUp() + + def tearDown(self): + if self.restore_proxy: + os.environ['http_proxy'] = self.restore_proxy + super(HttprettyTestCase, self).tearDown() + + +def populate_dir(path, files): + if not os.path.exists(path): + os.makedirs(path) + ret = [] + for (name, content) in files.items(): + p = os.path.sep.join([path, name]) + util.ensure_dir(os.path.dirname(p)) + with open(p, "wb") as fp: + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) + fp.close() + ret.append(p) + + return ret + + +def dir2dict(startdir, prefix=None): + flist = {} + if prefix is None: + prefix = startdir + for root, dirs, files in os.walk(startdir): + for fname in files: + fpath = os.path.join(root, fname) + key = fpath[len(prefix):] + flist[key] = util.load_file(fpath) + return flist + + +def json_dumps(data): + # print data in nicely formatted json. + return json.dumps(data, indent=1, sort_keys=True, + separators=(',', ': ')) + + +def wrap_and_call(prefix, mocks, func, *args, **kwargs): + """ + call func(args, **kwargs) with mocks applied, then unapplies mocks + nicer to read than repeating dectorators on each function + + prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None + mocks: dictionary of names (under 'prefix') to mock and either + a return value or a dictionary to pass to the mock.patch call + func: function to call with mocks applied + *args,**kwargs: arguments for 'func' + + return_value: return from 'func' + """ + delim = '.' + if prefix is None: + prefix = '' + prefix = prefix.rstrip(delim) + unwraps = [] + for fname, kw in mocks.items(): + if prefix: + fname = delim.join((prefix, fname)) + if not isinstance(kw, dict): + kw = {'return_value': kw} + p = mock.patch(fname, **kw) + p.start() + unwraps.append(p) + try: + return func(*args, **kwargs) + finally: + for p in unwraps: + p.stop() + + +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator + + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 349110d9..b778a3a7 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.url_helper import oauth_headers -from tests.unittests.helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf try: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py deleted file mode 100644 index bf1dc5df..00000000 --- a/tests/unittests/helpers.py +++ /dev/null @@ -1,391 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from __future__ import print_function - -import functools -import json -import logging -import os -import shutil -import sys -import tempfile -import unittest - -import mock -import six -import unittest2 - -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - -from cloudinit import helpers as ch -from cloudinit import util - -# Used for skipping tests -SkipTest = unittest2.SkipTest - -# Used for detecting different python versions -PY2 = False -PY26 = False -PY27 = False -PY3 = False - -_PY_VER = sys.version_info -_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] -if (_PY_MAJOR, _PY_MINOR) <= (2, 6): - if (_PY_MAJOR, _PY_MINOR) == (2, 6): - PY26 = True - if (_PY_MAJOR, _PY_MINOR) >= (2, 0): - PY2 = True -else: - if (_PY_MAJOR, _PY_MINOR) == (2, 7): - PY27 = True - PY2 = True - if (_PY_MAJOR, _PY_MINOR) >= (3, 0): - PY3 = True - - -# Makes the old path start -# with new base instead of whatever -# it previously had -def rebase_path(old_path, new_base): - if old_path.startswith(new_base): - # Already handled... - return old_path - # Retarget the base of that path - # to the new base instead of the - # old one... - path = os.path.join(new_base, old_path.lstrip("/")) - path = os.path.abspath(path) - return path - - -# Can work on anything that takes a path as arguments -def retarget_many_wrapper(new_base, am, old_func): - def wrapper(*args, **kwds): - n_args = list(args) - nam = am - if am == -1: - nam = len(n_args) - for i in range(0, nam): - path = args[i] - # patchOS() wraps various os and os.path functions, however in - # Python 3 some of these now accept file-descriptors (integers). - # That breaks rebase_path() so in lieu of a better solution, just - # don't rebase if we get a fd. - if isinstance(path, six.string_types): - n_args[i] = rebase_path(path, new_base) - return old_func(*n_args, **kwds) - return wrapper - - -class TestCase(unittest2.TestCase): - def reset_global_state(self): - """Reset any global state to its original settings. - - cloudinit caches some values in cloudinit.util. Unit tests that - involved those cached paths were then subject to failure if the order - of invocation changed (LP: #1703697). - - This function resets any of these global state variables to their - initial state. - - In the future this should really be done with some registry that - can then be cleaned in a more obvious way. - """ - util.PROC_CMDLINE = None - util._DNS_REDIRECT_IP = None - util._LSB_RELEASE = {} - - def setUp(self): - super(unittest2.TestCase, self).setUp() - self.reset_global_state() - - -class CiTestCase(TestCase): - """This is the preferred test case base class unless user - needs other test case classes below.""" - - # Subclass overrides for specific test behavior - # Whether or not a unit test needs logfile setup - with_logs = False - - def setUp(self): - super(CiTestCase, self).setUp() - if self.with_logs: - # Create a log handler so unit tests can search expected logs. - self.logger = logging.getLogger() - self.logs = six.StringIO() - formatter = logging.Formatter('%(levelname)s: %(message)s') - handler = logging.StreamHandler(self.logs) - handler.setFormatter(formatter) - self.old_handlers = self.logger.handlers - self.logger.handlers = [handler] - - def tearDown(self): - if self.with_logs: - # Remove the handler we setup - logging.getLogger().handlers = self.old_handlers - super(CiTestCase, self).tearDown() - - def tmp_dir(self, dir=None, cleanup=True): - # return a full path to a temporary directory that will be cleaned up. - if dir is None: - tmpd = tempfile.mkdtemp( - prefix="ci-%s." % self.__class__.__name__) - else: - tmpd = tempfile.mkdtemp(dir=dir) - self.addCleanup(functools.partial(shutil.rmtree, tmpd)) - return tmpd - - def tmp_path(self, path, dir=None): - # return an absolute path to 'path' under dir. - # if dir is None, one will be created with tmp_dir() - # the file is not created or modified. - if dir is None: - dir = self.tmp_dir() - return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - - -class ResourceUsingTestCase(CiTestCase): - def setUp(self): - super(ResourceUsingTestCase, self).setUp() - self.resource_path = None - - def resourceLocation(self, subname=None): - if self.resource_path is None: - paths = [ - os.path.join('tests', 'data'), - os.path.join('data'), - os.path.join(os.pardir, 'tests', 'data'), - os.path.join(os.pardir, 'data'), - ] - for p in paths: - if os.path.isdir(p): - self.resource_path = p - break - self.assertTrue((self.resource_path and - os.path.isdir(self.resource_path)), - msg="Unable to locate test resource data path!") - if not subname: - return self.resource_path - return os.path.join(self.resource_path, subname) - - def readResource(self, name): - where = self.resourceLocation(name) - with open(where, 'r') as fh: - return fh.read() - - def getCloudPaths(self, ds=None): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) - cp = ch.Paths({'cloud_dir': tmpdir, - 'templates_dir': self.resourceLocation()}, - ds=ds) - return cp - - -class FilesystemMockingTestCase(ResourceUsingTestCase): - def setUp(self): - super(FilesystemMockingTestCase, self).setUp() - self.patched_funcs = ExitStack() - - def tearDown(self): - self.patched_funcs.close() - ResourceUsingTestCase.tearDown(self) - - def replicateTestRoot(self, example_root, target_root): - real_root = self.resourceLocation() - real_root = os.path.join(real_root, 'roots', example_root) - for (dir_path, _dirnames, filenames) in os.walk(real_root): - real_path = dir_path - make_path = rebase_path(real_path[len(real_root):], target_root) - util.ensure_dir(make_path) - for f in filenames: - real_path = util.abs_join(real_path, f) - make_path = util.abs_join(make_path, f) - shutil.copy(real_path, make_path) - - def patchUtils(self, new_root): - patch_funcs = { - util: [('write_file', 1), - ('append_file', 1), - ('load_file', 1), - ('ensure_dir', 1), - ('chmod', 1), - ('delete_dir_contents', 1), - ('del_file', 1), - ('sym_link', -1), - ('copy', -1)], - } - for (mod, funcs) in patch_funcs.items(): - for (f, am) in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, am, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - # Handle subprocess calls - func = getattr(util, 'subp') - - def nsubp(*_args, **_kwargs): - return ('', '') - - self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) - - def null_func(*_args, **_kwargs): - return None - - for f in ['chownbyid', 'chownbyname']: - self.patched_funcs.enter_context( - mock.patch.object(util, f, null_func)) - - def patchOS(self, new_root): - patch_funcs = { - os.path: [('isfile', 1), ('exists', 1), - ('islink', 1), ('isdir', 1)], - os: [('listdir', 1), ('mkdir', 1), - ('lstat', 1), ('symlink', 2)], - } - for (mod, funcs) in patch_funcs.items(): - for f, nargs in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, nargs, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - def patchOpen(self, new_root): - trap_func = retarget_many_wrapper(new_root, 1, open) - name = 'builtins.open' if PY3 else '__builtin__.open' - self.patched_funcs.enter_context(mock.patch(name, trap_func)) - - def patchStdoutAndStderr(self, stdout=None, stderr=None): - if stdout is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stdout', stdout)) - if stderr is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stderr', stderr)) - - def reRoot(self, root=None): - if root is None: - root = self.tmp_dir() - self.patchUtils(root) - self.patchOS(root) - return root - - -class HttprettyTestCase(CiTestCase): - # necessary as http_proxy gets in the way of httpretty - # https://github.com/gabrielfalcao/HTTPretty/issues/122 - def setUp(self): - self.restore_proxy = os.environ.get('http_proxy') - if self.restore_proxy is not None: - del os.environ['http_proxy'] - super(HttprettyTestCase, self).setUp() - - def tearDown(self): - if self.restore_proxy: - os.environ['http_proxy'] = self.restore_proxy - super(HttprettyTestCase, self).tearDown() - - -def populate_dir(path, files): - if not os.path.exists(path): - os.makedirs(path) - ret = [] - for (name, content) in files.items(): - p = os.path.sep.join([path, name]) - util.ensure_dir(os.path.dirname(p)) - with open(p, "wb") as fp: - if isinstance(content, six.binary_type): - fp.write(content) - else: - fp.write(content.encode('utf-8')) - fp.close() - ret.append(p) - - return ret - - -def dir2dict(startdir, prefix=None): - flist = {} - if prefix is None: - prefix = startdir - for root, dirs, files in os.walk(startdir): - for fname in files: - fpath = os.path.join(root, fname) - key = fpath[len(prefix):] - flist[key] = util.load_file(fpath) - return flist - - -def json_dumps(data): - # print data in nicely formatted json. - return json.dumps(data, indent=1, sort_keys=True, - separators=(',', ': ')) - - -def wrap_and_call(prefix, mocks, func, *args, **kwargs): - """ - call func(args, **kwargs) with mocks applied, then unapplies mocks - nicer to read than repeating dectorators on each function - - prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None - mocks: dictionary of names (under 'prefix') to mock and either - a return value or a dictionary to pass to the mock.patch call - func: function to call with mocks applied - *args,**kwargs: arguments for 'func' - - return_value: return from 'func' - """ - delim = '.' - if prefix is None: - prefix = '' - prefix = prefix.rstrip(delim) - unwraps = [] - for fname, kw in mocks.items(): - if prefix: - fname = delim.join((prefix, fname)) - if not isinstance(kw, dict): - kw = {'return_value': kw} - p = mock.patch(fname, **kw) - p.start() - unwraps.append(p) - try: - return func(*args, **kwargs) - finally: - for p in unwraps: - p.stop() - - -try: - skipIf = unittest.skipIf -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skipIf(condition, reason): - def decorator(func): - def wrapper(*args, **kws): - if condition: - return func(*args, **kws) - else: - print(reason, file=sys.stderr) - return wrapper - return decorator - - -# older versions of mock do not have the useful 'assert_not_called' -if not hasattr(mock.Mock, 'assert_not_called'): - def __mock_assert_not_called(mmock): - if mmock.call_count != 0: - msg = ("[citest] Expected '%s' to not have been called. " - "Called %s times." % - (mmock._mock_name or 'mock', mmock.call_count)) - raise AssertionError(msg) - mock.Mock.assert_not_called = __mock_assert_not_called - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 781f6d54..25878d7a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -12,7 +12,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase, CiTestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, CiTestCase, ExitStack, mock class FakeModule(handlers.Handler): diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py index 515919d8..0101b0e3 100644 --- a/tests/unittests/test_atomic_helper.py +++ b/tests/unittests/test_atomic_helper.py @@ -6,7 +6,7 @@ import stat from cloudinit import atomic_helper -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestAtomicHelper(CiTestCase): diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index dd9d0357..9751ed95 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -11,7 +11,7 @@ try: except ImportError: import mock -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 12f01852..495bdc9f 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -2,7 +2,7 @@ import six -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cmd import main as cli diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index b8f5031c..ee88520d 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -2,7 +2,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cs_utils import Cepko diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4ad86bb6..6d621d26 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,7 +27,7 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -from . import helpers +from cloudinit.tests import helpers INSTANCE_ID = "i-testing" diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 996560e4..82ee9714 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -5,9 +5,9 @@ import httpretty import mock import os -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceAliYun as ay +from cloudinit.tests import helpers as test_helpers DEFAULT_METADATA = { 'instance-id': 'aliyun-test-vm-00', diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 9c46abc1..3b274d90 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -18,7 +18,7 @@ import tempfile from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import cloudinit.sources.DataSourceAltCloud as dsac diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 20e70fb7..0a117771 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz from cloudinit.util import find_freebsd_part from cloudinit.util import get_path_dev_freebsd -from ..helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, SkipTest) +from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock, + ExitStack, PY26, SkipTest) import crypt import os @@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase): class TestReadAzureOvf(TestCase): + def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(dsaz.BrokenAzureDataSource, @@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase): class TestAzureNetExists(CiTestCase): + def test_azure_net_must_exist_for_legacy_objpkl(self): """DataSourceAzureNet must exist for old obj.pkl files that reference it.""" diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b2d2971b..80ce003d 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -3,7 +3,7 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import ExitStack, mock, TestCase GOAL_STATE_TEMPLATE = """\ diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 5997102c..e4c59907 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko from cloudinit import sources from cloudinit.sources import DataSourceCloudSigma -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers SERVER_CONTEXT = { "cpu": 1000, diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index e94aad61..2dc90305 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase, mock, ExitStack +from cloudinit.tests.helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 4802f105..80b9c650 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -24,7 +24,7 @@ from cloudinit.sources import ( ) from cloudinit.sources import DataSourceNone as DSNone -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers DEFAULT_LOCAL = [ Azure.DataSourceAzure, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 337be667..237c189b 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index e97a679a..f264f361 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -13,7 +13,7 @@ from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from ..helpers import mock, TestCase +from cloudinit.tests.helpers import mock, TestCase DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"] diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index b7a84e21..9fb90483 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -4,9 +4,9 @@ import copy import httpretty import mock -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceEc2 as ec2 +from cloudinit.tests import helpers as test_helpers # collected from api version 2016-09-02/ with diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index ad608bec..50e49a10 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceGCE -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers GCE_META = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index c1911bf4..289c6a40 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -8,7 +8,7 @@ import yaml from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from ..helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir try: from unittest import mock diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index ff294395..fea9156b 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir, mock, ExitStack +from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack import os import shutil diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index b0f8e435..e7d55692 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import mock, populate_dir, TestCase +from cloudinit.tests.helpers import mock, populate_dir, TestCase import os import pwd diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c2905d1a..177e9808 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -9,7 +9,7 @@ import httpretty as hp import json import re -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from six.moves.urllib.parse import urlparse from six import StringIO diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 477cf8ed..9dbf4dd9 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -6,7 +6,7 @@ import base64 -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.sources import DataSourceOVF as dsovf diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 65d83ad7..436df9ee 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -9,7 +9,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceScaleway -from ..helpers import mock, HttprettyTestCase, TestCase +from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase class DataResponses(object): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index e3c99bbb..933d5b63 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,7 +33,7 @@ import six from cloudinit import helpers as c_helpers from cloudinit.util import b64e -from ..helpers import mock, FilesystemMockingTestCase, TestCase +from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase SDC_NICS = json.loads(""" [ diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py index 3d4c9a70..a95ba3b5 100644 --- a/tests/unittests/test_distros/test_arch.py +++ b/tests/unittests/test_distros/test_arch.py @@ -3,7 +3,7 @@ from cloudinit.distros.arch import _render_network from cloudinit import util -from ..helpers import (CiTestCase, dir2dict) +from cloudinit.tests.helpers import (CiTestCase, dir2dict) from . import _get_distro diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 1d02f7bd..aa13670a 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import distros -from ..helpers import (TestCase, mock) +from cloudinit.tests.helpers import (TestCase, mock) class MyBaseDistro(distros.Distro): diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index 72d3aad6..da16a797 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -2,7 +2,7 @@ from cloudinit import distros from cloudinit import util -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) @mock.patch("cloudinit.distros.debian.util.subp") diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index b355a19e..791fe612 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -3,7 +3,7 @@ from cloudinit import distros from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import os import shutil diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 6d89dba8..c4bd11bc 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -12,7 +12,7 @@ try: except ImportError: from contextlib2 import ExitStack -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py index bdb1d633..b9bb9b3e 100644 --- a/tests/unittests/test_distros/test_opensuse.py +++ b/tests/unittests/test_distros/test_opensuse.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 97168cf9..68ea0083 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -3,7 +3,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit.distros import rhel_util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import re import tempfile diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py index c656aacc..33e3c457 100644 --- a/tests/unittests/test_distros/test_sles.py +++ b/tests/unittests/test_distros/test_sles.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index 235ecebb..c1d5b693 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -4,7 +4,7 @@ import re from cloudinit.distros.parsers.sys_conf import SysConf -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase # Lots of good examples @ diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 88746e0a..0fa9cdb5 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -5,7 +5,7 @@ from cloudinit.distros import ug_util from cloudinit import helpers from cloudinit import settings -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import mock diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 8ccfe55c..1a81a89e 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -6,7 +6,8 @@ from uuid import uuid4 from cloudinit import safeyaml from cloudinit import util -from .helpers import CiTestCase, dir2dict, json_dumps, populate_dir +from cloudinit.tests.helpers import ( + CiTestCase, dir2dict, json_dumps, populate_dir) UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 65fdb519..af78997f 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -2,7 +2,7 @@ import httpretty as hp -from . import helpers +from cloudinit.tests import helpers from cloudinit import ec2_utils as eu from cloudinit import url_helper as uh diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 13137f6d..6364d38e 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -2,7 +2,7 @@ import copy -from .. import helpers +from cloudinit.tests import helpers from six.moves import filterfalse diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 554277ff..83f962a9 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import copy import os diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index f53ddbb2..d2b96f0b 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 1ca915b4..f7608c28 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 12502d05..3a3f95ca 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -20,7 +20,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import gpg from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 292d3f59..7bb1b7c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -28,7 +28,7 @@ from cloudinit import util from cloudinit.config import cc_apt_configure from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 7cee2c3f..06e14db0 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -5,7 +5,7 @@ from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 6a152ea2..e5785cfd 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -14,7 +14,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index 1873c3e1..787ba350 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -11,7 +11,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 8a6d49ed..5afcacaf 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -3,7 +3,7 @@ import random from cloudinit.config import cc_disk_setup -from ..helpers import CiTestCase, ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, TestCase class TestIsDiskUsed(TestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index c5fc8c9b..a3e46351 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -4,7 +4,7 @@ from cloudinit import cloud from cloudinit.config import cc_growpart from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import errno import logging diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py index 7c247fa9..db92a7e2 100644 --- a/tests/unittests/test_handler/test_handler_landscape.py +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -1,9 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.config import cc_landscape -from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call +from cloudinit.sources import DataSourceNone +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock, + wrap_and_call) from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index a789db32..e29a06f9 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 351226bf..f132a778 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_lxd from cloudinit.sources import DataSourceNoCloud from cloudinit import (distros, helpers, cloud) -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py index 2a9f3823..7eec7352 100644 --- a/tests/unittests/test_handler/test_handler_mcollective.py +++ b/tests/unittests/test_handler/test_handler_mcollective.py @@ -4,7 +4,7 @@ from cloudinit import (cloud, distros, helpers, util) from cloudinit.config import cc_mcollective from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import configobj import logging diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 650ca0ec..fe492d4b 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -6,7 +6,7 @@ import tempfile from cloudinit.config import cc_mounts -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers try: from unittest import mock diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 83d5faa2..4f291248 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_ntp from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf import os diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index e382210d..85a0fe0a 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,8 +4,8 @@ import sys from cloudinit.config import cc_power_state_change as psc -from .. import helpers as t_help -from ..helpers import mock +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 805c76ba..0b6e3b58 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_puppet from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import CiTestCase, mock +from cloudinit.tests.helpers import CiTestCase, mock import logging diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index cca06678..8c8e2838 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -9,7 +9,7 @@ from cloudinit.config.cc_rsyslog import ( parse_remotes_line, remotes_to_rsyslog_cfg) from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help class TestLoadConfig(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 7880ee72..374c1d31 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_runcmd from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf import logging import os diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index e5e607fb..f60dedc2 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -22,7 +22,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 8165bf9a..abdc17e7 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -7,7 +7,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index e4d07622..76b79c29 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -7,9 +7,9 @@ from cloudinit.config.cc_snap_config import ( from cloudinit import (distros, helpers, cloud, util) from cloudinit.config.cc_snap_config import handle as snap_handle from cloudinit.sources import DataSourceNone -from ..helpers import FilesystemMockingTestCase, mock +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging import os diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index 28b5892a..ddbf4a79 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_spacewalk from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import logging diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index c30fbdfe..27eedded 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index 1129e77d..7fa8fd21 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -4,7 +4,7 @@ from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import CiTestCase, FilesystemMockingTestCase +from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index c4396df5..b7adbe50 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_yum_add_repo from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers try: from configparser import ConfigParser diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 640f11d4..6137e3cf 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -6,7 +6,7 @@ from cloudinit.config.schema import ( validate_cloudconfig_schema, main) from cloudinit.util import write_file -from ..helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf from copy import copy from six import StringIO diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py index f1979e89..2e4582a0 100644 --- a/tests/unittests/test_helpers.py +++ b/tests/unittests/test_helpers.py @@ -4,7 +4,7 @@ import os -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import sources diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 68fb4b8d..cd6296d6 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -2,9 +2,9 @@ """Tests for cloudinit.log """ -from .helpers import CiTestCase from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT from cloudinit import log as ci_logging +from cloudinit.tests.helpers import CiTestCase import datetime import logging import six diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 0658b6b4..f51358da 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from . import helpers +from cloudinit.tests import helpers from cloudinit.handlers import cloud_config from cloudinit.handlers import (CONTENT_START, CONTENT_END) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f251024b..c10ef905 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -11,10 +11,10 @@ from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack from cloudinit import util -from .helpers import CiTestCase -from .helpers import dir2dict -from .helpers import mock -from .helpers import populate_dir +from cloudinit.tests.helpers import CiTestCase +from cloudinit.tests.helpers import dir2dict +from cloudinit.tests.helpers import mock +from cloudinit.tests.helpers import populate_dir import base64 import copy diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index a4ae284f..abbb29b8 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -2,7 +2,7 @@ from cloudinit import util -from .helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir import shutil import tempfile diff --git a/tests/unittests/test_registry.py b/tests/unittests/test_registry.py index acf0bf4f..2b625026 100644 --- a/tests/unittests/test_registry.py +++ b/tests/unittests/test_registry.py @@ -2,7 +2,7 @@ from cloudinit.registry import DictRegistry -from .helpers import (mock, TestCase) +from cloudinit.tests.helpers import (mock, TestCase) class TestDictRegistry(TestCase): diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index f3b8f992..571420ed 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -8,7 +8,7 @@ from cloudinit.reporting import handlers import mock -from .helpers import TestCase +from cloudinit.tests.helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index ca14cd46..e9d5702a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -7,7 +7,7 @@ import logging from cloudinit.config import cc_rh_subscription from cloudinit import util -from .helpers import TestCase, mock +from cloudinit.tests.helpers import TestCase, mock class GoodTests(TestCase): diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 65895273..add93653 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 55f15b55..5cf666fe 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 991f45a6..2a8e6abe 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -2,8 +2,8 @@ from mock import patch -from . import helpers as test_helpers from cloudinit import ssh_util +from cloudinit.tests import helpers as test_helpers VALID_CONTENT = { @@ -57,6 +57,7 @@ TEST_OPTIONS = ( class TestAuthKeyLineParser(test_helpers.TestCase): + def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 4e627826..b911d929 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -6,7 +6,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers import textwrap from cloudinit import templater diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 5f11c88f..3e4154ca 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -12,7 +12,7 @@ import six import yaml from cloudinit import importer, util -from . import helpers +from cloudinit.tests import helpers try: from unittest import mock diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py index 1662ce09..d012f69d 100644 --- a/tests/unittests/test_version.py +++ b/tests/unittests/test_version.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from cloudinit import version diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 03b36d31..d8651077 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,10 +8,10 @@ import logging import sys -from .helpers import CiTestCase from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) logger = logging.getLogger(__name__) -- cgit v1.2.3 From 5582e4a266118b63ff86b6258b23d66df6d129d5 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 6 Sep 2017 12:11:13 -0600 Subject: tests: mock missed openstack metadata uri network_data.json This missed mock in test_openstack resulted in a costly unit test timeout. LP: #1714376 --- tests/unittests/test_datasource/test_openstack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 177e9808..ed367e05 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -57,6 +57,8 @@ OS_FILES = { 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/network_data.json': json.dumps( + {'links': [], 'networks': [], 'services': []}), 'openstack/latest/user_data': USER_DATA, 'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA), } @@ -68,6 +70,7 @@ EC2_VERSIONS = [ ] +# TODO _register_uris should leverage test_ec2.register_mock_metaserver. def _register_uris(version, ec2_files, ec2_meta, os_files): """Registers a set of url patterns into httpretty that will mimic the same data returned by the openstack metadata service (and ec2 service).""" -- cgit v1.2.3 From dcbb901cc3e9e888bc8f87e87bdc0ca8436a2baa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 5 Sep 2017 16:57:10 -0400 Subject: ds-identify: Make OpenStack return maybe on arch other than intel. OpenStack Nova identifies itself only to Intel guests. Make ds-identify return 'MAYBE' for OpenStack on non-intel arches. An unnecessary change here is to rename the 'policy_nodmi' kwarg to 'policy_no_dmi' in the related unit tests. LP: #1715241 --- tests/unittests/test_ds_identify.py | 45 ++++++++++++++++++++++++++++++++++--- tools/ds-identify | 6 +++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 1a81a89e..92454d7c 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -11,6 +11,10 @@ from cloudinit.tests.helpers import ( UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") +UNAME_PPC64EL = ("Linux diamond 4.4.0-83-generic #106-Ubuntu SMP " + "Mon Jun 26 17:53:54 UTC 2017 " + "ppc64le ppc64le ppc64le GNU/Linux") + BLKID_EFI_ROOT = """ DEVNAME=/dev/sda1 UUID=8B36-5390 @@ -23,6 +27,8 @@ TYPE=ext4 PARTUUID=30c65c77-e07d-4039-b2fb-88b1fb5fa1fc """ +POLICY_FOUND_ONLY = "search,found=all,maybe=none,notfound=disabled" +POLICY_FOUND_OR_MAYBE = "search,found=all,maybe=all,notfound=disabled" DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=enabled" DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=disabled" @@ -48,6 +54,7 @@ P_SEED_DIR = "var/lib/cloud/seed" P_DSID_CFG = "etc/cloud/ds-identify.cfg" MOCK_VIRT_IS_KVM = {'name': 'detect_virt', 'RET': 'kvm', 'ret': 0} +MOCK_UNAME_IS_PPC64 = {'name': 'uname', 'out': UNAME_PPC64EL, 'ret': 0} class TestDsIdentify(CiTestCase): @@ -55,7 +62,7 @@ class TestDsIdentify(CiTestCase): def call(self, rootd=None, mocks=None, args=None, files=None, policy_dmi=DI_DEFAULT_POLICY, - policy_nodmi=DI_DEFAULT_POLICY_NO_DMI): + policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI): if args is None: args = [] if mocks is None: @@ -81,7 +88,7 @@ class TestDsIdentify(CiTestCase): "PATH_ROOT='%s'" % rootd, ". " + self.dsid_path, 'DI_DEFAULT_POLICY="%s"' % policy_dmi, - 'DI_DEFAULT_POLICY_NO_DMI="%s"' % policy_nodmi, + 'DI_DEFAULT_POLICY_NO_DMI="%s"' % policy_no_dmi, "" ] @@ -137,7 +144,7 @@ class TestDsIdentify(CiTestCase): def _call_via_dict(self, data, rootd=None, **kwargs): # return output of self.call with a dict input like VALID_CFG[item] xwargs = {'rootd': rootd} - for k in ('mocks', 'args', 'policy_dmi', 'policy_nodmi', 'files'): + for k in ('mocks', 'args', 'policy_dmi', 'policy_no_dmi', 'files'): if k in data: xwargs[k] = data[k] if k in kwargs: @@ -261,6 +268,31 @@ class TestDsIdentify(CiTestCase): self._check_via_dict(mydata, rc=RC_FOUND, dslist=['AliYun', DS_NONE], policy_dmi=policy) + def test_default_openstack_intel_is_found(self): + """On Intel, openstack must be identified.""" + self._test_ds_found('OpenStack') + + def test_openstack_on_non_intel_is_maybe(self): + """On non-Intel, openstack without dmi info is maybe. + + nova does not identify itself on platforms other than intel. + https://bugs.launchpad.net/cloud-init/+bugs?field.tag=dsid-nova""" + + data = VALID_CFG['OpenStack'].copy() + del data['files'][P_PRODUCT_NAME] + data.update({'policy_dmi': POLICY_FOUND_OR_MAYBE, + 'policy_no_dmi': POLICY_FOUND_OR_MAYBE}) + + # this should show not found as default uname in tests is intel. + # and intel openstack requires positive identification. + self._check_via_dict(data, RC_NOT_FOUND, dslist=None) + + # updating the uname to ppc64 though should get a maybe. + data.update({'mocks': [MOCK_VIRT_IS_KVM, MOCK_UNAME_IS_PPC64]}) + (_, _, err, _, _) = self._check_via_dict( + data, RC_FOUND, dslist=['OpenStack', 'None']) + self.assertIn("check for 'OpenStack' returned maybe", err) + def blkid_out(disks=None): """Convert a list of disk dictionaries into blkid content.""" @@ -341,6 +373,13 @@ VALID_CFG = { 'files': {P_PRODUCT_SERIAL: 'GoogleCloud-8f2e88f\n'}, 'mocks': [MOCK_VIRT_IS_KVM], }, + 'OpenStack': { + 'ds': 'OpenStack', + 'files': {P_PRODUCT_NAME: 'OpenStack Nova\n'}, + 'mocks': [MOCK_VIRT_IS_KVM], + 'policy_dmi': POLICY_FOUND_ONLY, + 'policy_no_dmi': POLICY_FOUND_ONLY, + }, 'ConfigDrive': { 'ds': 'ConfigDrive', 'mocks': [ diff --git a/tools/ds-identify b/tools/ds-identify index 33bd2991..ee5e05a4 100755 --- a/tools/ds-identify +++ b/tools/ds-identify @@ -833,6 +833,12 @@ dscheck_OpenStack() { return ${DS_FOUND} fi + # LP: #1715241 : arch other than intel are not identified properly. + case "$DI_UNAME_MACHINE" in + i?86|x86_64) :;; + *) return ${DS_MAYBE};; + esac + return ${DS_NOT_FOUND} } -- cgit v1.2.3 From 409918f9ba83e45e9bc5cc0b6c589e2fc8ae9b60 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 29 Aug 2017 09:59:20 -0400 Subject: Use /run/cloud-init for tempfile operations. During boot, the usage of /tmp is not safe. In systemd systems, systemd-tmpfiles-clean may run at any point and clear out a temp file while cloud-init is using it. The solution here is to use /run/cloud-init/tmp. LP: #1707222 --- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_snappy.py | 4 +- cloudinit/net/dhcp.py | 3 +- cloudinit/sources/helpers/azure.py | 4 +- cloudinit/temp_utils.py | 93 ++++++++++++++++++++++ cloudinit/util.py | 36 +-------- packages/bddeb | 5 +- .../unittests/test_datasource/test_azure_helper.py | 4 +- tests/unittests/test_net.py | 3 +- 10 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 cloudinit/temp_utils.py diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 604f93b0..9c0476af 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -37,6 +37,7 @@ specified either as lists or strings. For invocation details, see ``runcmd``. import os from cloudinit.settings import PER_ALWAYS +from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS @@ -49,7 +50,7 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return - with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: + with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 02c70b10..c192dd32 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -71,6 +71,7 @@ import itertools import json import os +from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -303,7 +304,7 @@ def install_chef(cloud, chef_cfg, log): "omnibus_url_retries", default=OMNIBUS_URL_RETRIES)) content = url_helper.readurl(url=url, retries=retries).contents - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd util.write_file(tmpf, content, mode=0o700) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a9682f19..eecb8178 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,11 +63,11 @@ is ``auto``. Options are: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import temp_utils from cloudinit import util import glob import os -import tempfile LOG = logging.getLogger(__name__) @@ -183,7 +183,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # config # Note, however, we do not touch config files on disk. nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = tempfile.mkstemp() + (fd, cfg_tmpf) = temp_utils.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c7febc57..c842c839 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -9,6 +9,7 @@ import os import re from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import temp_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: return dhcp_discovery(dhclient_path, nic, tmpdir) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index e22409d1..28ed0ae2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -6,10 +6,10 @@ import os import re import socket import struct -import tempfile import time from cloudinit import stages +from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree @@ -111,7 +111,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.mkdtemp() + self.tmpdir = temp_utils.mkdtemp() self.certificate = None self.generate_certificate() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py new file mode 100644 index 00000000..0355f19d --- /dev/null +++ b/cloudinit/temp_utils.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import contextlib +import errno +import os +import shutil +import tempfile + +_TMPDIR = None +_ROOT_TMPDIR = "/run/cloud-init/tmp" + + +def _tempfile_dir_arg(odir=None): + """Return the proper 'dir' argument for tempfile functions. + + When root, cloud-init will use /run/cloud-init/tmp to avoid + any cleaning that a distro boot might do on /tmp (such as + systemd-tmpfiles-clean). + + If the caller of this function (mkdtemp or mkstemp) was provided + with a 'dir' argument, then that is respected. + + @param odir: original 'dir' arg to 'mkdtemp' or other.""" + + if odir is not None: + return odir + + global _TMPDIR + if _TMPDIR: + return _TMPDIR + + if os.getuid() == 0: + tdir = _ROOT_TMPDIR + else: + tdir = os.environ.get('TMPDIR', '/tmp') + if not os.path.isdir(tdir): + os.makedirs(tdir) + os.chmod(tdir, 0o1777) + + _TMPDIR = tdir + return tdir + + +def ExtendedTemporaryFile(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + + def _unlink_if_exists(path): + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + + fh.unlink = _unlink_if_exists + + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + + setattr(fh, 'unlink_now', unlink_now) + return fh + + +@contextlib.contextmanager +def tempdir(**kwargs): + # This seems like it was only added in python 3.2 + # Make it since its useful... + # See: http://bugs.python.org/file12970/tempdir.patch + tdir = mkdtemp(**kwargs) + try: + yield tdir + finally: + shutil.rmtree(tdir) + + +def mkdtemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkdtemp(**kwargs) + + +def mkstemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkstemp(**kwargs) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 609e94c8..ae5cda8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -30,7 +30,6 @@ import stat import string import subprocess import sys -import tempfile import time from errno import ENOENT, ENOEXEC @@ -45,6 +44,7 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import mergers from cloudinit import safeyaml +from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import url_helper from cloudinit import version @@ -349,26 +349,6 @@ class DecompressionError(Exception): pass -def ExtendedTemporaryFile(**kwargs): - fh = tempfile.NamedTemporaryFile(**kwargs) - # Replace its unlink with a quiet version - # that does not raise errors when the - # file to unlink has been unlinked elsewhere.. - LOG.debug("Created temporary file %s", fh.name) - fh.unlink = del_file - - # Add a new method that will unlink - # right 'now' but still lets the exit - # method attempt to remove it (which will - # not throw due to our del file being quiet - # about files that are not there) - def unlink_now(): - fh.unlink(fh.name) - - setattr(fh, 'unlink_now', unlink_now) - return fh - - def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: @@ -790,18 +770,6 @@ def umask(n_msk): os.umask(old) -@contextlib.contextmanager -def tempdir(**kwargs): - # This seems like it was only added in python 3.2 - # Make it since its useful... - # See: http://bugs.python.org/file12970/tempdir.patch - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - del_dir(tdir) - - def center(text, fill, max_len): return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len) @@ -1587,7 +1555,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): mtypes = [''] mounted = mounts() - with tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: umount = False if os.path.realpath(device) in mounted: mountpoint = mounted[os.path.realpath(device)]['mountpoint'] diff --git a/packages/bddeb b/packages/bddeb index 7c123548..4f2e2ddf 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -21,8 +21,9 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import templater from cloudinit import util + from cloudinit import temp_utils + from cloudinit import templater DEBUILD_ARGS = ["-S", "-d"] @@ -148,7 +149,7 @@ def main(): capture = False templ_data = {'debian_release': args.release} - with util.tempdir() as tdir: + with temp_utils.tempdir() as tdir: # output like 0.7.6-1022-g36e92d3 ver_data = read_version() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 80ce003d..44b99eca 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -275,7 +275,7 @@ class TestOpenSSLManager(TestCase): mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp') def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() self.assertEqual(mkdtemp.return_value, manager.tmpdir) @@ -292,7 +292,7 @@ class TestOpenSSLManager(TestCase): manager.clean_up() @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock()) @mock.patch.object(azure_helper.util, 'del_dir') def test_clean_up(self, del_dir): manager = azure_helper.OpenSSLManager() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c10ef905..f2496151 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -9,6 +9,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack +from cloudinit import temp_utils from cloudinit import util from cloudinit.tests.helpers import CiTestCase @@ -2150,7 +2151,7 @@ class TestCmdlineConfigParsing(CiTestCase): static['mac_address'] = macs['eth1'] expected = {'version': 1, 'config': [dhcp, static]} - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: for fname, content in pairs: fp = os.path.join(tmpd, fname) files.append(fp) -- cgit v1.2.3 From 922c3c5c1a86f2d58e95a328e72b49a3bb234ca8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Sep 2017 09:59:47 -0400 Subject: Ec2: only attempt to operate at local mode on known platforms. This change makes the DataSourceEc2Local do nothing unless it is on actual AWS platform. The motivation is twofold: a.) It is generally safer to only make this function available to Ec2 clones that explicitly identify themselves to the guest. (It also gives them a reason to supply identification code to cloud-init.) b.) On non-intel OpenStack platforms ds-identify would enable both the Ec2 and OpenStack sources. That is because there is not good data (such as dmi) to positively identify the platform. Previously that would be fine as OpenStack would run first and be successful. The change to add Ec2Local meant that an Ec2 now runs first. The best case for 'b' would be a slow down as attempts at the Ec2 metadata service time out. The discovered case was worse. Additionally we add a simple check for datatype of 'network' in the metadata before attempting to read it. LP: #1715128 --- cloudinit/sources/DataSourceEc2.py | 43 +++++++++++++++++++++++------ tests/unittests/test_datasource/test_ec2.py | 29 ++++++++++++++++--- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 07c12bb4..41367a8b 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -27,6 +27,8 @@ SKIP_METADATA_URL_CODES = frozenset([uhelp.NOT_FOUND]) STRICT_ID_PATH = ("datasource", "Ec2", "strict_id") STRICT_ID_DEFAULT = "warn" +_unset = "_unset" + class Platforms(object): ALIYUN = "AliYun" @@ -57,7 +59,7 @@ class DataSourceEc2(sources.DataSource): _cloud_platform = None - _network_config = None # Used for caching calculated network config v1 + _network_config = _unset # Used for caching calculated network config v1 # Whether we want to get network configuration from the metadata service. get_network_metadata = False @@ -284,10 +286,24 @@ class DataSourceEc2(sources.DataSource): @property def network_config(self): """Return a network config dict for rendering ENI or netplan files.""" - if self._network_config is None: - if self.metadata is not None: - self._network_config = convert_ec2_metadata_network_config( - self.metadata) + if self._network_config != _unset: + return self._network_config + + if self.metadata is None: + # this would happen if get_data hadn't been called. leave as _unset + LOG.warning( + "Unexpected call to network_config when metadata is None.") + return None + + result = None + net_md = self.metadata.get('network') + if isinstance(net_md, dict): + result = convert_ec2_metadata_network_config(net_md) + else: + LOG.warning("unexpected metadata 'network' key not valid: %s", + net_md) + self._network_config = result + return self._network_config def _crawl_metadata(self): @@ -321,6 +337,14 @@ class DataSourceEc2Local(DataSourceEc2): """ get_network_metadata = True # Get metadata network config if present + def get_data(self): + supported_platforms = (Platforms.AWS,) + if self.cloud_platform not in supported_platforms: + LOG.debug("Local Ec2 mode only supported on %s, not %s", + supported_platforms, self.cloud_platform) + return False + return super(DataSourceEc2Local, self).get_data() + def read_strict_mode(cfgval, default): try: @@ -434,10 +458,13 @@ def _collect_platform_data(): return data -def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None): +def convert_ec2_metadata_network_config(network_md, macs_to_nics=None): """Convert ec2 metadata to network config version 1 data dict. - @param: metadata: Dictionary of metadata crawled from EC2 metadata url. + @param: network_md: 'network' portion of EC2 metadata. + generally formed as {"interfaces": {"macs": {}} where + 'macs' is a dictionary with mac address as key and contents like: + {"device-number": "0", "interface-id": "...", "local-ipv4s": ...} @param: macs_to_name: Optional dict mac addresses and the nic name. If not provided, get_interfaces_by_mac is called to get it from the OS. @@ -446,7 +473,7 @@ def convert_ec2_metadata_network_config(metadata=None, macs_to_nics=None): netcfg = {'version': 1, 'config': []} if not macs_to_nics: macs_to_nics = net.get_interfaces_by_mac() - macs_metadata = metadata['network']['interfaces']['macs'] + macs_metadata = network_md['interfaces']['macs'] for mac, nic_name in macs_to_nics.items(): nic_metadata = macs_metadata.get(mac) if not nic_metadata: diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 9fb90483..a7301dbf 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -279,6 +279,27 @@ class TestEc2(test_helpers.HttprettyTestCase): ret = ds.get_data() self.assertTrue(ret) + def test_ec2_local_returns_false_on_non_aws(self): + """DataSourceEc2Local returns False when platform is not AWS.""" + self.datasource = ec2.DataSourceEc2Local + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': False}}}, + md=DEFAULT_METADATA) + platform_attrs = [ + attr for attr in ec2.Platforms.__dict__.keys() + if not attr.startswith('__')] + for attr_name in platform_attrs: + platform_name = getattr(ec2.Platforms, attr_name) + if platform_name != 'AWS': + ds._cloud_platform = platform_name + ret = ds.get_data() + self.assertFalse(ret) + message = ( + "Local Ec2 mode only supported on ('AWS',)," + ' not {0}'.format(platform_name)) + self.assertIn(message, self.logs.getvalue()) + @httpretty.activate @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD') def test_ec2_local_returns_false_on_bsd(self, m_is_freebsd): @@ -336,8 +357,8 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): super(TestConvertEc2MetadataNetworkConfig, self).setUp() self.mac1 = '06:17:04:d7:26:09' self.network_metadata = { - 'network': {'interfaces': {'macs': { - self.mac1: {'public-ipv4s': '172.31.2.16'}}}}} + 'interfaces': {'macs': { + self.mac1: {'public-ipv4s': '172.31.2.16'}}}} def test_convert_ec2_metadata_network_config_skips_absent_macs(self): """Any mac absent from metadata is skipped by network config.""" @@ -357,7 +378,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): macs_to_nics = {self.mac1: 'eth9'} network_metadata_ipv6 = copy.deepcopy(self.network_metadata) nic1_metadata = ( - network_metadata_ipv6['network']['interfaces']['macs'][self.mac1]) + network_metadata_ipv6['interfaces']['macs'][self.mac1]) nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' nic1_metadata.pop('public-ipv4s') expected = {'version': 1, 'config': [ @@ -373,7 +394,7 @@ class TestConvertEc2MetadataNetworkConfig(test_helpers.CiTestCase): macs_to_nics = {self.mac1: 'eth9'} network_metadata_both = copy.deepcopy(self.network_metadata) nic1_metadata = ( - network_metadata_both['network']['interfaces']['macs'][self.mac1]) + network_metadata_both['interfaces']['macs'][self.mac1]) nic1_metadata['ipv6s'] = '2620:0:1009:fd00:e442:c88d:c04d:dc85/64' expected = {'version': 1, 'config': [ {'mac_address': self.mac1, 'type': 'physical', -- cgit v1.2.3 From a1dfdda2a2ae20fe026881980ddf7d16110f06e2 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Thu, 7 Sep 2017 22:16:16 -0600 Subject: vmware customization: return network config format For customizing the machines hosted on 'VMWare' hypervisor, the datasource should return the 'network config' data in 'curtin' format. This branch also fixes /etc/network/interfaces replacing the line "source /etc/network/interfaces.d/*.cfg" which is incorrectly removed when VMWare's Perl Customization Engine writes /etc/network/interfaces. Modify the code to read the customization configuration and return the converted data. Added few tests. LP: #1675063 --- cloudinit/sources/DataSourceOVF.py | 91 ++++++--- cloudinit/sources/helpers/vmware/imc/config_nic.py | 201 ++++++++++++------- .../sources/helpers/vmware/imc/guestcust_util.py | 12 +- tests/unittests/test_vmware_config_file.py | 217 +++++++++++++++++++++ 4 files changed, 418 insertions(+), 103 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 73d38771..aa5f798d 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -51,6 +51,10 @@ class DataSourceOVF(sources.DataSource): self.cfg = {} self.supported_seed_starts = ("/", "file://") self.vmware_customization_supported = True + self._network_config = None + self._vmware_nics_to_enable = None + self._vmware_cust_conf = None + self._vmware_cust_found = False def __str__(self): root = sources.DataSource.__str__(self) @@ -60,8 +64,8 @@ class DataSourceOVF(sources.DataSource): found = [] md = {} ud = "" - vmwarePlatformFound = False - vmwareImcConfigFilePath = '' + vmwareImcConfigFilePath = None + nicspath = None defaults = { "instance-id": "iid-dsovf", @@ -101,25 +105,26 @@ class DataSourceOVF(sources.DataSource): logfunc=LOG.debug, msg="waiting for configuration file", func=wait_for_imc_cfg_file, - args=("/var/run/vmware-imc", "cust.cfg", max_wait)) + args=("cust.cfg", max_wait)) if vmwareImcConfigFilePath: LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) + nicspath = wait_for_imc_cfg_file( + filename="nics.txt", maxwait=10, naplen=5) else: LOG.debug("Did not find VMware Customization Config File") else: LOG.debug("Customization for VMware platform is disabled.") if vmwareImcConfigFilePath: - nics = "" + self._vmware_nics_to_enable = "" try: cf = ConfigFile(vmwareImcConfigFilePath) - conf = Config(cf) - (md, ud, cfg) = read_vmware_imc(conf) - dirpath = os.path.dirname(vmwareImcConfigFilePath) - nics = get_nics_to_enable(dirpath) - markerid = conf.marker_id + self._vmware_cust_conf = Config(cf) + (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) + self._vmware_nics_to_enable = get_nics_to_enable(nicspath) + markerid = self._vmware_cust_conf.marker_id markerexists = check_marker_exists(markerid) except Exception as e: LOG.debug("Error parsing the customization Config File") @@ -127,28 +132,29 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) - return False + raise e finally: util.del_dir(os.path.dirname(vmwareImcConfigFilePath)) try: - LOG.debug("Applying the Network customization") - nicConfigurator = NicConfigurator(conf.nics) - nicConfigurator.configure() + LOG.debug("Preparing the Network configuration") + self._network_config = get_network_config_from_conf( + self._vmware_cust_conf, + True, + True, + self.distro.osfamily) except Exception as e: - LOG.debug("Error applying the Network Configuration") LOG.exception(e) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) - enable_nics(nics) - return False + raise e + if markerid and not markerexists: LOG.debug("Applying password customization") pwdConfigurator = PasswordConfigurator() - adminpwd = conf.admin_password + adminpwd = self._vmware_cust_conf.admin_password try: - resetpwd = conf.reset_password + resetpwd = self._vmware_cust_conf.reset_password if adminpwd or resetpwd: pwdConfigurator.configure(adminpwd, resetpwd, self.distro) @@ -159,7 +165,6 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) return False if markerid: LOG.debug("Handle marker creation") @@ -170,14 +175,18 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) return False - vmwarePlatformFound = True + self._vmware_cust_found = True + found.append('vmware-tools') + + # TODO: Need to set the status to DONE only when the + # customization is done successfully. set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) - enable_nics(nics) + enable_nics(self._vmware_nics_to_enable) + else: np = {'iso': transport_iso9660, 'vmware-guestd': transport_vmware_guestd, } @@ -192,7 +201,7 @@ class DataSourceOVF(sources.DataSource): found.append(name) # There was no OVF transports found - if len(found) == 0 and not vmwarePlatformFound: + if len(found) == 0: return False if 'seedfrom' in md and md['seedfrom']: @@ -237,6 +246,10 @@ class DataSourceOVF(sources.DataSource): def get_config_obj(self): return self.cfg + @property + def network_config(self): + return self._network_config + class DataSourceOVFNet(DataSourceOVF): def __init__(self, sys_cfg, distro, paths): @@ -268,12 +281,13 @@ def get_max_wait_from_cfg(cfg): return max_wait -def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): +def wait_for_imc_cfg_file(filename, maxwait=180, naplen=5, + dirpath="/var/run/vmware-imc"): waited = 0 while waited < maxwait: - fileFullPath = search_file(dirpath, filename) - if fileFullPath: + fileFullPath = os.path.join(dirpath, filename) + if os.path.isfile(fileFullPath): return fileFullPath LOG.debug("Waiting for VMware Customization Config File") time.sleep(naplen) @@ -281,6 +295,26 @@ def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): return None +def get_network_config_from_conf(config, use_system_devices=True, + configure=False, osfamily=None): + nicConfigurator = NicConfigurator(config.nics, use_system_devices) + nics_cfg_list = nicConfigurator.generate(configure, osfamily) + + return get_network_config(nics_cfg_list, + config.name_servers, + config.dns_suffixes) + + +def get_network_config(nics=None, nameservers=None, search=None): + config_list = nics + + if nameservers or search: + config_list.append({'type': 'nameserver', 'address': nameservers, + 'search': search}) + + return {'version': 1, 'config': config_list} + + # This will return a dict with some content # meta-data, user-data, some config def read_vmware_imc(config): @@ -296,6 +330,9 @@ def read_vmware_imc(config): if config.timezone: cfg['timezone'] = config.timezone + # Generate a unique instance-id so that re-customization will + # happen in cloud-init + md['instance-id'] = "iid-vmware-" + util.rand_str(strlen=8) return (md, ud, cfg) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 67ac21db..2fb07c59 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -9,22 +9,48 @@ import logging import os import re +from cloudinit.net.network_state import mask_to_net_prefix from cloudinit import util logger = logging.getLogger(__name__) +def gen_subnet(ip, netmask): + """ + Return the subnet for a given ip address and a netmask + @return (str): the subnet + @param ip: ip address + @param netmask: netmask + """ + ip_array = ip.split(".") + mask_array = netmask.split(".") + result = [] + for index in list(range(4)): + result.append(int(ip_array[index]) & int(mask_array[index])) + + return ".".join([str(x) for x in result]) + + class NicConfigurator(object): - def __init__(self, nics): + def __init__(self, nics, use_system_devices=True): """ Initialize the Nic Configurator @param nics (list) an array of nics to configure + @param use_system_devices (Bool) Get the MAC names from the system + if this is True. If False, then mac names will be retrieved from + the specified nics. """ self.nics = nics self.mac2Name = {} self.ipv4PrimaryGateway = None self.ipv6PrimaryGateway = None - self.find_devices() + + if use_system_devices: + self.find_devices() + else: + for nic in self.nics: + self.mac2Name[nic.mac.lower()] = nic.name + self._primaryNic = self.get_primary_nic() def get_primary_nic(self): @@ -61,138 +87,163 @@ class NicConfigurator(object): def gen_one_nic(self, nic): """ - Return the lines needed to configure a nic - @return (str list): the string list to configure the nic + Return the config list needed to configure a nic + @return (list): the subnets and routes list to configure the nic @param nic (NicBase): the nic to configure """ - lines = [] - name = self.mac2Name.get(nic.mac.lower()) + mac = nic.mac.lower() + name = self.mac2Name.get(mac) if not name: raise ValueError('No known device has MACADDR: %s' % nic.mac) - if nic.onboot: - lines.append('auto %s' % name) + nics_cfg_list = [] + + cfg = {'type': 'physical', 'name': name, 'mac_address': mac} + + subnet_list = [] + route_list = [] # Customize IPv4 - lines.extend(self.gen_ipv4(name, nic)) + (subnets, routes) = self.gen_ipv4(name, nic) + subnet_list.extend(subnets) + route_list.extend(routes) # Customize IPv6 - lines.extend(self.gen_ipv6(name, nic)) + (subnets, routes) = self.gen_ipv6(name, nic) + subnet_list.extend(subnets) + route_list.extend(routes) + + cfg.update({'subnets': subnet_list}) - lines.append('') + nics_cfg_list.append(cfg) + if route_list: + nics_cfg_list.extend(route_list) - return lines + return nics_cfg_list def gen_ipv4(self, name, nic): """ - Return the lines needed to configure the IPv4 setting of a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic + Return the set of subnets and routes needed to configure the + IPv4 settings of a nic + @return (set): the set of subnet and routes to configure the gateways + @param name (str): subnet and route list for the nic @param nic (NicBase): the nic to configure """ - lines = [] + + subnet = {} + route_list = [] + + if nic.onboot: + subnet.update({'control': 'auto'}) bootproto = nic.bootProto.lower() if nic.ipv4_mode.lower() == 'disabled': bootproto = 'manual' - lines.append('iface %s inet %s' % (name, bootproto)) if bootproto != 'static': - return lines + subnet.update({'type': 'dhcp'}) + return ([subnet], route_list) + else: + subnet.update({'type': 'static'}) # Static Ipv4 addrs = nic.staticIpv4 if not addrs: - return lines + return ([subnet], route_list) v4 = addrs[0] if v4.ip: - lines.append(' address %s' % v4.ip) + subnet.update({'address': v4.ip}) if v4.netmask: - lines.append(' netmask %s' % v4.netmask) + subnet.update({'netmask': v4.netmask}) # Add the primary gateway if nic.primary and v4.gateways: self.ipv4PrimaryGateway = v4.gateways[0] - lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) - return lines + subnet.update({'gateway': self.ipv4PrimaryGateway}) + return [subnet] # Add routes if there is no primary nic if not self._primaryNic: - lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + route_list.extend(self.gen_ipv4_route(nic, + v4.gateways, + v4.netmask)) - return lines + return ([subnet], route_list) - def gen_ipv4_route(self, nic, gateways): + def gen_ipv4_route(self, nic, gateways, netmask): """ - Return the lines needed to configure additional Ipv4 route - @return (str list): the string list to configure the gateways + Return the routes list needed to configure additional Ipv4 route + @return (list): the route list to configure the gateways @param nic (NicBase): the nic to configure @param gateways (str list): the list of gateways """ - lines = [] + route_list = [] + + cidr = mask_to_net_prefix(netmask) for gateway in gateways: - lines.append(' up route add default gw %s metric 10000' % - gateway) + destination = "%s/%d" % (gen_subnet(gateway, netmask), cidr) + route_list.append({'destination': destination, + 'type': 'route', + 'gateway': gateway, + 'metric': 10000}) - return lines + return route_list def gen_ipv6(self, name, nic): """ - Return the lines needed to configure the gateways for a nic - @return (str list): the string list to configure the gateways + Return the set of subnets and routes needed to configure the + gateways for a nic + @return (set): the set of subnets and routes to configure the gateways @param name (str): name of the nic @param nic (NicBase): the nic to configure """ - lines = [] if not nic.staticIpv6: - return lines + return ([], []) + subnet_list = [] # Static Ipv6 addrs = nic.staticIpv6 - lines.append('iface %s inet6 static' % name) - lines.append(' address %s' % addrs[0].ip) - lines.append(' netmask %s' % addrs[0].netmask) - for addr in addrs[1:]: - lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, - addr.netmask)) - # Add the primary gateway - if nic.primary: - for addr in addrs: - if addr.gateway: - self.ipv6PrimaryGateway = addr.gateway - lines.append(' gateway %s' % self.ipv6PrimaryGateway) - return lines + for addr in addrs: + subnet = {'type': 'static6', + 'address': addr.ip, + 'netmask': addr.netmask} + subnet_list.append(subnet) - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self._genIpv6Route(name, nic, addrs)) + # TODO: Add the primary gateway + + route_list = [] + # TODO: Add routes if there is no primary nic + # if not self._primaryNic: + # route_list.extend(self._genIpv6Route(name, nic, addrs)) - return lines + return (subnet_list, route_list) def _genIpv6Route(self, name, nic, addrs): - lines = [] + route_list = [] for addr in addrs: - lines.append(' up route -A inet6 add default gw ' - '%s metric 10000' % addr.gateway) + route_list.append({'type': 'route', + 'gateway': addr.gateway, + 'metric': 10000}) + + return route_list - return lines + def generate(self, configure=False, osfamily=None): + """Return the config elements that are needed to configure the nics""" + if configure: + logger.info("Configuring the interfaces file") + self.configure(osfamily) - def generate(self): - """Return the lines that is needed to configure the nics""" - lines = [] - lines.append('iface lo inet loopback') - lines.append('auto lo') - lines.append('') + nics_cfg_list = [] for nic in self.nics: - lines.extend(self.gen_one_nic(nic)) + nics_cfg_list.extend(self.gen_one_nic(nic)) - return lines + return nics_cfg_list def clear_dhcp(self): logger.info('Clearing DHCP leases') @@ -201,11 +252,16 @@ class NicConfigurator(object): util.subp(["pkill", "dhclient"], rcs=[0, 1]) util.subp(["rm", "-f", "/var/lib/dhcp/*"]) - def configure(self): + def configure(self, osfamily=None): """ - Configure the /etc/network/intefaces + Configure the /etc/network/interfaces Make a back up of the original """ + + if not osfamily or osfamily != "debian": + logger.info("Debian OS not detected. Skipping the configure step") + return + containingDir = '/etc/network' interfaceFile = os.path.join(containingDir, 'interfaces') @@ -215,10 +271,13 @@ class NicConfigurator(object): if not os.path.exists(originalFile) and os.path.exists(interfaceFile): os.rename(interfaceFile, originalFile) - lines = self.generate() - with open(interfaceFile, 'w') as fp: - for line in lines: - fp.write('%s\n' % line) + lines = [ + "# DO NOT EDIT THIS FILE BY HAND --" + " AUTOMATICALLY GENERATED BY cloud-init", + "source /etc/network/interfaces.d/*.cfg", + ] + + util.write_file(interfaceFile, content='\n'.join(lines)) self.clear_dhcp() diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 1ab6bd41..44075255 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -59,14 +59,16 @@ def set_customization_status(custstate, custerror, errormessage=None): return (out, err) -# This will read the file nics.txt in the specified directory -# and return the content -def get_nics_to_enable(dirpath): - if not dirpath: +def get_nics_to_enable(nicsfilepath): + """Reads the NICS from the specified file path and returns the content + + @param nicsfilepath: Absolute file path to the NICS.txt file. + """ + + if not nicsfilepath: return None NICS_SIZE = 1024 - nicsfilepath = os.path.join(dirpath, "nics.txt") if not os.path.exists(nicsfilepath): return None diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index d8651077..808d303a 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,9 +8,13 @@ import logging import sys +from cloudinit.sources.DataSourceOVF import get_network_config_from_conf +from cloudinit.sources.DataSourceOVF import read_vmware_imc from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic import gen_subnet +from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) @@ -20,6 +24,7 @@ logger = logging.getLogger(__name__) class TestVmwareConfigFile(CiTestCase): def test_utility_methods(self): + """Tests basic utility methods of ConfigFile class""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") cf.clear() @@ -43,7 +48,26 @@ class TestVmwareConfigFile(CiTestCase): self.assertFalse(cf.should_keep_current_value("BAR"), "keepBar") self.assertTrue(cf.should_remove_current_value("BAR"), "removeBar") + def test_datasource_instance_id(self): + """Tests instance id for the DatasourceOVF""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + instance_id_prefix = 'iid-vmware-' + + conf = Config(cf) + + (md1, _, _) = read_vmware_imc(conf) + self.assertIn(instance_id_prefix, md1["instance-id"]) + self.assertEqual(len(md1["instance-id"]), len(instance_id_prefix) + 8) + + (md2, _, _) = read_vmware_imc(conf) + self.assertIn(instance_id_prefix, md2["instance-id"]) + self.assertEqual(len(md2["instance-id"]), len(instance_id_prefix) + 8) + + self.assertNotEqual(md1["instance-id"], md2["instance-id"]) + def test_configfile_static_2nics(self): + """Tests Config class for a configuration with two static NICs.""" cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") conf = Config(cf) @@ -81,6 +105,7 @@ class TestVmwareConfigFile(CiTestCase): self.assertTrue(not nics[1].staticIpv6, "ipv61 dhcp") def test_config_file_dhcp_2nics(self): + """Tests Config class for a configuration with two DHCP NICs.""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") conf = Config(cf) @@ -117,5 +142,197 @@ class TestVmwareConfigFile(CiTestCase): conf = Config(cf) self.assertTrue(conf.reset_password, "reset password") + def test_get_config_nameservers(self): + """Tests DNS and nameserver settings in a configuration.""" + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + config = Config(cf) + + network_config = get_network_config_from_conf(config, False) + + self.assertEqual(1, network_config.get('version')) + + config_types = network_config.get('config') + name_servers = None + dns_suffixes = None + + for type in config_types: + if type.get('type') == 'nameserver': + name_servers = type.get('address') + dns_suffixes = type.get('search') + break + + self.assertEqual(['10.20.145.1', '10.20.145.2'], + name_servers, + "dns") + self.assertEqual(['eng.vmware.com', 'proxy.vmware.com'], + dns_suffixes, + "suffixes") + + def test_gen_subnet(self): + """Tests if gen_subnet properly calculates network subnet from + IPv4 address and netmask""" + ip_subnet_list = [['10.20.87.253', '255.255.252.0', '10.20.84.0'], + ['10.20.92.105', '255.255.252.0', '10.20.92.0'], + ['192.168.0.10', '255.255.0.0', '192.168.0.0']] + for entry in ip_subnet_list: + self.assertEqual(entry[2], gen_subnet(entry[0], entry[1]), + "Subnet for a specified ip and netmask") + + def test_get_config_dns_suffixes(self): + """Tests if get_network_config_from_conf properly + generates nameservers and dns settings from a + specified configuration""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + config = Config(cf) + + network_config = get_network_config_from_conf(config, False) + + self.assertEqual(1, network_config.get('version')) + + config_types = network_config.get('config') + name_servers = None + dns_suffixes = None + + for type in config_types: + if type.get('type') == 'nameserver': + name_servers = type.get('address') + dns_suffixes = type.get('search') + break + + self.assertEqual([], + name_servers, + "dns") + self.assertEqual(['eng.vmware.com'], + dns_suffixes, + "suffixes") + + def test_get_nics_list_dhcp(self): + """Tests if NicConfigurator properly calculates network subnets + for a configuration with a list of DHCP NICs""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + config = Config(cf) + + nicConfigurator = NicConfigurator(config.nics, False) + nics_cfg_list = nicConfigurator.generate() + + self.assertEqual(2, len(nics_cfg_list), "number of config elements") + + nic1 = {'name': 'NIC1'} + nic2 = {'name': 'NIC2'} + for cfg in nics_cfg_list: + if cfg.get('name') == nic1.get('name'): + nic1.update(cfg) + elif cfg.get('name') == nic2.get('name'): + nic2.update(cfg) + + self.assertEqual('physical', nic1.get('type'), 'type of NIC1') + self.assertEqual('NIC1', nic1.get('name'), 'name of NIC1') + self.assertEqual('00:50:56:a6:8c:08', nic1.get('mac_address'), + 'mac address of NIC1') + subnets = nic1.get('subnets') + self.assertEqual(1, len(subnets), 'number of subnets for NIC1') + subnet = subnets[0] + self.assertEqual('dhcp', subnet.get('type'), 'DHCP type for NIC1') + self.assertEqual('auto', subnet.get('control'), 'NIC1 Control type') + + self.assertEqual('physical', nic2.get('type'), 'type of NIC2') + self.assertEqual('NIC2', nic2.get('name'), 'name of NIC2') + self.assertEqual('00:50:56:a6:5a:de', nic2.get('mac_address'), + 'mac address of NIC2') + subnets = nic2.get('subnets') + self.assertEqual(1, len(subnets), 'number of subnets for NIC2') + subnet = subnets[0] + self.assertEqual('dhcp', subnet.get('type'), 'DHCP type for NIC2') + self.assertEqual('auto', subnet.get('control'), 'NIC2 Control type') + + def test_get_nics_list_static(self): + """Tests if NicConfigurator properly calculates network subnets + for a configuration with 2 static NICs""" + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + config = Config(cf) + + nicConfigurator = NicConfigurator(config.nics, False) + nics_cfg_list = nicConfigurator.generate() + + self.assertEqual(5, len(nics_cfg_list), "number of elements") + + nic1 = {'name': 'NIC1'} + nic2 = {'name': 'NIC2'} + route_list = [] + for cfg in nics_cfg_list: + cfg_type = cfg.get('type') + if cfg_type == 'physical': + if cfg.get('name') == nic1.get('name'): + nic1.update(cfg) + elif cfg.get('name') == nic2.get('name'): + nic2.update(cfg) + elif cfg_type == 'route': + route_list.append(cfg) + + self.assertEqual('physical', nic1.get('type'), 'type of NIC1') + self.assertEqual('NIC1', nic1.get('name'), 'name of NIC1') + self.assertEqual('00:50:56:a6:8c:08', nic1.get('mac_address'), + 'mac address of NIC1') + + subnets = nic1.get('subnets') + self.assertEqual(2, len(subnets), 'Number of subnets') + + static_subnet = [] + static6_subnet = [] + + for subnet in subnets: + subnet_type = subnet.get('type') + if subnet_type == 'static': + static_subnet.append(subnet) + elif subnet_type == 'static6': + static6_subnet.append(subnet) + else: + self.assertEqual(True, False, 'Unknown type') + + self.assertEqual(1, len(static_subnet), 'Number of static subnet') + self.assertEqual(1, len(static6_subnet), 'Number of static6 subnet') + + subnet = static_subnet[0] + self.assertEqual('10.20.87.154', subnet.get('address'), + 'IPv4 address of static subnet') + self.assertEqual('255.255.252.0', subnet.get('netmask'), + 'NetMask of static subnet') + self.assertEqual('auto', subnet.get('control'), + 'control for static subnet') + + subnet = static6_subnet[0] + self.assertEqual('fc00:10:20:87::154', subnet.get('address'), + 'IPv6 address of static subnet') + self.assertEqual('64', subnet.get('netmask'), + 'NetMask of static6 subnet') + + route_set = set(['10.20.87.253', '10.20.87.105', '192.168.0.10']) + for route in route_list: + self.assertEqual(10000, route.get('metric'), 'metric of route') + gateway = route.get('gateway') + if gateway in route_set: + route_set.discard(gateway) + else: + self.assertEqual(True, False, 'invalid gateway %s' % (gateway)) + + self.assertEqual('physical', nic2.get('type'), 'type of NIC2') + self.assertEqual('NIC2', nic2.get('name'), 'name of NIC2') + self.assertEqual('00:50:56:a6:ef:7d', nic2.get('mac_address'), + 'mac address of NIC2') + + subnets = nic2.get('subnets') + self.assertEqual(1, len(subnets), 'Number of subnets for NIC2') + + subnet = subnets[0] + self.assertEqual('static', subnet.get('type'), 'Subnet type') + self.assertEqual('192.168.6.102', subnet.get('address'), + 'Subnet address') + self.assertEqual('255.255.0.0', subnet.get('netmask'), + 'Subnet netmask') + # vi: ts=4 expandtab -- cgit v1.2.3 From a4c1d578070145023ae88a9f79f8517e36b52559 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 11 Sep 2017 10:34:00 -0700 Subject: tools: Add xkvm script, wrapper around qemu-system The xkvm script will be utilized by pending NoCloud qemu testing. If this turns out to not be the case, then we will drop it. --- tools/xkvm | 664 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 664 insertions(+) create mode 100755 tools/xkvm diff --git a/tools/xkvm b/tools/xkvm new file mode 100755 index 00000000..a30ba916 --- /dev/null +++ b/tools/xkvm @@ -0,0 +1,664 @@ +#!/bin/bash + +set -f + +VERBOSITY=0 +KVM_PID="" +DRY_RUN=false +TEMP_D="" +DEF_BRIDGE="virbr0" +TAPDEVS=( ) +# OVS_CLEANUP gets populated with bridge:devname pairs used with ovs +OVS_CLEANUP=( ) +MAC_PREFIX="52:54:00:12:34" +KVM="kvm" +declare -A KVM_DEVOPTS + +error() { echo "$@" 1>&2; } +fail() { [ $# -eq 0 ] || error "$@"; exit 1; } + +bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; exit 1; } +randmac() { + # return random mac addr within final 3 tokens + local random="" + random=$(printf "%02x:%02x:%02x" \ + "$((${RANDOM}%256))" "$((${RANDOM}%256))" "$((${RANDOM}%256))") + padmac "$random" +} + +cleanup() { + [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" + [ -z "${KVM_PID}" ] || kill "$KVM_PID" + if [ ${#TAPDEVS[@]} -ne 0 ]; then + local name item + for item in "${TAPDEVS[@]}"; do + [ "${item}" = "skip" ] && continue + debug 1 "removing" "$item" + name="${item%:*}" + if $DRY_RUN; then + error ip tuntap del mode tap "$name" + else + ip tuntap del mode tap "$name" + fi + [ $? -eq 0 ] || error "failed removal of $name" + done + if [ ${#OVS_CLEANUP[@]} -ne 0 ]; then + # with linux bridges, there seems to be no harm in just deleting + # the device (not detaching from the bridge). However, with + # ovs, you have to remove them from the bridge, or later it + # will refuse to add the same name. + error "cleaning up ovs ports: ${OVS_CLEANUP[@]}" + if ${DRY_RUN}; then + error sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}" + else + sudo "$0" tap-control ovs-cleanup "${OVS_CLEANUP[@]}" + fi + fi + fi +} + +debug() { + local level=${1}; shift; + [ "${level}" -gt "${VERBOSITY}" ] && return + error "${@}" +} + +Usage() { + cat <&1) && + out=$(echo "$out" | sed -e "s,[^.]*[.],," -e 's,=.*,,') && + KVM_DEVOPTS[$model]="$out" || + { error "bad device model $model?"; exit 1; } + fi + opts=( ${KVM_DEVOPTS[$model]} ) + for opt in "${opts[@]}"; do + [ "$input" = "$opt" ] && return 0 + done + return 1 +} + +padmac() { + # return a full mac, given a subset. + # assume whatever is input is the last portion to be + # returned, and fill it out with entries from MAC_PREFIX + local mac="$1" num="$2" prefix="${3:-$MAC_PREFIX}" itoks="" ptoks="" + # if input is empty set to :$num + [ -n "$mac" ] || mac=$(printf "%02x" "$num") || return + itoks=( ${mac//:/ } ) + ptoks=( ${prefix//:/ } ) + rtoks=( ) + for r in ${ptoks[@]:0:6-${#itoks[@]}} ${itoks[@]}; do + rtoks[${#rtoks[@]}]="0x$r" + done + _RET=$(printf "%02x:%02x:%02x:%02x:%02x:%02x" "${rtoks[@]}") +} + +make_nics_Usage() { + cat <: for each tap created + # type is one of "ovs" or "brctl" + local short_opts="v" + local long_opts="--verbose" + local getopt_out="" + getopt_out=$(getopt --name "${0##*/} make-nics" \ + --options "${short_opts}" --long "${long_opts}" -- "$@") && + eval set -- "${getopt_out}" || { make_nics_Usage 1>&2; return 1; } + + local cur="" next="" + while [ $# -ne 0 ]; do + cur=${1}; next=${2}; + case "$cur" in + -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; + --) shift; break;; + esac + shift; + done + + [ $# -ne 0 ] || { + make_nics_Usage 1>&2; error "must give bridge"; + return 1; + } + + local owner="" ovsbrs="" tap="" tapnum="0" brtype="" bridge="" + [ "$(id -u)" = "0" ] || { error "must be root for make-nics"; return 1; } + owner="${SUDO_USER:-root}" + ovsbrs="" + if command -v ovs-vsctl >/dev/null 2>&1; then + out=$(ovs-vsctl list-br) + out=$(echo "$out" | sed "s/\n/,/") + ovsbrs=",$out," + fi + for bridge in "$@"; do + [ "$bridge" = "user" ] && echo skip && continue + [ "${ovsbrs#*,${bridge},}" != "$ovsbrs" ] && + btype="ovs" || btype="brctl" + tapnum=0; + while [ -e /sys/class/net/tapvm$tapnum ]; do tapnum=$(($tapnum+1)); done + tap="tapvm$tapnum" + debug 1 "creating $tap:$btype on $bridge" 1>&2 + ip tuntap add mode tap user "$owner" "$tap" || + { error "failed to create tap '$tap' for '$owner'"; return 1; } + ip link set "$tap" up 1>&2 || { + error "failed to bring up $tap"; + ip tuntap del mode tap "$tap"; + return 1; + } + if [ "$btype" = "ovs" ]; then + ovs-vsctl add-port "$bridge" "$tap" 1>&2 || { + error "failed: ovs-vsctl add-port $bridge $tap"; + ovs-vsctl del-port "$bridge" "$tap" + return 1; + } + else + ip link set "$tap" master "$bridge" 1>&2 || { + error "failed to add tap '$tap' to '$bridge'" + ip tuntap del mode tap "$tap"; + return 1 + } + fi + echo "$tap:$btype" + done +} + +ovs_cleanup() { + [ "$(id -u)" = "0" ] || + { error "must be root for ovs-cleanup"; return 1; } + local item="" errors=0 + # TODO: if get owner (SUDO_USERNAME) and if that isn't + # the owner, then do not delete. + for item in "$@"; do + name=${item#*:} + bridge=${item%:*} + ovs-vsctl del-port "$bridge" "$name" || errors=$((errors+1)) + done + return $errors +} + +quote_cmd() { + local quote='"' x="" vline="" + for x in "$@"; do + if [ "${x#* }" != "${x}" ]; then + if [ "${x#*$quote}" = "${x}" ]; then + x="\"$x\"" + else + x="'$x'" + fi + fi + vline="${vline} $x" + done + echo "$vline" +} + +get_bios_opts() { + # get_bios_opts(bios, uefi, nvram) + # bios is a explicit bios to boot. + # uefi is boolean indicating uefi + # nvram is optional and indicates that ovmf vars should be copied + # to that file if it does not exist. if it exists, use it. + local bios="$1" uefi="${2:-false}" nvram="$3" + local ovmf_dir="/usr/share/OVMF" + local bios_opts="" pflash_common="if=pflash,format=raw" + unset _RET + _RET=( ) + if [ -n "$bios" ]; then + _RET=( -drive "${pflash_common},file=$bios" ) + return 0 + elif ! $uefi; then + return 0 + fi + + # ovmf in older releases (14.04) shipped only a single file + # /usr/share/ovmf/OVMF.fd + # newer ovmf ships split files + # /usr/share/OVMF/OVMF_CODE.fd + # /usr/share/OVMF/OVMF_VARS.fd + # with single file, pass only one file and read-write + # with split, pass code as readonly and vars as read-write + local joined="/usr/share/ovmf/OVMF.fd" + local code="/usr/share/OVMF/OVMF_CODE.fd" + local vars="/usr/share/OVMF/OVMF_VARS.fd" + local split="" nvram_src="" + if [ -e "$code" -o -e "$vars" ]; then + split=true + nvram_src="$vars" + elif [ -e "$joined" ]; then + split=false + nvram_src="$joined" + elif [ -n "$nvram" -a -e "$nvram" ]; then + error "WARN: nvram given, but did not find expected ovmf files." + error " assuming this is code and vars (OVMF.fd)" + split=false + else + error "uefi support requires ovmf bios: apt-get install -qy ovmf" + return 1 + fi + + if [ -n "$nvram" ]; then + if [ ! -f "$nvram" ]; then + cp "$nvram_src" "$nvram" || + { error "failed copy $nvram_src to $nvram"; return 1; } + debug 1 "copied $nvram_src to $nvram" + fi + else + debug 1 "uefi without --uefi-nvram storage." \ + "nvram settings likely will not persist." + nvram="${nvram_src}" + fi + + if [ ! -w "$nvram" ]; then + debug 1 "nvram file ${nvram} is readonly" + nvram_ro="readonly" + fi + + if $split; then + # to ensure bootability firmware must be first, then variables + _RET=( -drive "${pflash_common},file=$code,readonly" ) + fi + _RET=( "${_RET[@]}" + -drive "${pflash_common},file=$nvram${nvram_ro:+,${nvram_ro}}" ) +} + +main() { + local short_opts="hd:n:v" + local long_opts="bios:,help,dowait,disk:,dry-run,kvm:,no-dowait,netdev:,uefi,uefi-nvram:,verbose" + local getopt_out="" + getopt_out=$(getopt --name "${0##*/}" \ + --options "${short_opts}" --long "${long_opts}" -- "$@") && + eval set -- "${getopt_out}" || { bad_Usage; return 1; } + + local bridge="$DEF_BRIDGE" oifs="$IFS" + local netdevs="" need_tap="" ret="" p="" i="" pt="" cur="" conn="" + local kvm="" kvmcmd="" archopts="" + local def_disk_driver=${DEF_DISK_DRIVER:-"virtio-blk"} + local def_netmodel=${DEF_NETMODEL:-"virtio-net-pci"} + local bios="" uefi=false uefi_nvram="" + + archopts=( ) + kvmcmd=( ) + netdevs=( ) + addargs=( ) + diskdevs=( ) + diskargs=( ) + + # dowait: run qemu-system with a '&' and then 'wait' on the pid. + # the reason to do this or not do this has to do with interactivity + # if detached with &, then user input will not go to xkvm. + # if *not* detached, then signal handling is blocked until + # the foreground subprocess returns. which means we can't handle + # a sigterm and kill the qemu-system process. + # We default to dowait=false if input and output are a terminal + local dowait="" + [ -t 0 -a -t 1 ] && dowait=false || dowait=true + while [ $# -ne 0 ]; do + cur=${1}; next=${2}; + case "$cur" in + -h|--help) Usage; exit 0;; + -d|--disk) + diskdevs[${#diskdevs[@]}]="$next"; shift;; + --dry-run) DRY_RUN=true;; + --kvm) kvm="$next"; shift;; + -n|--netdev) + netdevs[${#netdevs[@]}]=$next; shift;; + -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));; + --dowait) dowait=true;; + --no-dowait) dowait=false;; + --bios) bios="$next"; shift;; + --uefi) uefi=true;; + --uefi-nvram) uefi=true; uefi_nvram="$next"; shift;; + --) shift; break;; + esac + shift; + done + + [ ${#netdevs[@]} -eq 0 ] && netdevs=( "${DEF_BRIDGE}" ) + pt=( "$@" ) + + local kvm_pkg="" virtio_scsi_bus="virtio-scsi-pci" + [ -n "$kvm" ] && kvm_pkg="none" + case $(uname -m) in + i?86) + [ -n "$kvm" ] || + { kvm="qemu-system-i386"; kvm_pkg="qemu-system-x86"; } + ;; + x86_64) + [ -n "$kvm" ] || + { kvm="qemu-system-x86_64"; kvm_pkg="qemu-system-x86"; } + ;; + s390x) + [ -n "$kvm" ] || + { kvm="qemu-system-s390x"; kvm_pkg="qemu-system-misc"; } + def_netmodel=${DEF_NETMODEL:-"virtio-net-ccw"} + virtio_scsi_bus="virtio-scsi-ccw" + ;; + ppc64*) + [ -n "$kvm" ] || + { kvm="qemu-system-ppc64"; kvm_pkg="qemu-system-ppc"; } + def_netmodel="virtio-net-pci" + # virtio seems functional on in 14.10, but might want scsi here + #def_diskif="scsi" + archopts=( "${archopts[@]}" -machine pseries,usb=off ) + archopts=( "${archopts[@]}" -device spapr-vscsi ) + ;; + *) kvm=qemu-system-$(uname -m);; + esac + KVM="$kvm" + kvmcmd=( $kvm -enable-kvm ) + + local bios_opts="" + if [ -n "$bios" ] && $uefi; then + error "--uefi (or --uefi-nvram) is incompatible with --bios" + return 1 + fi + get_bios_opts "$bios" "$uefi" "$uefi_nvram" || + { error "failed to get bios opts"; return 1; } + bios_opts=( "${_RET[@]}" ) + + local out="" fmt="" bus="" unit="" index="" serial="" driver="" devopts="" + local busorindex="" driveopts="" cur="" val="" file="" + for((i=0;i<${#diskdevs[@]};i++)); do + cur=${diskdevs[$i]} + IFS=","; set -- $cur; IFS="$oifs" + driver="" + id=$(printf "disk%02d" "$i") + file="" + fmt="" + bus="" + unit="" + index="" + serial="" + for tok in "$@"; do + [ "${tok#*=}" = "${tok}" -a -f "${tok}" -a -z "$file" ] && file="$tok" + val=${tok#*=} + case "$tok" in + driver=*) driver=$val;; + if=virtio) driver=virtio-blk;; + if=scsi) driver=scsi-hd;; + if=pflash) driver=;; + if=sd|if=mtd|floppy) fail "do not know what to do with $tok on $cur";; + id=*) id=$val;; + file=*) file=$val;; + fmt=*|format=*) fmt=$val;; + serial=*) serial=$val;; + bus=*) bus=$val;; + unit=*) unit=$val;; + index=*) index=$val;; + esac + done + [ -z "$file" ] && fail "did not read a file from $cur" + if [ -f "$file" -a -z "$fmt" ]; then + out=$(LANG=C qemu-img info "$file") && + fmt=$(echo "$out" | awk '$0 ~ /^file format:/ { print $3 }') || + { error "failed to determine format of $file"; return 1; } + else + fmt=raw + fi + if [ -z "$driver" ]; then + driver="$def_disk_driver" + fi + if [ -z "$serial" ]; then + serial="${file##*/}" + fi + + # make sure we add either bus= or index= + if [ -n "$bus" -o "$unit" ] && [ -n "$index" ]; then + fail "bus and index cant be specified together: $cur" + elif [ -z "$bus" -a -z "$unit" -a -z "$index" ]; then + index=$i + elif [ -n "$bus" -a -z "$unit" ]; then + unit=$i + fi + + busorindex="${bus:+bus=$bus,unit=$unit}${index:+index=${index}}" + diskopts="file=${file},id=$id,if=none,format=$fmt,$busorindex" + devopts="$driver,drive=$id${serial:+,serial=${serial}}" + for tok in "$@"; do + case "$tok" in + id=*|if=*|driver=*|$file|file=*) continue;; + fmt=*|format=*) continue;; + serial=*|bus=*|unit=*|index=*) continue;; + esac + isdevopt "$driver" "$tok" && devopts="${devopts},$tok" || + diskopts="${diskopts},${tok}" + done + + diskargs=( "${diskargs[@]}" -drive "$diskopts" -device "$devopts" ) + done + + local mnics_vflag="" + for((i=0;i<${VERBOSITY}-1;i++)); do mnics_vflag="${mnics_vflag}v"; done + [ -n "$mnics_vflag" ] && mnics_vflag="-${mnics_vflag}" + + # now go through and split out options + # -device virtio-net-pci,netdev=virtnet0,mac=52:54:31:15:63:02 + # -netdev type=tap,id=virtnet0,vhost=on,script=/etc/kvm/kvm-ifup.br0,downscript=no + local netopts="" devopts="" id="" need_taps=0 model="" + local device_args netdev_args + device_args=( ) + netdev_args=( ) + connections=( ) + for((i=0;i<${#netdevs[@]};i++)); do + id=$(printf "net%02d" "$i") + netopts=""; + devopts="" + # mac=auto is 'unspecified' (let qemu assign one) + mac="auto" + #vhost="off" + + IFS=","; set -- ${netdevs[$i]}; IFS="$oifs" + bridge=$1; shift; + if [ "$bridge" = "user" ]; then + netopts="type=user" + ntype="user" + connections[$i]="user" + else + need_taps=1 + ntype="tap" + netopts="type=tap" + connections[$i]="$bridge" + fi + netopts="${netopts},id=$id" + [ "$ntype" = "tap" ] && netopts="${netopts},script=no,downscript=no" + + model="${def_netmodel}" + for tok in "$@"; do + [ "${tok#model=}" = "${tok}" ] && continue + case "${tok#model=}" in + virtio) model=virtio-net-pci;; + *) model=${tok#model=};; + esac + done + + for tok in "$@"; do + case "$tok" in + mac=*) mac="${tok#mac=}"; continue;; + macaddr=*) mac=${tok#macaddr=}; continue;; + model=*) continue;; + esac + + isdevopt "$model" "$tok" && devopts="${devopts},$tok" || + netopts="${netopts},${tok}" + done + devopts=${devopts#,} + netopts=${netopts#,} + + if [ "$mac" != "auto" ]; then + [ "$mac" = "random" ] && randmac && mac="$_RET" + padmac "$mac" "$i" + devopts="${devopts:+${devopts},}mac=$_RET" + fi + devopts="$model,netdev=$id${devopts:+,${devopts}}" + #netopts="${netopts},vhost=${vhost}" + + device_args[$i]="$devopts" + netdev_args[$i]="$netopts" + done + + trap cleanup EXIT + + reqs=( "$kvm" ) + pkgs=( "$kvm_pkg" ) + for((i=0;i<${#reqs[@]};i++)); do + req=${reqs[$i]} + pkg=${pkgs[$i]} + [ "$pkg" = "none" ] && continue + command -v "$req" >/dev/null || { + missing="${missing:+${missing} }${req}" + missing_pkgs="${missing_pkgs:+${missing_pkgs} }$pkg" + } + done + if [ -n "$missing" ]; then + local reply cmd="" + cmd=( sudo apt-get --quiet install ${missing_pkgs} ) + error "missing prereqs: $missing"; + error "install them now with the following?: ${cmd[*]}" + read reply && [ "$reply" = "y" -o "$reply" = "Y" ] || + { error "run: apt-get install ${missing_pkgs}"; return 1; } + "${cmd[@]}" || { error "failed to install packages"; return 1; } + fi + + if [ $need_taps -ne 0 ]; then + local missing="" missing_pkgs="" reqs="" req="" pkgs="" pkg="" + for i in "${connections[@]}"; do + [ "$i" = "user" -o -e "/sys/class/net/$i" ] || + missing="${missing} $i" + done + [ -z "$missing" ] || { + error "cannot create connection on: ${missing# }." + error "bridges do not exist."; + return 1; + } + error "creating tap devices: ${connections[*]}" + if $DRY_RUN; then + error "sudo $0 tap-control make-nics" \ + $mnics_vflag "${connections[@]}" + taps="" + for((i=0;i<${#connections[@]};i++)); do + if [ "${connections[$i]}" = "user" ]; then + taps="${taps} skip" + else + taps="${taps} dryruntap$i:brctl" + fi + done + else + taps=$(sudo "$0" tap-control make-nics \ + ${mnics_vflag} "${connections[@]}") || + { error "$failed to make-nics ${connections[*]}"; return 1; } + fi + TAPDEVS=( ${taps} ) + for((i=0;i<${#TAPDEVS[@]};i++)); do + cur=${TAPDEVS[$i]} + [ "${cur#*:}" = "ovs" ] || continue + conn=${connections[$i]} + OVS_CLEANUP[${#OVS_CLEANUP[@]}]="${conn}:${cur%:*}" + done + + debug 2 "tapdevs='${TAPDEVS[@]}'" + [ ${#OVS_CLEANUP[@]} -eq 0 ] || error "OVS_CLEANUP='${OVS_CLEANUP[*]}'" + + for((i=0;i<${#TAPDEVS[@]};i++)); do + cur=${TAPDEVS[$i]} + [ "$cur" = "skip" ] && continue + netdev_args[$i]="${netdev_args[$i]},ifname=${cur%:*}"; + done + fi + + netargs=() + for((i=0;i<${#device_args[@]};i++)); do + netargs=( "${netargs[@]}" -device "${device_args[$i]}" + -netdev "${netdev_args[$i]}") + done + + local bus_devices + bus_devices=( -device "$virtio_scsi_bus,id=virtio-scsi-xkvm" ) + cmd=( "${kvmcmd[@]}" "${archopts[@]}" + "${bios_opts[@]}" + "${bus_devices[@]}" + "${netargs[@]}" + "${diskargs[@]}" "${pt[@]}" ) + local pcmd=$(quote_cmd "${cmd[@]}") + error "$pcmd" + ${DRY_RUN} && return 0 + + if $dowait; then + "${cmd[@]}" & + KVM_PID=$! + debug 1 "kvm pid=$KVM_PID. my pid=$$" + wait + ret=$? + KVM_PID="" + else + "${cmd[@]}" + ret=$? + fi + return $ret +} + + +if [ "$1" = "tap-control" ]; then + shift + mode=$1 + shift || fail "must give mode to tap-control" + case "$mode" in + make-nics) make_nics "$@";; + ovs-cleanup) ovs_cleanup "$@";; + *) fail "tap mode must be either make-nics or ovs-cleanup";; + esac +else + main "$@" +fi + +# vi: ts=4 expandtab -- cgit v1.2.3 From ed8f1b159174715403cb1ffa200ff6d080770152 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Sat, 2 Sep 2017 01:51:29 -0600 Subject: schema and docs: Add jsonschema to resizefs and bootcmd modules Add schema definitions to both cc_resizefs and cc_bootcmd modules. Extend schema.py to parse and document enumerated json types. Schema definitions are used to generate module documention and log warnings for schema infractions. This branch also does the following: - drops vestigial 'resize_rootfs_tmp' option from cc_resizefs. That option only created the specified directory and didn't make use of that directory for any resize operations. - Drop yaml.dumps calls from schema documentation generation to avoid yaml import costs on module load - Add __doc__ = get_schema_doc(schema) definitions it each module to supplement python help() calls for cc_runcmd, cc_bootcmd, cc_ntp and cc_resizefs - Add a SCHEMA_EXAMPLES_SPACER_TEMPLATE string to docs for modules which contain more than one example --- cloudinit/config/cc_bootcmd.py | 87 ++++--- cloudinit/config/cc_ntp.py | 48 +--- cloudinit/config/cc_resizefs.py | 149 +++++++----- cloudinit/config/cc_runcmd.py | 5 +- cloudinit/config/schema.py | 19 +- .../unittests/test_handler/test_handler_bootcmd.py | 145 ++++++++++++ .../test_handler/test_handler_resizefs.py | 257 ++++++++++++++++++++- tests/unittests/test_handler/test_schema.py | 44 ++-- 8 files changed, 586 insertions(+), 168 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_bootcmd.py diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 9c0476af..233da1ef 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -3,45 +3,73 @@ # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Chad Smith # # This file is part of cloud-init. See LICENSE file for license information. -""" -Bootcmd -------- -**Summary:** run commands early in boot process - -This module runs arbitrary commands very early in the boot process, -only slightly after a boothook would run. This is very similar to a -boothook, but more user friendly. The environment variable ``INSTANCE_ID`` -will be set to the current instance id for all run commands. Commands can be -specified either as lists or strings. For invocation details, see ``runcmd``. - -.. note:: - bootcmd should only be used for things that could not be done later in the - boot process. - -**Internal name:** ``cc_bootcmd`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - bootcmd: - - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts - - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] -""" +"""Bootcmd: run arbitrary commands early in the boot process.""" import os +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +distros = ['all'] + +schema = { + 'id': 'cc_bootcmd', + 'name': 'Bootcmd', + 'title': 'Run arbitrary commands early in the boot process', + 'description': dedent("""\ + This module runs arbitrary commands very early in the boot process, + only slightly after a boothook would run. This is very similar to a + boothook, but more user friendly. The environment variable + ``INSTANCE_ID`` will be set to the current instance id for all run + commands. Commands can be specified either as lists or strings. For + invocation details, see ``runcmd``. + + .. note:: + bootcmd should only be used for things that could not be done later + in the boot process."""), + 'distros': distros, + 'examples': [dedent("""\ + bootcmd: + - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts + - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] + """)], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'bootcmd': { + 'type': 'array', + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject items of non-string non-list + 'additionalProperties': False, + 'minItems': 1, + 'required': [], + 'uniqueItems': True + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): @@ -50,13 +78,14 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return + validate_cloudconfig_schema(cfg, schema) with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except Exception: - util.logexc(log, "Failed to shellify bootcmd") + except Exception as e: + util.logexc(log, "Failed to shellify bootcmd: %s", str(e)) raise try: diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index a02b4bf1..15ae1ecd 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -4,39 +4,10 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -NTP ---- -**Summary:** enable and configure ntp - -Handle ntp configuration. If ntp is not installed on the system and ntp -configuration is specified, ntp will be installed. If there is a default ntp -config file in the image or one is present in the distro's ntp package, it will -be copied to ``/etc/ntp.conf.dist`` before any changes are made. A list of ntp -pools and ntp servers can be provided under the ``ntp`` config key. If no ntp -servers or pools are provided, 4 pools will be used in the format -``{0-3}.{distro}.pool.ntp.org``. - -**Internal name:** ``cc_ntp`` - -**Module frequency:** per instance - -**Supported distros:** centos, debian, fedora, opensuse, ubuntu - -**Config keys**:: - - ntp: - pools: - - 0.company.pool.ntp.org - - 1.company.pool.ntp.org - - ntp.myorg.org - servers: - - my.ntp.server.local - - ntp.ubuntu.com - - 192.168.23.2 -""" +"""NTP: enable and configure ntp""" -from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater @@ -76,10 +47,13 @@ schema = { ``{0-3}.{distro}.pool.ntp.org``."""), 'distros': distros, 'examples': [ - {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org', - 'ntp.myorg.org'], - 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com', - '192.168.23.2']}}], + dedent("""\ + ntp: + pools: [0.int.pool.ntp.org, 1.int.pool.ntp.org, ntp.myorg.org] + servers: + - ntp.server.local + - ntp.ubuntu.com + - 192.168.23.2""")], 'frequency': PER_INSTANCE, 'type': 'object', 'properties': { @@ -117,6 +91,8 @@ schema = { } } +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): """Enable and configure ntp.""" diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index ceee952b..f14d3836 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -6,31 +6,8 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Resizefs --------- -**Summary:** resize filesystem +"""Resizefs: cloud-config module which resizes the filesystem""" -Resize a filesystem to use all avaliable space on partition. This module is -useful along with ``cc_growpart`` and will ensure that if the root partition -has been resized the root filesystem will be resized along with it. By default, -``cc_resizefs`` will resize the root partition and will block the boot process -while the resize command is running. Optionally, the resize operation can be -performed in the background while cloud-init continues running modules. This -can be enabled by setting ``resize_rootfs`` to ``true``. This module can be -disabled altogether by setting ``resize_rootfs`` to ``false``. - -**Internal name:** ``cc_resizefs`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - resize_rootfs: - resize_rootfs_tmp: -""" import errno import getopt @@ -38,11 +15,47 @@ import os import re import shlex import stat +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import util +NOBLOCK = "noblock" + frequency = PER_ALWAYS +distros = ['all'] + +schema = { + 'id': 'cc_resizefs', + 'name': 'Resizefs', + 'title': 'Resize filesystem', + 'description': dedent("""\ + Resize a filesystem to use all avaliable space on partition. This + module is useful along with ``cc_growpart`` and will ensure that if the + root partition has been resized the root filesystem will be resized + along with it. By default, ``cc_resizefs`` will resize the root + partition and will block the boot process while the resize command is + running. Optionally, the resize operation can be performed in the + background while cloud-init continues running modules. This can be + enabled by setting ``resize_rootfs`` to ``true``. This module can be + disabled altogether by setting ``resize_rootfs`` to ``false``."""), + 'distros': distros, + 'examples': [ + 'resize_rootfs: false # disable root filesystem resize operation'], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'resize_rootfs': { + 'enum': [True, False, NOBLOCK], + 'description': dedent("""\ + Whether to resize the root partition. Default: 'true'""") + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() def _resize_btrfs(mount_point, devpth): @@ -131,8 +144,6 @@ RESIZE_FS_PRECHECK_CMDS = { 'ufs': _can_skip_resize_ufs } -NOBLOCK = "noblock" - def rootdev_from_cmdline(cmdline): found = None @@ -161,71 +172,81 @@ def can_skip_resize(fs_type, resize_what, devpth): return False -def handle(name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = args[0] - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if not util.translate_bool(resize_root, addons=[NOBLOCK]): - log.debug("Skipping module named %s, resizing disabled", name) - return - - # TODO(harlowja) is the directory ok to be used?? - resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") - util.ensure_dir(resize_root_d) +def is_device_path_writable_block(devpath, info, log): + """Return True if devpath is a writable block device. - # TODO(harlowja): allow what is to be resized to be configurable?? - resize_what = "/" - result = util.get_mount_info(resize_what, log) - if not result: - log.warn("Could not determine filesystem type of %s", resize_what) - return - - (devpth, fs_type, mount_point) = result - - info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) - log.debug("resize_info: %s" % info) + @param devpath: Path to the root device we want to resize. + @param info: String representing information about the requested device. + @param log: Logger to which logs will be added upon error. + @returns Boolean True if block device is writable + """ container = util.is_container() # Ensure the path is a block device. - if (devpth == "/dev/root" and not os.path.exists(devpth) and + if (devpath == "/dev/root" and not os.path.exists(devpath) and not container): - devpth = util.rootdev_from_cmdline(util.get_cmdline()) - if devpth is None: + devpath = util.rootdev_from_cmdline(util.get_cmdline()) + if devpath is None: log.warn("Unable to find device '/dev/root'") - return - log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth) + return False + log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) try: - statret = os.stat(devpth) + statret = os.stat(devpath) except OSError as exc: if container and exc.errno == errno.ENOENT: log.debug("Device '%s' did not exist in container. " - "cannot resize: %s", devpth, info) + "cannot resize: %s", devpath, info) elif exc.errno == errno.ENOENT: log.warn("Device '%s' did not exist. cannot resize: %s", - devpth, info) + devpath, info) else: raise exc - return + return False - if not os.access(devpth, os.W_OK): + if not os.access(devpath, os.W_OK): if container: log.debug("'%s' not writable in container. cannot resize: %s", - devpth, info) + devpath, info) else: - log.warn("'%s' not writable. cannot resize: %s", devpth, info) + log.warn("'%s' not writable. cannot resize: %s", devpath, info) return if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." - " cannot resize: %s" % (devpth, info)) + " cannot resize: %s" % (devpath, info)) else: log.warn("device '%s' not a block device. cannot resize: %s" % - (devpth, info)) + (devpath, info)) + return False + return True + + +def handle(name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = args[0] + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + validate_cloudconfig_schema(cfg, schema) + if not util.translate_bool(resize_root, addons=[NOBLOCK]): + log.debug("Skipping module named %s, resizing disabled", name) + return + + # TODO(harlowja): allow what is to be resized to be configurable?? + resize_what = "/" + result = util.get_mount_info(resize_what, log) + if not result: + log.warn("Could not determine filesystem type of %s", resize_what) + return + + (devpth, fs_type, mount_point) = result + + info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) + log.debug("resize_info: %s" % info) + + if not is_device_path_writable_block(devpth, info, log): return resizer = None diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 7c3ccd41..7f995693 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -8,7 +8,8 @@ """Runcmd: run arbitrary commands at rc.local with output to the console""" -from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -67,6 +68,8 @@ schema = { } } +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 73dd5c2e..c17d973e 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -14,6 +14,7 @@ import re import sys import yaml +_YAML_MAP = {True: 'true', False: 'false', None: 'null'} SCHEMA_UNDEFINED = b'UNDEFINED' CLOUD_CONFIG_HEADER = b'#cloud-config' SCHEMA_DOC_TMPL = """ @@ -34,6 +35,8 @@ SCHEMA_DOC_TMPL = """ {examples} """ SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' +SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n' +SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n # --- Example{0} ---' class SchemaValidationError(ValueError): @@ -212,6 +215,9 @@ def _schemapath_for_cloudconfig(config, original_content): def _get_property_type(property_dict): """Return a string representing a property type from a given jsonschema.""" property_type = property_dict.get('type', SCHEMA_UNDEFINED) + if property_type == SCHEMA_UNDEFINED and property_dict.get('enum'): + property_type = [ + str(_YAML_MAP.get(k, k)) for k in property_dict['enum']] if isinstance(property_type, list): property_type = '/'.join(property_type) items = property_dict.get('items', {}) @@ -249,15 +255,14 @@ def _get_schema_examples(schema, prefix=''): examples = schema.get('examples') if not examples: return '' - rst_content = '\n**Examples**::\n\n' - for example in examples: - if isinstance(example, str): - example_content = example - else: - example_content = yaml.dump(example, default_flow_style=False) + rst_content = SCHEMA_EXAMPLES_HEADER + for count, example in enumerate(examples): # Python2.6 is missing textwrapper.indent - lines = example_content.split('\n') + lines = example.split('\n') indented_lines = [' {0}'.format(line) for line in lines] + if rst_content != SCHEMA_EXAMPLES_HEADER: + indented_lines.insert( + 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1)) rst_content += '\n'.join(indented_lines) return rst_content diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py new file mode 100644 index 00000000..580017ed --- /dev/null +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -0,0 +1,145 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_bootcmd +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from cloudinit.tests.helpers import CiTestCase, mock, skipIf + +import logging +import tempfile + +try: + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False +except ImportError: + _missing_jsonschema_dep = True + +LOG = logging.getLogger(__name__) + + +class FakeExtendedTempFile(object): + def __init__(self, suffix): + self.suffix = suffix + self.handle = tempfile.NamedTemporaryFile( + prefix="ci-%s." % self.__class__.__name__, delete=False) + + def __enter__(self): + return self.handle + + def __exit__(self, exc_type, exc_value, traceback): + self.handle.close() + + +class TestBootcmd(CiTestCase): + + with_logs = True + + _etmpfile_path = ('cloudinit.config.cc_bootcmd.temp_utils.' + 'ExtendedTemporaryFile') + + def setUp(self): + super(TestBootcmd, self).setUp() + self.subp = util.subp + self.new_root = self.tmp_dir() + + def _get_cloud(self, distro): + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + paths.datasource = myds + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skip_if_no_bootcmd(self): + """When the provided config doesn't contain bootcmd, skip it.""" + cfg = {} + mycloud = self._get_cloud('ubuntu') + cc_bootcmd.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "Skipping module named notimportant, no 'bootcmd' key", + self.logs.getvalue()) + + def test_handler_invalid_command_set(self): + """Commands which can't be converted to shell will raise errors.""" + invalid_config = {'bootcmd': 1} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError) as context_manager: + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + self.assertIn('Failed to shellify bootcmd', self.logs.getvalue()) + self.assertEqual( + "'int' object is not iterable", + str(context_manager.exception)) + + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + def test_handler_schema_validation_warns_non_array_type(self): + """Schema validation warns of non-array type for bootcmd key. + + Schema validation is not strict, so bootcmd attempts to shellify the + invalid content. + """ + invalid_config = {'bootcmd': 1} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError): + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + self.assertIn( + 'Invalid config:\nbootcmd: 1 is not of type \'array\'', + self.logs.getvalue()) + self.assertIn('Failed to shellify', self.logs.getvalue()) + + @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency') + def test_handler_schema_validation_warns_non_array_item_type(self): + """Schema validation warns of non-array or string bootcmd items. + + Schema validation is not strict, so bootcmd attempts to shellify the + invalid content. + """ + invalid_config = { + 'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} + cc = self._get_cloud('ubuntu') + with self.assertRaises(RuntimeError) as context_manager: + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + expected_warnings = [ + 'bootcmd.1: 20 is not valid under any of the given schemas', + 'bootcmd.3: {\'a\': \'n\'} is not valid under any of the given' + ' schema' + ] + logs = self.logs.getvalue() + for warning in expected_warnings: + self.assertIn(warning, logs) + self.assertIn('Failed to shellify', logs) + self.assertEqual( + 'Unable to shellify type int which is not a list or string', + str(context_manager.exception)) + + def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self): + """Valid schema runs a bootcmd script with INSTANCE_ID in the env.""" + cc = self._get_cloud('ubuntu') + out_file = self.tmp_path('bootcmd.out', self.new_root) + my_id = "b6ea0f59-e27d-49c6-9f87-79f19765a425" + valid_config = {'bootcmd': [ + 'echo {0} $INSTANCE_ID > {1}'.format(my_id, out_file)]} + + with mock.patch(self._etmpfile_path, FakeExtendedTempFile): + cc_bootcmd.handle('cc_bootcmd', valid_config, cc, LOG, []) + self.assertEqual(my_id + ' iid-datasource-none\n', + util.load_file(out_file)) + + def test_handler_runs_bootcmd_script_with_error(self): + """When a valid script generates an error, that error is raised.""" + cc = self._get_cloud('ubuntu') + valid_config = {'bootcmd': ['exit 1']} # Script with error + + with mock.patch(self._etmpfile_path, FakeExtendedTempFile): + with self.assertRaises(util.ProcessExecutionError) as ctxt_manager: + cc_bootcmd.handle('does-not-matter', valid_config, cc, LOG, []) + self.assertIn( + 'Unexpected error while running command.\n' + "Command: ['/bin/sh',", + str(ctxt_manager.exception)) + self.assertIn( + 'Failed to run bootcmd module does-not-matter', + self.logs.getvalue()) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 52591b8b..76dddbf8 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -1,17 +1,30 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config import cc_resizefs +from cloudinit.config.cc_resizefs import ( + can_skip_resize, handle, is_device_path_writable_block, + rootdev_from_cmdline) +import logging import textwrap -import unittest + +from cloudinit.tests.helpers import (CiTestCase, mock, skipIf, util, + wrap_and_call) + + +LOG = logging.getLogger(__name__) + try: - from unittest import mock + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False except ImportError: - import mock + _missing_jsonschema_dep = True + +class TestResizefs(CiTestCase): + with_logs = True -class TestResizefs(unittest.TestCase): def setUp(self): super(TestResizefs, self).setUp() self.name = "resizefs" @@ -34,7 +47,7 @@ class TestResizefs(unittest.TestCase): 58720296 3145728 3 freebsd-swap (1.5G) 61866024 1048496 - free - (512M) """) - res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth) + res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') @@ -52,8 +65,238 @@ class TestResizefs(unittest.TestCase): => 34 297086 da0 GPT (145M) 34 297086 1 freebsd-ufs (145M) """) - res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth) + res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) + def test_handle_noops_on_disabled(self): + """The handle function logs when the configuration disables resize.""" + cfg = {'resize_rootfs': False} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + self.assertIn( + 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', + self.logs.getvalue()) + + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + def test_handle_schema_validation_logs_invalid_resize_rootfs_value(self): + """The handle reports json schema violations as a warning. + + Invalid values for resize_rootfs result in disabling the module. + """ + cfg = {'resize_rootfs': 'junk'} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + logs = self.logs.getvalue() + self.assertIn( + "WARNING: Invalid config:\nresize_rootfs: 'junk' is not one of" + " [True, False, 'noblock']", + logs) + self.assertIn( + 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', + logs) + + @mock.patch('cloudinit.config.cc_resizefs.util.get_mount_info') + def test_handle_warns_on_unknown_mount_info(self, m_get_mount_info): + """handle warns when get_mount_info sees unknown filesystem for /.""" + m_get_mount_info.return_value = None + cfg = {'resize_rootfs': True} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + logs = self.logs.getvalue() + self.assertNotIn("WARNING: Invalid config:\nresize_rootfs:", logs) + self.assertIn( + 'WARNING: Could not determine filesystem type of /\n', + logs) + self.assertEqual( + [mock.call('/', LOG)], + m_get_mount_info.call_args_list) + + def test_handle_warns_on_undiscoverable_root_path_in_commandline(self): + """handle noops when the root path is not found on the commandline.""" + cfg = {'resize_rootfs': True} + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + + def fake_mount_info(path, log): + self.assertEqual('/', path) + self.assertEqual(LOG, log) + return ('/dev/root', 'ext4', '/') + + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'get_mount_info': {'side_effect': fake_mount_info}, + 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, + handle, 'cc_resizefs', cfg, _cloud=None, log=LOG, + args=[]) + logs = self.logs.getvalue() + self.assertIn("WARNING: Unable to find device '/dev/root'", logs) + + +class TestRootDevFromCmdline(CiTestCase): + + def test_rootdev_from_cmdline_with_no_root(self): + """Return None from rootdev_from_cmdline when root is not present.""" + invalid_cases = [ + 'BOOT_IMAGE=/adsf asdfa werasef root adf', 'BOOT_IMAGE=/adsf', ''] + for case in invalid_cases: + self.assertIsNone(rootdev_from_cmdline(case)) + + def test_rootdev_from_cmdline_with_root_startswith_dev(self): + """Return the cmdline root when the path starts with /dev.""" + self.assertEqual( + '/dev/this', rootdev_from_cmdline('asdf root=/dev/this')) + + def test_rootdev_from_cmdline_with_root_without_dev_prefix(self): + """Add /dev prefix to cmdline root when the path lacks the prefix.""" + self.assertEqual('/dev/this', rootdev_from_cmdline('asdf root=this')) + + def test_rootdev_from_cmdline_with_root_with_label(self): + """When cmdline root contains a LABEL, our root is disk/by-label.""" + self.assertEqual( + '/dev/disk/by-label/unique', + rootdev_from_cmdline('asdf root=LABEL=unique')) + + def test_rootdev_from_cmdline_with_root_with_uuid(self): + """When cmdline root contains a UUID, our root is disk/by-uuid.""" + self.assertEqual( + '/dev/disk/by-uuid/adsfdsaf-adsf', + rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf')) + + +class TestIsDevicePathWritableBlock(CiTestCase): + + with_logs = True + + def test_is_device_path_writable_block_warns_missing_cmdline_root(self): + """When root does not exist isn't in the cmdline, log warning.""" + info = 'does not matter' + + def fake_mount_info(path, log): + self.assertEqual('/', path) + self.assertEqual(LOG, log) + return ('/dev/root', 'ext4', '/') + + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'get_mount_info': {'side_effect': fake_mount_info}, + 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, + is_device_path_writable_block, '/dev/root', info, LOG) + self.assertFalse(is_valid) + logs = self.logs.getvalue() + self.assertIn("WARNING: Unable to find device '/dev/root'", logs) + + def test_is_device_path_writable_block_does_not_exist(self): + """When devpath does not exist, a warning is logged.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "WARNING: Device '/I/dont/exist' did not exist." + ' cannot resize: %s' % info, + self.logs.getvalue()) + + def test_is_device_path_writable_block_does_not_exist_in_container(self): + """When devpath does not exist in a container, log a debug message.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: Device '/I/dont/exist' did not exist in container." + ' cannot resize: %s' % info, + self.logs.getvalue()) + + def test_is_device_path_writable_block_raises_oserror(self): + """When unexpected OSError is raises by os.stat it is reraised.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + with self.assertRaises(OSError) as context_manager: + wrap_and_call( + 'cloudinit.config.cc_resizefs', + {'util.is_container': {'return_value': True}, + 'os.stat': {'side_effect': OSError('Something unexpected')}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertEqual( + 'Something unexpected', str(context_manager.exception)) + + def test_is_device_path_writable_block_readonly(self): + """When root device is readonly, emit a warning and return False.""" + fake_devpath = self.tmp_path('dev/readonly') + util.write_file(fake_devpath, '', mode=0o400) # read-only + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'rootdev_from_cmdline': {'return_value': fake_devpath}}, + is_device_path_writable_block, '/dev/root', info, LOG) + self.assertFalse(is_valid) + logs = self.logs.getvalue() + self.assertIn( + "Converted /dev/root to '{0}' per kernel cmdline".format( + fake_devpath), + logs) + self.assertIn( + "WARNING: '{0}' not writable. cannot resize".format(fake_devpath), + logs) + + def test_is_device_path_writable_block_readonly_in_container(self): + """When root device is readonly, emit debug log and return False.""" + fake_devpath = self.tmp_path('dev/readonly') + util.write_file(fake_devpath, '', mode=0o400) # read-only + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: '{0}' not writable in container. cannot resize".format( + fake_devpath), + self.logs.getvalue()) + + def test_is_device_path_writable_block_non_block(self): + """When device is not a block device, emit warning return False.""" + fake_devpath = self.tmp_path('dev/readwrite') + util.write_file(fake_devpath, '', mode=0o600) # read-write + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "WARNING: device '{0}' not a block device. cannot resize".format( + fake_devpath), + self.logs.getvalue()) + + def test_is_device_path_writable_block_non_block_on_container(self): + """When device is non-block device in container, emit debug log.""" + fake_devpath = self.tmp_path('dev/readwrite') + util.write_file(fake_devpath, '', mode=0o600) # read-write + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: device '{0}' not a block device in container." + ' cannot resize'.format(fake_devpath), + self.logs.getvalue()) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 6137e3cf..745bb0ff 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,7 +27,7 @@ class GetSchemaTest(CiTestCase): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() self.assertItemsEqual( - ['cc_ntp', 'cc_runcmd'], + ['cc_bootcmd', 'cc_ntp', 'cc_resizefs', 'cc_runcmd'], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) self.assertEqual( @@ -205,6 +205,17 @@ class GetSchemaDocTest(CiTestCase): '**prop1:** (string/integer) prop-description', get_schema_doc(full_schema)) + def test_get_schema_doc_handles_enum_types(self): + """get_schema_doc converts enum types to yaml and delimits with '/'.""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'prop1': {'enum': [True, False, 'stuff'], + 'description': 'prop-description'}}}) + self.assertIn( + '**prop1:** (true/false/stuff) prop-description', + get_schema_doc(full_schema)) + def test_get_schema_doc_handles_nested_oneof_property_types(self): """get_schema_doc describes array items oneOf declarations in type.""" full_schema = copy(self.required_schema) @@ -219,29 +230,11 @@ class GetSchemaDocTest(CiTestCase): '**prop1:** (array of (string)/(integer)) prop-description', get_schema_doc(full_schema)) - def test_get_schema_doc_returns_restructured_text_with_examples(self): - """get_schema_doc returns indented examples when present in schema.""" - full_schema = copy(self.required_schema) - full_schema.update( - {'examples': [{'ex1': [1, 2, 3]}], - 'properties': { - 'prop1': {'type': 'array', 'description': 'prop-description', - 'items': {'type': 'integer'}}}}) - self.assertIn( - dedent(""" - **Config schema**: - **prop1:** (array of integer) prop-description - - **Examples**:: - - ex1"""), - get_schema_doc(full_schema)) - - def test_get_schema_doc_handles_unstructured_examples(self): - """get_schema_doc properly indented examples which as just strings.""" + def test_get_schema_doc_handles_string_examples(self): + """get_schema_doc properly indented examples as a list of strings.""" full_schema = copy(self.required_schema) full_schema.update( - {'examples': ['My example:\n [don\'t, expand, "this"]'], + {'examples': ['ex1:\n [don\'t, expand, "this"]', 'ex2: true'], 'properties': { 'prop1': {'type': 'array', 'description': 'prop-description', 'items': {'type': 'integer'}}}}) @@ -252,8 +245,11 @@ class GetSchemaDocTest(CiTestCase): **Examples**:: - My example: - [don't, expand, "this"]"""), + ex1: + [don't, expand, "this"] + # --- Example2 --- + ex2: true + """), get_schema_doc(full_schema)) def test_get_schema_doc_raises_key_errors(self): -- cgit v1.2.3 From 1ac4bc2a4758d330bb94cd1b2391121cf461ff6a Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 11 Sep 2017 10:29:19 -0700 Subject: tests: execute: support command as string If a string is passed to execute, then invoke 'bash', '-c', 'string'. That allows the less verbose execution of simple commands: image.execute("ls /run") compared to the more explicit but longer winded: image.execute(["ls", "/run"]) If 'env' was ever modified in execute or a method that it called, then the next invocation's default value would be changed. Instead use None and then set to a new empty dict in the method. --- tests/cloud_tests/bddeb.py | 3 +-- tests/cloud_tests/instances/base.py | 10 ++++++---- tests/cloud_tests/instances/lxd.py | 10 +++++++++- tests/cloud_tests/setup_image.py | 12 ++++++------ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index fe805356..fba8a0c7 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -28,8 +28,7 @@ def build_deb(args, instance): # update remote system package list and install build deps LOG.debug('installing pre-reqs') pkgs = ' '.join(pre_reqs) - cmd = 'apt-get update && apt-get install --yes {}'.format(pkgs) - instance.execute(['/bin/sh', '-c', cmd]) + instance.execute('apt-get update && apt-get install --yes {}'.format(pkgs)) # local tmpfile that must be deleted local_tarball = tempfile.NamedTemporaryFile().name diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py index 959e9cce..58f45b14 100644 --- a/tests/cloud_tests/instances/base.py +++ b/tests/cloud_tests/instances/base.py @@ -23,7 +23,7 @@ class Instance(object): self.config = config self.features = features - def execute(self, command, stdout=None, stderr=None, env={}, + def execute(self, command, stdout=None, stderr=None, env=None, rcs=None, description=None): """Execute command in instance, recording output, error and exit code. @@ -31,6 +31,8 @@ class Instance(object): target filesystem being available at /. @param command: the command to execute as root inside the image + if command is a string, then it will be executed as: + ['sh', '-c', command] @param stdout, stderr: file handles to write output and error to @param env: environment variables @param rcs: allowed return codes from command @@ -137,9 +139,9 @@ class Instance(object): tests.append(self.config['cloud_init_ready_script']) formatted_tests = ' && '.join(clean_test(t) for t in tests) - test_cmd = ('for ((i=0;i<{time};i++)); do {test} && exit 0; sleep 1; ' - 'done; exit 1;').format(time=time, test=formatted_tests) - cmd = ['/bin/bash', '-c', test_cmd] + cmd = ('i=0; while [ $i -lt {time} ] && i=$(($i+1)); do {test} && ' + 'exit 0; sleep 1; done; exit 1').format(time=time, + test=formatted_tests) if self.execute(cmd, rcs=(0, 1))[-1] != 0: raise OSError('timeout: after {}s system not started'.format(time)) diff --git a/tests/cloud_tests/instances/lxd.py b/tests/cloud_tests/instances/lxd.py index b9c2cc6b..a43918c2 100644 --- a/tests/cloud_tests/instances/lxd.py +++ b/tests/cloud_tests/instances/lxd.py @@ -31,7 +31,7 @@ class LXDInstance(base.Instance): self._pylxd_container.sync() return self._pylxd_container - def execute(self, command, stdout=None, stderr=None, env={}, + def execute(self, command, stdout=None, stderr=None, env=None, rcs=None, description=None): """Execute command in instance, recording output, error and exit code. @@ -39,6 +39,8 @@ class LXDInstance(base.Instance): target filesystem being available at /. @param command: the command to execute as root inside the image + if command is a string, then it will be executed as: + ['sh', '-c', command] @param stdout: file handler to write output @param stderr: file handler to write error @param env: environment variables @@ -46,6 +48,12 @@ class LXDInstance(base.Instance): @param description: purpose of command @return_value: tuple containing stdout data, stderr data, exit code """ + if env is None: + env = {} + + if isinstance(command, str): + command = ['sh', '-c', command] + # ensure instance is running and execute the command self.start() res = self.pylxd_container.execute(command, environment=env) diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 8053a093..3c0fff62 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -49,8 +49,8 @@ def install_deb(args, image): LOG.debug(msg) remote_path = os.path.join('/tmp', os.path.basename(args.deb)) image.push_file(args.deb, remote_path) - cmd = 'dpkg -i {} || apt-get install --yes -f'.format(remote_path) - image.execute(['/bin/sh', '-c', cmd], description=msg) + cmd = 'dpkg -i {}; apt-get install --yes -f'.format(remote_path) + image.execute(cmd, description=msg) # check installed deb version matches package fmt = ['-W', "--showformat='${Version}'"] @@ -113,7 +113,7 @@ def upgrade(args, image): msg = 'upgrading cloud-init' LOG.debug(msg) - image.execute(['/bin/sh', '-c', cmd], description=msg) + image.execute(cmd, description=msg) def upgrade_full(args, image): @@ -134,7 +134,7 @@ def upgrade_full(args, image): msg = 'full system upgrade' LOG.debug(msg) - image.execute(['/bin/sh', '-c', cmd], description=msg) + image.execute(cmd, description=msg) def run_script(args, image): @@ -165,7 +165,7 @@ def enable_ppa(args, image): msg = 'enable ppa: "{}" in target'.format(ppa) LOG.debug(msg) cmd = 'add-apt-repository --yes {} && apt-get update'.format(ppa) - image.execute(['/bin/sh', '-c', cmd], description=msg) + image.execute(cmd, description=msg) def enable_repo(args, image): @@ -188,7 +188,7 @@ def enable_repo(args, image): msg = 'enable repo: "{}" in target'.format(args.repo) LOG.debug(msg) - image.execute(['/bin/sh', '-c', cmd], description=msg) + image.execute(cmd, description=msg) def setup_image(args, image): -- cgit v1.2.3 From cf10a2ff2e2f666d9370f38297a5a105e809ea3c Mon Sep 17 00:00:00 2001 From: Ethan Apodaca Date: Wed, 13 Sep 2017 22:18:26 -0600 Subject: chef: Add option to pin chef omnibus install version Most users of chef will want to pin the version that is installed. Typically new versions of chef have to be evaluated for breakage etc. This change proposes a new optional `omnibus_version` field to the chef configuration. The changeset also adds documentation referencing the new field. LP: #1462693 --- cloudinit/config/cc_chef.py | 45 ++++++++---- cloudinit/util.py | 25 +++++++ doc/examples/cloud-config-chef.txt | 4 ++ tests/unittests/test_handler/test_handler_chef.py | 88 +++++++++++++++++++---- 4 files changed, 138 insertions(+), 24 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index c192dd32..46abedd1 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,6 +58,9 @@ file). log_level: log_location: node_name: + omnibus_url: + omnibus_url_retries: + omnibus_version: pid_file: server_url: show_time: @@ -71,7 +74,6 @@ import itertools import json import os -from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -280,6 +282,31 @@ def run_chef(chef_cfg, log): util.subp(cmd, capture=False) +def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): + """Install an omnibus unified package from url. + + @param url: URL where blob of chef content may be downloaded. Defaults to + OMNIBUS_URL. + @param retries: Number of retries to perform when attempting to read url. + Defaults to OMNIBUS_URL_RETRIES + @param omnibus_version: Optional version string to require for omnibus + install. + """ + if url is None: + url = OMNIBUS_URL + if retries is None: + retries = OMNIBUS_URL_RETRIES + + if omnibus_version is None: + args = [] + else: + args = ['-v', omnibus_version] + content = url_helper.readurl(url=url, retries=retries).contents + return util.subp_blob_in_tempfile( + blob=content, args=args, + basename='chef-omnibus-install', capture=False) + + def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', @@ -298,17 +325,11 @@ def install_chef(cloud, chef_cfg, log): # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) elif install_type == 'omnibus': - # This will install as a omnibus unified package - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - retries = max(0, util.get_cfg_option_int(chef_cfg, - "omnibus_url_retries", - default=OMNIBUS_URL_RETRIES)) - content = url_helper.readurl(url=url, retries=retries).contents - with temp_utils.tempdir() as tmpd: - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, content, mode=0o700) - util.subp([tmpf], capture=False) + omnibus_version = util.get_cfg_option_str(chef_cfg, "omnibus_version") + install_chef_from_omnibus( + url=util.get_cfg_option_str(chef_cfg, "omnibus_url"), + retries=util.get_cfg_option_int(chef_cfg, "omnibus_url_retries"), + omnibus_version=omnibus_version) else: log.warn("Unknown chef install type '%s'", install_type) run = False diff --git a/cloudinit/util.py b/cloudinit/util.py index ae5cda8d..7e9d94fc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1742,6 +1742,31 @@ def delete_dir_contents(dirname): del_file(node_fullpath) +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir() as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + write_file(tmpf, blob, mode=0o700) + return subp(*args, **kwargs) + + def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, logstring=False, decode="replace", target=None, update_env=None): diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 9d235817..58d5fdc7 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -94,6 +94,10 @@ chef: # if install_type is 'omnibus', change the url to download omnibus_url: "https://www.chef.io/chef/install.sh" + # if install_type is 'omnibus', pass pinned version string + # to the install script + omnibus_version: "12.3.0" + # Capture all subprocess output into a logfile # Useful for troubleshooting cloud-init issues diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index e5785cfd..0136a93d 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,11 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +import httpretty import json import logging import os -import shutil import six -import tempfile from cloudinit import cloud from cloudinit.config import cc_chef @@ -14,18 +13,83 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import ( + CiTestCase, FilesystemMockingTestCase, mock, skipIf) LOG = logging.getLogger(__name__) CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) -class TestChef(t_help.FilesystemMockingTestCase): +class TestInstallChefOmnibus(CiTestCase): + + def setUp(self): + self.new_root = self.tmp_dir() + + @httpretty.activate + def test_install_chef_from_omnibus_runs_chef_url_content(self): + """install_chef_from_omnibus runs downloaded OMNIBUS_URL as script.""" + chef_outfile = self.tmp_path('chef.out', self.new_root) + response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + httpretty.register_uri( + httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200) + cc_chef.install_chef_from_omnibus() + self.assertEqual('Hi Mom\n', util.load_file(chef_outfile)) + + @mock.patch('cloudinit.config.cc_chef.url_helper.readurl') + @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl): + """install_chef_from_omnibus retries OMNIBUS_URL upon failure.""" + + class FakeURLResponse(object): + contents = '#!/bin/bash\necho "Hi Mom" > {0}/chef.out'.format( + self.new_root) + + m_rdurl.return_value = FakeURLResponse() + + cc_chef.install_chef_from_omnibus() + expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, + 'url': cc_chef.OMNIBUS_URL} + self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + cc_chef.install_chef_from_omnibus(retries=10) + expected_kwargs = {'retries': 10, + 'url': cc_chef.OMNIBUS_URL} + self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + expected_subp_kwargs = { + 'args': ['-v', '2.0'], + 'basename': 'chef-omnibus-install', + 'blob': m_rdurl.return_value.contents, + 'capture': False + } + self.assertItemsEqual( + expected_subp_kwargs, + m_subp_blob.call_args_list[0][1]) + + @httpretty.activate + @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): + """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" + chef_outfile = self.tmp_path('chef.out', self.new_root) + response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + httpretty.register_uri( + httpretty.GET, cc_chef.OMNIBUS_URL, body=response) + cc_chef.install_chef_from_omnibus(omnibus_version='2.0') + + called_kwargs = m_subp_blob.call_args_list[0][1] + expected_kwargs = { + 'args': ['-v', '2.0'], + 'basename': 'chef-omnibus-install', + 'blob': response, + 'capture': False + } + self.assertItemsEqual(expected_kwargs, called_kwargs) + + +class TestChef(FilesystemMockingTestCase): + def setUp(self): super(TestChef, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) + self.tmp = self.tmp_dir() def fetch_cloud(self, distro_kind): cls = distros.fetch(distro_kind) @@ -43,8 +107,8 @@ class TestChef(t_help.FilesystemMockingTestCase): for d in cc_chef.CHEF_DIRS: self.assertFalse(os.path.isdir(d)) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_basic_config(self): """ test basic config looks sane @@ -122,8 +186,8 @@ class TestChef(t_help.FilesystemMockingTestCase): 'c': 'd', }, json.loads(c)) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_template_deletes(self): tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) @@ -143,8 +207,8 @@ class TestChef(t_help.FilesystemMockingTestCase): self.assertNotIn('json_attribs', c) self.assertNotIn('Formatter.show_time', c) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_validation_cert_and_validation_key(self): # test validation_cert content is written to validation_key path tpl_file = util.load_file('templates/chef_client.rb.tmpl') -- cgit v1.2.3 From f761f2b5f58c8cf13cfee63619f32046216cf66a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 13 Sep 2017 22:26:03 -0600 Subject: cloud-config modules: honor distros definitions in each module Modules can optionally define a list of supported distros on which they can run by declaring a distros attribute in the cc_*py module. This branch fixes handling of cloudinit.stages.Modules.run_section. The behavior of run_section is now the following: - always run a module if the module doesn't declare a distros attribute - always run a module if the module declares distros = [ALL_DISTROS] - skip a module if the distribution on which we run isn't in module.distros - force a run of a skipped module if unverified_modules configuration contains the module name LP: #1715738 LP: #1715690 --- cloudinit/config/cc_runcmd.py | 3 +- cloudinit/distros/__init__.py | 4 + cloudinit/stages.py | 33 ++++--- tests/unittests/test_runs/test_simple_run.py | 125 ++++++++++++++++++++++----- 4 files changed, 131 insertions(+), 34 deletions(-) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 7f995693..449872f0 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -10,6 +10,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) +from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -23,7 +24,7 @@ from textwrap import dedent # configuration options before actually attempting to deploy with said # configuration. -distros = ['all'] +distros = [ALL_DISTROS] schema = { 'id': 'cc_runcmd', diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index b714b9ab..d5becd12 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -30,6 +30,10 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +# Used when a cloud-config module can be run on all cloud-init distibutions. +# The value 'all' is surfaced in module documentation for distro support. +ALL_DISTROS = 'all' + OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['centos', 'fedora', 'rhel'], diff --git a/cloudinit/stages.py b/cloudinit/stages.py index a1c4a517..d0452688 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -821,28 +821,35 @@ class Modules(object): skipped = [] forced = [] overridden = self.cfg.get('unverified_modules', []) + active_mods = [] + all_distros = set([distros.ALL_DISTROS]) for (mod, name, _freq, _args) in mostly_mods: - worked_distros = set(mod.distros) + worked_distros = set(mod.distros) # Minimally [] per fixup_modules worked_distros.update( distros.Distro.expand_osfamily(mod.osfamilies)) - # module does not declare 'distros' or lists this distro - if not worked_distros or d_name in worked_distros: - continue - - if name in overridden: - forced.append(name) - else: - skipped.append(name) + # Skip only when the following conditions are all met: + # - distros are defined in the module != ALL_DISTROS + # - the current d_name isn't in distros + # - and the module is unverified and not in the unverified_modules + # override list + if worked_distros and worked_distros != all_distros: + if d_name not in worked_distros: + if name not in overridden: + skipped.append(name) + continue + forced.append(name) + active_mods.append([mod, name, _freq, _args]) if skipped: - LOG.info("Skipping modules %s because they are not verified " + LOG.info("Skipping modules '%s' because they are not verified " "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) + "'unverified_modules' in config.", + ','.join(skipped), d_name) if forced: - LOG.info("running unverified_modules: %s", forced) + LOG.info("running unverified_modules: '%s'", ', '.join(forced)) - return self._run_modules(mostly_mods) + return self._run_modules(active_mods) def read_runtime_config(): diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 5cf666fe..b8fb4794 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import shutil -import tempfile from cloudinit.tests import helpers @@ -12,16 +10,19 @@ from cloudinit import util class TestSimpleRun(helpers.FilesystemMockingTestCase): - def _patchIn(self, root): - self.patchOS(root) - self.patchUtils(root) - - def test_none_ds(self): - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.replicateTestRoot('simple_ubuntu', new_root) - cfg = { + + with_logs = True + + def setUp(self): + super(TestSimpleRun, self).setUp() + self.new_root = self.tmp_dir() + self.replicateTestRoot('simple_ubuntu', self.new_root) + + # Seed cloud.cfg file for our tests + self.cfg = { 'datasource_list': ['None'], + 'runcmd': ['ls /etc'], # test ALL_DISTROS + 'spacewalk': {}, # test non-ubuntu distros module definition 'write_files': [ { 'path': '/etc/blah.ini', @@ -29,14 +30,17 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): 'permissions': 0o755, }, ], - 'cloud_init_modules': ['write-files'], + 'cloud_init_modules': ['write-files', 'spacewalk', 'runcmd'], } - cloud_cfg = util.yaml_dumps(cfg) - util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) - util.write_file(os.path.join(new_root, 'etc', + cloud_cfg = util.yaml_dumps(self.cfg) + util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) + util.write_file(os.path.join(self.new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) - self._patchIn(new_root) + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + def test_none_ds_populates_var_lib_cloud(self): + """Init and run_section default behavior creates appropriate dirs.""" # Now start verifying whats created initer = stages.Init() initer.read_cfg() @@ -51,10 +55,16 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) - initer.cloudify().run('consume_data', - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) + def test_none_ds_runs_modules_which_do_not_define_distros(self): + """Any modules which do not define a distros attribute are run.""" + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') @@ -63,5 +73,80 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') self.assertEqual(contents, 'blah') + self.assertNotIn( + "Skipping modules ['write-files'] because they are not verified on" + " distro 'ubuntu'", + self.logs.getvalue()) + + def test_none_ds_skips_modules_which_define_unmatched_distros(self): + """Skip modules which define distros which don't match the current.""" + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertIn( + "Skipping modules 'spacewalk' because they are not verified on" + " distro 'ubuntu'", + self.logs.getvalue()) + self.assertNotIn('spacewalk', which_ran) + + def test_none_ds_runs_modules_which_distros_all(self): + """Skip modules which define distros attribute as supporting 'all'. + + This is done in the module with the declaration: + distros = [ALL_DISTROS]. runcmd is an example. + """ + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertIn('runcmd', which_ran) + self.assertNotIn( + "Skipping modules 'runcmd' because they are not verified on" + " distro 'ubuntu'", + self.logs.getvalue()) + + def test_none_ds_forces_run_via_unverified_modules(self): + """run_section forced skipped modules by using unverified_modules.""" + + # re-write cloud.cfg with unverified_modules override + self.cfg['unverified_modules'] = ['spacewalk'] # Would have skipped + cloud_cfg = util.yaml_dumps(self.cfg) + util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) + util.write_file(os.path.join(self.new_root, 'etc', + 'cloud', 'cloud.cfg'), cloud_cfg) + + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertIn('spacewalk', which_ran) + self.assertIn( + "running unverified_modules: 'spacewalk'", + self.logs.getvalue()) # vi: ts=4 expandtab -- cgit v1.2.3 From 7629702ca0956fb26d27ee19ed99306f73421c66 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Mon, 11 Sep 2017 07:50:01 -0700 Subject: vmware: Enable nics before sending the SUCCESS event. The network devices should be enabled before sending the 'SUCCESS' event to the underlying hypervisor. --- cloudinit/sources/DataSourceOVF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index aa5f798d..24b45d55 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -182,10 +182,10 @@ class DataSourceOVF(sources.DataSource): # TODO: Need to set the status to DONE only when the # customization is done successfully. + enable_nics(self._vmware_nics_to_enable) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) - enable_nics(self._vmware_nics_to_enable) else: np = {'iso': transport_iso9660, -- cgit v1.2.3 From 29a9296cd68516e76d0bd8da320754a222c4ee45 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Wed, 13 Sep 2017 11:08:56 -0400 Subject: resizefs: pass mount point to xfs_growfs Supposedly it was never a feature to be able to pass a path to a block device to xfs_growfs and have it grow the filesystem. The behavior changed upstream recently. It is only supported to pass the mount point of a mounted XFS filesystem. This causes breakages in cloud-init. Upstream xfs change was commit b97815a0321072a7154ecab63e297af84066fc78. https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git/commit/?id=b97815a0321 rhbz: rhbz: https://bugzilla.redhat.com/show_bug.cgi?id=1490505 Signed-off-by: Dusty Mabe --- cloudinit/config/cc_resizefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index f14d3836..f42b6a63 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -67,7 +67,7 @@ def _resize_ext(mount_point, devpth): def _resize_xfs(mount_point, devpth): - return ('xfs_growfs', devpth) + return ('xfs_growfs', mount_point) def _resize_ufs(mount_point, devpth): -- cgit v1.2.3 From 376168e251a1d4f2ee3643fed6092b8907f057ec Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 11 Jul 2017 14:28:11 -0700 Subject: tests: Enable the NoCloud KVM platform The NoCloud KVM platform includes: * Downloads daily Ubuntu images using streams and store in /srv/images * Image customization, if required, is done using mount-image-callback otherwise image is untouched * Launches KVM via the xkvm script, a wrapper around qemu-system, and sets custom port for SSH * Generation and inject an SSH (RSA 4096) key pair to use for communication with the guest to collect test artifacts * Add method to produce safe shell strings by base64 encoding the command Additional Changes: * Set default backend to use LXD * Verify not running script as root in order to prevent images from becoming owned by root * Removed extra quotes around that were added when collecting the cloud-init version from the image * Added info about each release as previously the lxd backend was able to query that information from pylxd image info, however, other backends will not be able to obtain the same information as easily --- tests/cloud_tests/__main__.py | 5 +- tests/cloud_tests/args.py | 4 +- tests/cloud_tests/collect.py | 3 + tests/cloud_tests/config.py | 1 + tests/cloud_tests/images/nocloudkvm.py | 88 ++++++++++++ tests/cloud_tests/instances/base.py | 2 +- tests/cloud_tests/instances/nocloudkvm.py | 216 ++++++++++++++++++++++++++++++ tests/cloud_tests/platforms.yaml | 4 + tests/cloud_tests/platforms/__init__.py | 2 + tests/cloud_tests/platforms/nocloudkvm.py | 90 +++++++++++++ tests/cloud_tests/releases.yaml | 19 ++- tests/cloud_tests/setup_image.py | 20 ++- tests/cloud_tests/snapshots/nocloudkvm.py | 74 ++++++++++ tests/cloud_tests/util.py | 43 ++++++ 14 files changed, 564 insertions(+), 7 deletions(-) create mode 100644 tests/cloud_tests/images/nocloudkvm.py create mode 100644 tests/cloud_tests/instances/nocloudkvm.py create mode 100644 tests/cloud_tests/platforms/nocloudkvm.py create mode 100644 tests/cloud_tests/snapshots/nocloudkvm.py diff --git a/tests/cloud_tests/__main__.py b/tests/cloud_tests/__main__.py index 260ddb3f..7ee29cad 100644 --- a/tests/cloud_tests/__main__.py +++ b/tests/cloud_tests/__main__.py @@ -4,6 +4,7 @@ import argparse import logging +import os import sys from tests.cloud_tests import args, bddeb, collect, manage, run_funcs, verify @@ -50,7 +51,7 @@ def main(): return -1 # run handler - LOG.debug('running with args: %s\n', parsed) + LOG.debug('running with args: %s', parsed) return { 'bddeb': bddeb.bddeb, 'collect': collect.collect, @@ -63,6 +64,8 @@ def main(): if __name__ == "__main__": + if os.geteuid() == 0: + sys.exit('Do not run as root') sys.exit(main()) # vi: ts=4 expandtab diff --git a/tests/cloud_tests/args.py b/tests/cloud_tests/args.py index 369d60db..c6c1877b 100644 --- a/tests/cloud_tests/args.py +++ b/tests/cloud_tests/args.py @@ -170,9 +170,9 @@ def normalize_collect_args(args): @param args: parsed args @return_value: updated args, or None if errors occurred """ - # platform should default to all supported + # platform should default to lxd if len(args.platform) == 0: - args.platform = config.ENABLED_PLATFORMS + args.platform = ['lxd'] args.platform = util.sorted_unique(args.platform) # os name should default to all enabled diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index b44e8bdd..4a2422ed 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -120,6 +120,7 @@ def collect_image(args, platform, os_name): os_config = config.load_os_config( platform.platform_name, os_name, require_enabled=True, feature_overrides=args.feature_override) + LOG.debug('os config: %s', os_config) component = PlatformComponent( partial(images.get_image, platform, os_config)) @@ -144,6 +145,8 @@ def collect_platform(args, platform_name): platform_config = config.load_platform_config( platform_name, require_enabled=True) + platform_config['data_dir'] = args.data_dir + LOG.debug('platform config: %s', platform_config) component = PlatformComponent( partial(platforms.get_platform, platform_name, platform_config)) diff --git a/tests/cloud_tests/config.py b/tests/cloud_tests/config.py index 4d5dc801..52fc2bda 100644 --- a/tests/cloud_tests/config.py +++ b/tests/cloud_tests/config.py @@ -112,6 +112,7 @@ def load_os_config(platform_name, os_name, require_enabled=False, feature_conf = main_conf['features'] feature_groups = conf.get('feature_groups', []) overrides = merge_config(get(conf, 'features'), feature_overrides) + conf['arch'] = c_util.get_architecture() conf['features'] = merge_feature_groups( feature_conf, feature_groups, overrides) diff --git a/tests/cloud_tests/images/nocloudkvm.py b/tests/cloud_tests/images/nocloudkvm.py new file mode 100644 index 00000000..a7af0e59 --- /dev/null +++ b/tests/cloud_tests/images/nocloudkvm.py @@ -0,0 +1,88 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""NoCloud KVM Image Base Class.""" + +from tests.cloud_tests.images import base +from tests.cloud_tests.snapshots import nocloudkvm as nocloud_kvm_snapshot + + +class NoCloudKVMImage(base.Image): + """NoCloud KVM backed image.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, config, img_path): + """Set up image. + + @param platform: platform object + @param config: image configuration + @param img_path: path to the image + """ + self.modified = False + self._instance = None + self._img_path = img_path + + super(NoCloudKVMImage, self).__init__(platform, config) + + @property + def instance(self): + """Returns an instance of an image.""" + if not self._instance: + if not self._img_path: + raise RuntimeError() + + self._instance = self.platform.create_image( + self.properties, self.config, self.features, self._img_path, + image_desc=str(self), use_desc='image-modification') + return self._instance + + @property + def properties(self): + """Dictionary containing: 'arch', 'os', 'version', 'release'.""" + return { + 'arch': self.config['arch'], + 'os': self.config['family'], + 'release': self.config['release'], + 'version': self.config['version'], + } + + def execute(self, *args, **kwargs): + """Execute command in image, modifying image.""" + return self.instance.execute(*args, **kwargs) + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'.""" + return self.instance.push_file(local_path, remote_path) + + def run_script(self, *args, **kwargs): + """Run script in image, modifying image. + + @return_value: script output + """ + return self.instance.run_script(*args, **kwargs) + + def snapshot(self): + """Create snapshot of image, block until done.""" + if not self._img_path: + raise RuntimeError() + + instance = self.platform.create_image( + self.properties, self.config, self.features, + self._img_path, image_desc=str(self), use_desc='snapshot') + + return nocloud_kvm_snapshot.NoCloudKVMSnapshot( + self.platform, self.properties, self.config, + self.features, instance) + + def destroy(self): + """Unset path to signal image is no longer used. + + The removal of the images and all other items is handled by the + framework. In some cases we want to keep the images, so let the + framework decide whether to keep or destroy everything. + """ + self._img_path = None + self._instance.destroy() + super(NoCloudKVMImage, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/base.py b/tests/cloud_tests/instances/base.py index 58f45b14..9bdda608 100644 --- a/tests/cloud_tests/instances/base.py +++ b/tests/cloud_tests/instances/base.py @@ -90,7 +90,7 @@ class Instance(object): return self.execute( ['/bin/bash', script_path], rcs=rcs, description=description) finally: - self.execute(['rm', script_path], rcs=rcs) + self.execute(['rm', '-f', script_path], rcs=rcs) def tmpfile(self): """Get a tmp file in the target. diff --git a/tests/cloud_tests/instances/nocloudkvm.py b/tests/cloud_tests/instances/nocloudkvm.py new file mode 100644 index 00000000..7abfe737 --- /dev/null +++ b/tests/cloud_tests/instances/nocloudkvm.py @@ -0,0 +1,216 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM instance.""" + +import os +import paramiko +import shlex +import socket +import subprocess +import time + +from cloudinit import util as c_util +from tests.cloud_tests.instances import base +from tests.cloud_tests import util + + +class NoCloudKVMInstance(base.Instance): + """NoCloud KVM backed instance.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, name, properties, config, features, + user_data, meta_data): + """Set up instance. + + @param platform: platform object + @param name: image path + @param properties: dictionary of properties + @param config: dictionary of configuration values + @param features: dictionary of supported feature flags + """ + self.user_data = user_data + self.meta_data = meta_data + self.ssh_key_file = os.path.join(platform.config['data_dir'], + platform.config['private_key']) + self.ssh_port = None + self.pid = None + self.pid_file = None + + super(NoCloudKVMInstance, self).__init__( + platform, name, properties, config, features) + + def destroy(self): + """Clean up instance.""" + if self.pid: + try: + c_util.subp(['kill', '-9', self.pid]) + except util.ProcessExectuionError: + pass + + if self.pid_file: + os.remove(self.pid_file) + + self.pid = None + super(NoCloudKVMInstance, self).destroy() + + def execute(self, command, stdout=None, stderr=None, env=None, + rcs=None, description=None): + """Execute command in instance. + + Assumes functional networking and execution as root with the + target filesystem being available at /. + + @param command: the command to execute as root inside the image + if command is a string, then it will be executed as: + ['sh', '-c', command] + @param stdout, stderr: file handles to write output and error to + @param env: environment variables + @param rcs: allowed return codes from command + @param description: purpose of command + @return_value: tuple containing stdout data, stderr data, exit code + """ + if env is None: + env = {} + + if isinstance(command, str): + command = ['sh', '-c', command] + + if self.pid: + return self.ssh(command) + else: + return self.mount_image_callback(command) + (0,) + + def mount_image_callback(self, cmd): + """Run mount-image-callback.""" + mic = ('sudo mount-image-callback --system-mounts --system-resolvconf ' + '%s -- chroot _MOUNTPOINT_ ' % self.name) + + out, err = c_util.subp(shlex.split(mic) + cmd) + + return out, err + + def generate_seed(self, tmpdir): + """Generate nocloud seed from user-data""" + seed_file = os.path.join(tmpdir, '%s_seed.img' % self.name) + user_data_file = os.path.join(tmpdir, '%s_user_data' % self.name) + + with open(user_data_file, "w") as ud_file: + ud_file.write(self.user_data) + + c_util.subp(['cloud-localds', seed_file, user_data_file]) + + return seed_file + + def get_free_port(self): + """Get a free port assigned by the kernel.""" + s = socket.socket() + s.bind(('', 0)) + num = s.getsockname()[1] + s.close() + return num + + def push_file(self, local_path, remote_path): + """Copy file at 'local_path' to instance at 'remote_path'. + + If we have a pid then SSH is up, otherwise, use + mount-image-callback. + + @param local_path: path on local instance + @param remote_path: path on remote instance + """ + if self.pid: + super(NoCloudKVMInstance, self).push_file() + else: + cmd = ("sudo mount-image-callback --system-mounts " + "--system-resolvconf %s -- chroot _MOUNTPOINT_ " + "/bin/sh -c 'cat - > %s'" % (self.name, remote_path)) + local_file = open(local_path) + p = subprocess.Popen(shlex.split(cmd), + stdin=local_file, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + p.wait() + + def sftp_put(self, path, data): + """SFTP put a file.""" + client = self._ssh_connect() + sftp = client.open_sftp() + + with sftp.open(path, 'w') as f: + f.write(data) + + client.close() + + def ssh(self, command): + """Run a command via SSH.""" + client = self._ssh_connect() + + try: + _, out, err = client.exec_command(util.shell_pack(command)) + except paramiko.SSHException: + raise util.InTargetExecuteError('', '', -1, command, self.name) + + exit = out.channel.recv_exit_status() + out = ''.join(out.readlines()) + err = ''.join(err.readlines()) + client.close() + + return out, err, exit + + def _ssh_connect(self, hostname='localhost', username='ubuntu', + banner_timeout=120, retry_attempts=30): + """Connect via SSH.""" + private_key = paramiko.RSAKey.from_private_key_file(self.ssh_key_file) + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + while retry_attempts: + try: + client.connect(hostname=hostname, username=username, + port=self.ssh_port, pkey=private_key, + banner_timeout=banner_timeout) + return client + except (paramiko.SSHException, TypeError): + time.sleep(1) + retry_attempts = retry_attempts - 1 + + error_desc = 'Failed command to: %s@%s:%s' % (username, hostname, + self.ssh_port) + raise util.InTargetExecuteError('', '', -1, 'ssh connect', + self.name, error_desc) + + def start(self, wait=True, wait_for_cloud_init=False): + """Start instance.""" + tmpdir = self.platform.config['data_dir'] + seed = self.generate_seed(tmpdir) + self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name) + self.ssh_port = self.get_free_port() + + cmd = ('./tools/xkvm --disk %s,cache=unsafe --disk %s,cache=unsafe ' + '--netdev user,hostfwd=tcp::%s-:22 ' + '-- -pidfile %s -vnc none -m 2G -smp 2' + % (self.name, seed, self.ssh_port, self.pid_file)) + + subprocess.Popen(shlex.split(cmd), close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + while not os.path.exists(self.pid_file): + time.sleep(1) + + with open(self.pid_file, 'r') as pid_f: + self.pid = pid_f.readlines()[0].strip() + + if wait: + self._wait_for_system(wait_for_cloud_init) + + def write_data(self, remote_path, data): + """Write data to instance filesystem. + + @param remote_path: path in instance + @param data: data to write, either str or bytes + """ + self.sftp_put(remote_path, data) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/platforms.yaml b/tests/cloud_tests/platforms.yaml index b91834ab..fa4f845e 100644 --- a/tests/cloud_tests/platforms.yaml +++ b/tests/cloud_tests/platforms.yaml @@ -59,6 +59,10 @@ platforms: {{ config_get("user.user-data", properties.default) }} cloud-init-vendor.tpl: | {{ config_get("user.vendor-data", properties.default) }} + nocloud-kvm: + enabled: true + private_key: id_rsa + public_key: id_rsa.pub ec2: {} azure: {} diff --git a/tests/cloud_tests/platforms/__init__.py b/tests/cloud_tests/platforms/__init__.py index 443f6d44..3490fe87 100644 --- a/tests/cloud_tests/platforms/__init__.py +++ b/tests/cloud_tests/platforms/__init__.py @@ -3,8 +3,10 @@ """Main init.""" from tests.cloud_tests.platforms import lxd +from tests.cloud_tests.platforms import nocloudkvm PLATFORMS = { + 'nocloud-kvm': nocloudkvm.NoCloudKVMPlatform, 'lxd': lxd.LXDPlatform, } diff --git a/tests/cloud_tests/platforms/nocloudkvm.py b/tests/cloud_tests/platforms/nocloudkvm.py new file mode 100644 index 00000000..f1f81877 --- /dev/null +++ b/tests/cloud_tests/platforms/nocloudkvm.py @@ -0,0 +1,90 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM platform.""" +import glob +import os + +from simplestreams import filters +from simplestreams import mirrors +from simplestreams import objectstores +from simplestreams import util as s_util + +from cloudinit import util as c_util +from tests.cloud_tests.images import nocloudkvm as nocloud_kvm_image +from tests.cloud_tests.instances import nocloudkvm as nocloud_kvm_instance +from tests.cloud_tests.platforms import base +from tests.cloud_tests import util + + +class NoCloudKVMPlatform(base.Platform): + """NoCloud KVM test platform.""" + + platform_name = 'nocloud-kvm' + + def get_image(self, img_conf): + """Get image using specified image configuration. + + @param img_conf: configuration for image + @return_value: cloud_tests.images instance + """ + (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) + + filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), + 'release=%s' % img_conf['release'], + 'ftype=disk1.img']) + mirror_config = {'filters': filter, + 'keep_items': False, + 'max_items': 1, + 'checksumming_reader': True, + 'item_download': True + } + + def policy(content, path): + return s_util.read_signed(content, keyring=img_conf['keyring']) + + smirror = mirrors.UrlMirrorReader(url, policy=policy) + tstore = objectstores.FileStore(img_conf['mirror_dir']) + tmirror = mirrors.ObjectFilterMirror(config=mirror_config, + objectstore=tstore) + tmirror.sync(smirror, path) + + search_d = os.path.join(img_conf['mirror_dir'], '**', + img_conf['release'], '**', '*.img') + + images = [] + for fname in glob.iglob(search_d, recursive=True): + images.append(fname) + + if len(images) != 1: + raise Exception('No unique images found') + + image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) + if img_conf.get('override_templates', False): + image.update_templates(self.config.get('template_overrides', {}), + self.config.get('template_files', {})) + return image + + def create_image(self, properties, config, features, + src_img_path, image_desc=None, use_desc=None, + user_data=None, meta_data=None): + """Create an image + + @param src_img_path: image path to launch from + @param properties: image properties + @param config: image configuration + @param features: image features + @param image_desc: description of image being launched + @param use_desc: description of container's use + @return_value: cloud_tests.instances instance + """ + name = util.gen_instance_name(image_desc=image_desc, use_desc=use_desc) + img_path = os.path.join(self.config['data_dir'], name + '.qcow2') + c_util.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', src_img_path, img_path]) + + return nocloud_kvm_instance.NoCloudKVMInstance(self, img_path, + properties, config, + features, user_data, + meta_data) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index c8dd1427..ec7e2d5b 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -27,7 +27,12 @@ default_release_config: # features groups and additional feature settings feature_groups: [] features: {} - + nocloud-kvm: + mirror_url: https://cloud-images.ubuntu.com/daily + mirror_dir: '/srv/citest/nocloud-kvm' + keyring: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg + setup_overrides: null + override_templates: false # lxd specific default configuration options lxd: # default sstreams server to use for lxd image retrieval @@ -121,6 +126,9 @@ releases: # EOL: Jul 2018 default: enabled: true + release: artful + version: 17.10 + family: ubuntu feature_groups: - base - debian_base @@ -134,6 +142,9 @@ releases: # EOL: Jan 2018 default: enabled: true + release: zesty + version: 17.04 + family: ubuntu feature_groups: - base - debian_base @@ -147,6 +158,9 @@ releases: # EOL: Apr 2021 default: enabled: true + release: xenial + version: 16.04 + family: ubuntu feature_groups: - base - debian_base @@ -160,6 +174,9 @@ releases: # EOL: Apr 2019 default: enabled: true + release: trusty + version: 14.04 + family: ubuntu feature_groups: - base - debian_base diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 3c0fff62..6672ffb3 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -5,6 +5,7 @@ from functools import partial import os +from cloudinit import util as c_util from tests.cloud_tests import LOG from tests.cloud_tests import stage, util @@ -19,7 +20,7 @@ def installed_package_version(image, package, ensure_installed=True): """ os_family = util.get_os_family(image.properties['os']) if os_family == 'debian': - cmd = ['dpkg-query', '-W', "--showformat='${Version}'", package] + cmd = ['dpkg-query', '-W', "--showformat=${Version}", package] elif os_family == 'redhat': cmd = ['rpm', '-q', '--queryformat', "'%{VERSION}'", package] else: @@ -53,7 +54,7 @@ def install_deb(args, image): image.execute(cmd, description=msg) # check installed deb version matches package - fmt = ['-W', "--showformat='${Version}'"] + fmt = ['-W', "--showformat=${Version}"] (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path]) expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') @@ -191,6 +192,20 @@ def enable_repo(args, image): image.execute(cmd, description=msg) +def generate_ssh_keys(data_dir): + """Generate SSH keys to be used with image.""" + LOG.info('generating SSH keys') + filename = os.path.join(data_dir, 'id_rsa') + + if os.path.exists(filename): + c_util.del_file(filename) + + c_util.subp(['ssh-keygen', '-t', 'rsa', '-b', '4096', + '-f', filename, '-P', '', + '-C', 'ubuntu@cloud_test'], + capture=True) + + def setup_image(args, image): """Set up image as specified in args. @@ -226,6 +241,7 @@ def setup_image(args, image): 'set up for {}'.format(image), calls, continue_after_error=False) LOG.debug('after setup complete, installed cloud-init version is: %s', installed_package_version(image, 'cloud-init')) + generate_ssh_keys(args.data_dir) return res # vi: ts=4 expandtab diff --git a/tests/cloud_tests/snapshots/nocloudkvm.py b/tests/cloud_tests/snapshots/nocloudkvm.py new file mode 100644 index 00000000..09998349 --- /dev/null +++ b/tests/cloud_tests/snapshots/nocloudkvm.py @@ -0,0 +1,74 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Base NoCloud KVM snapshot.""" +import os + +from tests.cloud_tests.snapshots import base + + +class NoCloudKVMSnapshot(base.Snapshot): + """NoCloud KVM image copy backed snapshot.""" + + platform_name = "nocloud-kvm" + + def __init__(self, platform, properties, config, features, + instance): + """Set up snapshot. + + @param platform: platform object + @param properties: image properties + @param config: image config + @param features: supported feature flags + """ + self.instance = instance + + super(NoCloudKVMSnapshot, self).__init__( + platform, properties, config, features) + + def launch(self, user_data, meta_data=None, block=True, start=True, + use_desc=None): + """Launch instance. + + @param user_data: user-data for the instance + @param instance_id: instance-id for the instance + @param block: wait until instance is created + @param start: start instance and wait until fully started + @param use_desc: description of snapshot instance use + @return_value: an Instance + """ + key_file = os.path.join(self.platform.config['data_dir'], + self.platform.config['public_key']) + user_data = self.inject_ssh_key(user_data, key_file) + + instance = self.platform.create_image( + self.properties, self.config, self.features, + self.instance.name, image_desc=str(self), use_desc=use_desc, + user_data=user_data, meta_data=meta_data) + + if start: + instance.start() + + return instance + + def inject_ssh_key(self, user_data, key_file): + """Inject the authorized key into the user_data.""" + with open(key_file) as f: + value = f.read() + + key = 'ssh_authorized_keys:' + value = ' - %s' % value.strip() + user_data = user_data.split('\n') + if key in user_data: + user_data.insert(user_data.index(key) + 1, '%s' % value) + else: + user_data.insert(-1, '%s' % key) + user_data.insert(-1, '%s' % value) + + return '\n'.join(user_data) + + def destroy(self): + """Clean up snapshot data.""" + self.instance.destroy() + super(NoCloudKVMSnapshot, self).destroy() + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 2bbe21c7..4357fbb0 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -2,12 +2,14 @@ """Utilities for re-use across integration tests.""" +import base64 import copy import glob import os import random import shutil import string +import subprocess import tempfile import yaml @@ -242,6 +244,47 @@ def update_user_data(user_data, updates, dump_to_yaml=True): if dump_to_yaml else user_data) +def shell_safe(cmd): + """Produce string safe shell string. + + Create a string that can be passed to: + set -- + to produce the same array that cmd represents. + + Internally we utilize 'getopt's ability/knowledge on how to quote + strings to be safe for shell. This implementation could be changed + to be pure python. It is just a matter of correctly escaping + or quoting characters like: ' " ^ & $ ; ( ) ... + + @param cmd: command as a list + """ + out = subprocess.check_output( + ["getopt", "--shell", "sh", "--options", "", "--", "--"] + list(cmd)) + # out contains ' -- \n'. drop the ' -- ' and the '\n' + return out[4:-1].decode() + + +def shell_pack(cmd): + """Return a string that can shuffled through 'sh' and execute cmd. + + In Python subprocess terms: + check_output(cmd) == check_output(shell_pack(cmd), shell=True) + + @param cmd: list or string of command to pack up + """ + + if isinstance(cmd, str): + cmd = [cmd] + else: + cmd = list(cmd) + + stuffed = shell_safe(cmd) + # for whatever reason b64encode returns bytes when it is clearly + # representable as a string by nature of being base64 encoded. + b64 = base64.b64encode(stuffed.encode()).decode() + return 'eval set -- "$(echo %s | base64 --decode)" && exec "$@"' % b64 + + class InTargetExecuteError(c_util.ProcessExecutionError): """Error type for in target commands that fail.""" -- cgit v1.2.3 From a2f8ce9c80debdb788e7ab37401aa98c2c270f26 Mon Sep 17 00:00:00 2001 From: Balint Reczey Date: Fri, 15 Sep 2017 17:50:52 +0200 Subject: Do not provide systemd-fsck drop-in which could cause ordering cycles. Revert "centos: do not package systemd-fsck drop-in." Revert "systemd: make systemd-fsck run after cloud-init.service" The systemd-fsck drop-in caused regressions by introducing ordering The change reverts the original commit that added systemd-fsck drop-in and another commit that had removed that from the centos packaging: 1f5489c258a26f4e26261c40786537951d67df1e 8a5296c41db45be3a172862f324ad44e732a2250 The result is to no longer provide the systemd-fsck drop-in. LP: #1717477 --- packages/redhat/cloud-init.spec.in | 6 ------ setup.py | 4 ---- systemd/systemd-fsck@.service.d/cloud-init.conf | 2 -- 3 files changed, 12 deletions(-) delete mode 100644 systemd/systemd-fsck@.service.d/cloud-init.conf diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index d995b85f..6ab0d20b 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -115,12 +115,6 @@ rm -rf $RPM_BUILD_ROOT%{python_sitelib}/tests mkdir -p $RPM_BUILD_ROOT/%{_sharedstatedir}/cloud mkdir -p $RPM_BUILD_ROOT/%{_libexecdir}/%{name} -# LP: #1691489: Remove systemd-fsck dropin (currently not expected to work) -%if "%{init_system}" == "systemd" -rm $RPM_BUILD_ROOT/usr/lib/systemd/system/systemd-fsck@.service.d/cloud-init.conf -%endif - - %clean rm -rf $RPM_BUILD_ROOT diff --git a/setup.py b/setup.py index 7662bd8b..91993174 100755 --- a/setup.py +++ b/setup.py @@ -125,7 +125,6 @@ INITSYS_FILES = { for f in (glob('systemd/*.tmpl') + glob('systemd/*.service') + glob('systemd/*.target')) if is_f(f)], - 'systemd.fsck-dropin': ['systemd/systemd-fsck@.service.d/cloud-init.conf'], 'systemd.generators': [f for f in glob('systemd/*-generator') if is_f(f)], 'upstart': [f for f in glob('upstart/*') if is_f(f)], } @@ -135,9 +134,6 @@ INITSYS_ROOTS = { 'sysvinit_deb': 'etc/init.d', 'sysvinit_openrc': 'etc/init.d', 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), - 'systemd.fsck-dropin': ( - os.path.sep.join([pkg_config_read('systemd', 'systemdsystemunitdir'), - 'systemd-fsck@.service.d'])), 'systemd.generators': pkg_config_read('systemd', 'systemdsystemgeneratordir'), 'upstart': 'etc/init/', diff --git a/systemd/systemd-fsck@.service.d/cloud-init.conf b/systemd/systemd-fsck@.service.d/cloud-init.conf deleted file mode 100644 index 0bfa465b..00000000 --- a/systemd/systemd-fsck@.service.d/cloud-init.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Unit] -After=cloud-init.service -- cgit v1.2.3 From 024ecc1f758d36cc0aa5ebce65704eed6bd66d45 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Sep 2017 14:05:13 -0600 Subject: resizefs: Drop check for read-only device file, do not warn on overlayroot. As root user, os.access(, os.W_OK) will always return True so that path will never get executed. Also avoid a warning if the root is overlayroot, which is the common case on a MAAS booted 'ephemeral' system. --- cloudinit/config/cc_resizefs.py | 12 ++-- .../test_handler/test_handler_resizefs.py | 72 +++++++--------------- 2 files changed, 26 insertions(+), 58 deletions(-) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index f42b6a63..f774baa3 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -192,6 +192,10 @@ def is_device_path_writable_block(devpath, info, log): return False log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) + if devpath == 'overlayroot': + log.debug("Not attempting to resize devpath '%s': %s", devpath, info) + return False + try: statret = os.stat(devpath) except OSError as exc: @@ -205,14 +209,6 @@ def is_device_path_writable_block(devpath, info, log): raise exc return False - if not os.access(devpath, os.W_OK): - if container: - log.debug("'%s' not writable in container. cannot resize: %s", - devpath, info) - else: - log.warn("'%s' not writable. cannot resize: %s", devpath, info) - return - if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 76dddbf8..3e5d436c 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -166,6 +166,18 @@ class TestIsDevicePathWritableBlock(CiTestCase): with_logs = True + def test_is_device_path_writable_block_false_on_overlayroot(self): + """When devpath is overlayroot (on MAAS), is_dev_writable is False.""" + info = 'does not matter' + is_writable = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, 'overlayroot', info, LOG) + self.assertFalse(is_writable) + self.assertIn( + "Not attempting to resize devpath 'overlayroot'", + self.logs.getvalue()) + def test_is_device_path_writable_block_warns_missing_cmdline_root(self): """When root does not exist isn't in the cmdline, log warning.""" info = 'does not matter' @@ -178,24 +190,24 @@ class TestIsDevicePathWritableBlock(CiTestCase): exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' with mock.patch(exists_mock_path) as m_exists: m_exists.return_value = False - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}, 'get_mount_info': {'side_effect': fake_mount_info}, 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, is_device_path_writable_block, '/dev/root', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) logs = self.logs.getvalue() self.assertIn("WARNING: Unable to find device '/dev/root'", logs) def test_is_device_path_writable_block_does_not_exist(self): """When devpath does not exist, a warning is logged.""" info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}}, is_device_path_writable_block, '/I/dont/exist', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "WARNING: Device '/I/dont/exist' did not exist." ' cannot resize: %s' % info, @@ -204,11 +216,11 @@ class TestIsDevicePathWritableBlock(CiTestCase): def test_is_device_path_writable_block_does_not_exist_in_container(self): """When devpath does not exist in a container, log a debug message.""" info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': True}}, is_device_path_writable_block, '/I/dont/exist', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "DEBUG: Device '/I/dont/exist' did not exist in container." ' cannot resize: %s' % info, @@ -226,57 +238,17 @@ class TestIsDevicePathWritableBlock(CiTestCase): self.assertEqual( 'Something unexpected', str(context_manager.exception)) - def test_is_device_path_writable_block_readonly(self): - """When root device is readonly, emit a warning and return False.""" - fake_devpath = self.tmp_path('dev/readonly') - util.write_file(fake_devpath, '', mode=0o400) # read-only - info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - - exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' - with mock.patch(exists_mock_path) as m_exists: - m_exists.return_value = False - is_valid = wrap_and_call( - 'cloudinit.config.cc_resizefs.util', - {'is_container': {'return_value': False}, - 'rootdev_from_cmdline': {'return_value': fake_devpath}}, - is_device_path_writable_block, '/dev/root', info, LOG) - self.assertFalse(is_valid) - logs = self.logs.getvalue() - self.assertIn( - "Converted /dev/root to '{0}' per kernel cmdline".format( - fake_devpath), - logs) - self.assertIn( - "WARNING: '{0}' not writable. cannot resize".format(fake_devpath), - logs) - - def test_is_device_path_writable_block_readonly_in_container(self): - """When root device is readonly, emit debug log and return False.""" - fake_devpath = self.tmp_path('dev/readonly') - util.write_file(fake_devpath, '', mode=0o400) # read-only - info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - - is_valid = wrap_and_call( - 'cloudinit.config.cc_resizefs.util', - {'is_container': {'return_value': True}}, - is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) - self.assertIn( - "DEBUG: '{0}' not writable in container. cannot resize".format( - fake_devpath), - self.logs.getvalue()) - def test_is_device_path_writable_block_non_block(self): """When device is not a block device, emit warning return False.""" fake_devpath = self.tmp_path('dev/readwrite') util.write_file(fake_devpath, '', mode=0o600) # read-write info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}}, is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "WARNING: device '{0}' not a block device. cannot resize".format( fake_devpath), @@ -288,11 +260,11 @@ class TestIsDevicePathWritableBlock(CiTestCase): util.write_file(fake_devpath, '', mode=0o600) # read-write info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': True}}, is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "DEBUG: device '{0}' not a block device in container." ' cannot resize'.format(fake_devpath), -- cgit v1.2.3 From da1db792b2721d94ef85df8c136e78012c49c6e5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 15 Sep 2017 13:37:55 -0600 Subject: CloudStack: consider dhclient lease files named with a hyphen. A regression in 'get_latest_lease' made it ignore files starting with 'dhclient-' rather than just 'dhclient.'. The fix here is to allow those files to be considered. There is a lot more we could do here to better ensure that we pick the most recent lease, but this change fixes the regression. LP: #1717147 --- cloudinit/sources/DataSourceCloudStack.py | 34 +++++++--- tests/unittests/test_datasource/test_cloudstack.py | 79 +++++++++++++++++++++- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 0188d894..7e0f9bb8 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -187,22 +187,36 @@ def get_dhclient_d(): return None -def get_latest_lease(): +def get_latest_lease(lease_d=None): # find latest lease file - lease_d = get_dhclient_d() + if lease_d is None: + lease_d = get_dhclient_d() if not lease_d: return None lease_files = os.listdir(lease_d) latest_mtime = -1 latest_file = None - for file_name in lease_files: - if file_name.startswith("dhclient.") and \ - (file_name.endswith(".lease") or file_name.endswith(".leases")): - abs_path = os.path.join(lease_d, file_name) - mtime = os.path.getmtime(abs_path) - if mtime > latest_mtime: - latest_mtime = mtime - latest_file = abs_path + + # lease files are named inconsistently across distros. + # We assume that 'dhclient6' indicates ipv6 and ignore it. + # ubuntu: + # dhclient..leases, dhclient.leases, dhclient6.leases + # centos6: + # dhclient-.leases, dhclient6.leases + # centos7: ('--' is not a typo) + # dhclient--.lease, dhclient6.leases + for fname in lease_files: + if fname.startswith("dhclient6"): + # avoid files that start with dhclient6 assuming dhcpv6. + continue + if not (fname.endswith(".lease") or fname.endswith(".leases")): + continue + + abs_path = os.path.join(lease_d, fname) + mtime = os.path.getmtime(abs_path) + if mtime > latest_mtime: + latest_mtime = mtime + latest_file = abs_path return latest_file diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 2dc90305..8e98e1bb 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -1,12 +1,17 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import helpers -from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack +from cloudinit import util +from cloudinit.sources.DataSourceCloudStack import ( + DataSourceCloudStack, get_latest_lease) -from cloudinit.tests.helpers import TestCase, mock, ExitStack +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock +import os +import time -class TestCloudStackPasswordFetching(TestCase): + +class TestCloudStackPasswordFetching(CiTestCase): def setUp(self): super(TestCloudStackPasswordFetching, self).setUp() @@ -89,4 +94,72 @@ class TestCloudStackPasswordFetching(TestCase): def test_password_not_saved_if_bad_request(self): self._check_password_not_saved_for('bad_request') + +class TestGetLatestLease(CiTestCase): + + def _populate_dir_list(self, bdir, files): + """populate_dir_list([(name, data), (name, data)]) + + writes files to bdir, and updates timestamps to ensure + that their mtime increases with each file.""" + + start = int(time.time()) + for num, fname in enumerate(reversed(files)): + fpath = os.path.sep.join((bdir, fname)) + util.write_file(fpath, fname.encode()) + os.utime(fpath, (start - num, start - num)) + + def _pop_and_test(self, files, expected): + lease_d = self.tmp_dir() + self._populate_dir_list(lease_d, files) + self.assertEqual(self.tmp_path(expected, lease_d), + get_latest_lease(lease_d)) + + def test_skips_dhcpv6_files(self): + """files started with dhclient6 should be skipped.""" + expected = "dhclient.lease" + self._pop_and_test([expected, "dhclient6.lease"], expected) + + def test_selects_dhclient_dot_files(self): + """files named dhclient.lease or dhclient.leases should be used. + + Ubuntu names files dhclient.eth0.leases dhclient6.leases and + sometimes dhclient.leases.""" + self._pop_and_test(["dhclient.lease"], "dhclient.lease") + self._pop_and_test(["dhclient.leases"], "dhclient.leases") + + def test_selects_dhclient_dash_files(self): + """files named dhclient-lease or dhclient-leases should be used. + + Redhat/Centos names files with dhclient--eth0.lease (centos 7) or + dhclient-eth0.leases (centos 6). + """ + self._pop_and_test(["dhclient-eth0.lease"], "dhclient-eth0.lease") + self._pop_and_test(["dhclient--eth0.lease"], "dhclient--eth0.lease") + + def test_ignores_by_extension(self): + """only .lease or .leases file should be considered.""" + + self._pop_and_test(["dhclient.lease", "dhclient.lease.bk", + "dhclient.lease-old", "dhclient.leaselease"], + "dhclient.lease") + + def test_selects_newest_matching(self): + """If multiple files match, the newest written should be used.""" + lease_d = self.tmp_dir() + valid_1 = "dhclient.leases" + valid_2 = "dhclient.lease" + valid_1_path = self.tmp_path(valid_1, lease_d) + valid_2_path = self.tmp_path(valid_2, lease_d) + + self._populate_dir_list(lease_d, [valid_1, valid_2]) + self.assertEqual(valid_2_path, get_latest_lease(lease_d)) + + # now update mtime on valid_2 to be older than valid_1 and re-check. + mtime = int(os.path.getmtime(valid_1_path)) - 1 + os.utime(valid_2_path, (mtime, mtime)) + + self.assertEqual(valid_1_path, get_latest_lease(lease_d)) + + # vi: ts=4 expandtab -- cgit v1.2.3 From e626966ee7d339b53d2c8b14a8f2ff8e3fe892ee Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 12 Sep 2017 10:27:07 -0600 Subject: cmdline: add collect-logs subcommand. Add a new collect-logs sub command to the cloud-init CLI. This script will collect all logs pertinent to a cloud-init run and store them in a compressed tar-gzipped file. This tarfile can be attached to any cloud-init bug filed in order to aid in bug triage and resolution. A cloudinit.apport module is also added that allows apport interaction. Here is an example bug filed via ubuntu-bug cloud-init: LP: #1716975. Once the apport launcher is packaged in cloud-init, bugs can be filed against cloud-init with the following command: ubuntu-bug cloud-init LP: #1607345 --- cloudinit/apport.py | 105 +++++++++++++++++++++++++++++ cloudinit/cmd/devel/logs.py | 101 +++++++++++++++++++++++++++ cloudinit/cmd/devel/tests/__init__.py | 0 cloudinit/cmd/devel/tests/test_logs.py | 120 +++++++++++++++++++++++++++++++++ cloudinit/cmd/main.py | 11 ++- packages/debian/rules.in | 1 + tests/unittests/test_cli.py | 22 ++++-- 7 files changed, 354 insertions(+), 6 deletions(-) create mode 100644 cloudinit/apport.py create mode 100644 cloudinit/cmd/devel/logs.py create mode 100644 cloudinit/cmd/devel/tests/__init__.py create mode 100644 cloudinit/cmd/devel/tests/test_logs.py diff --git a/cloudinit/apport.py b/cloudinit/apport.py new file mode 100644 index 00000000..221f341c --- /dev/null +++ b/cloudinit/apport.py @@ -0,0 +1,105 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +'''Cloud-init apport interface''' + +try: + from apport.hookutils import ( + attach_file, attach_root_command_outputs, root_command_output) + has_apport = True +except ImportError: + has_apport = False + + +KNOWN_CLOUD_NAMES = [ + 'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma', + 'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', 'MAAS', + 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', 'Scaleway', 'SmartOS', + 'VMware', 'Other'] + +# Potentially clear text collected logs +CLOUDINIT_LOG = '/var/log/cloud-init.log' +CLOUDINIT_OUTPUT_LOG = '/var/log/cloud-init-output.log' +USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt' # Optional + + +def attach_cloud_init_logs(report, ui=None): + '''Attach cloud-init logs and tarfile from 'cloud-init collect-logs'.''' + attach_root_command_outputs(report, { + 'cloud-init-log-warnings': + 'egrep -i "warn|error" /var/log/cloud-init.log', + 'cloud-init-output.log.txt': 'cat /var/log/cloud-init-output.log'}) + root_command_output( + ['cloud-init', 'collect-logs', '-t', '/tmp/cloud-init-logs.tgz']) + attach_file(report, '/tmp/cloud-init-logs.tgz', 'logs.tgz') + + +def attach_hwinfo(report, ui=None): + '''Optionally attach hardware info from lshw.''' + prompt = ( + 'Your device details (lshw) may be useful to developers when' + ' addressing this bug, but gathering it requires admin privileges.' + ' Would you like to include this info?') + if ui and ui.yesno(prompt): + attach_root_command_outputs(report, {'lshw.txt': 'lshw'}) + + +def attach_cloud_info(report, ui=None): + '''Prompt for cloud details if available.''' + if ui: + prompt = 'Is this machine running in a cloud environment?' + response = ui.yesno(prompt) + if response is None: + raise StopIteration # User cancelled + if response: + prompt = ('Please select the cloud vendor or environment in which' + ' this instance is running') + response = ui.choice(prompt, KNOWN_CLOUD_NAMES) + if response: + report['CloudName'] = KNOWN_CLOUD_NAMES[response[0]] + else: + report['CloudName'] = 'None' + + +def attach_user_data(report, ui=None): + '''Optionally provide user-data if desired.''' + if ui: + prompt = ( + 'Your user-data or cloud-config file can optionally be provided' + ' from {0} and could be useful to developers when addressing this' + ' bug. Do you wish to attach user-data to this bug?'.format( + USER_DATA_FILE)) + response = ui.yesno(prompt) + if response is None: + raise StopIteration # User cancelled + if response: + attach_file(report, USER_DATA_FILE, 'user_data.txt') + + +def add_bug_tags(report): + '''Add any appropriate tags to the bug.''' + if 'JournalErrors' in report.keys(): + errors = report['JournalErrors'] + if 'Breaking ordering cycle' in errors: + report['Tags'] = 'systemd-ordering' + + +def add_info(report, ui): + '''This is an entry point to run cloud-init's apport functionality. + + Distros which want apport support will have a cloud-init package-hook at + /usr/share/apport/package-hooks/cloud-init.py which defines an add_info + function and returns the result of cloudinit.apport.add_info(report, ui). + ''' + if not has_apport: + raise RuntimeError( + 'No apport imports discovered. Apport functionality disabled') + attach_cloud_init_logs(report, ui) + attach_hwinfo(report, ui) + attach_cloud_info(report, ui) + attach_user_data(report, ui) + add_bug_tags(report) + return True + +# vi: ts=4 expandtab diff --git a/cloudinit/cmd/devel/logs.py b/cloudinit/cmd/devel/logs.py new file mode 100644 index 00000000..35ca478f --- /dev/null +++ b/cloudinit/cmd/devel/logs.py @@ -0,0 +1,101 @@ +# Copyright (C) 2017 Canonical Ltd. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""Define 'collect-logs' utility and handler to include in cloud-init cmd.""" + +import argparse +from cloudinit.util import ( + ProcessExecutionError, chdir, copy, ensure_dir, subp, write_file) +from cloudinit.temp_utils import tempdir +from datetime import datetime +import os +import shutil + + +CLOUDINIT_LOGS = ['/var/log/cloud-init.log', '/var/log/cloud-init-output.log'] +CLOUDINIT_RUN_DIR = '/run/cloud-init' +USER_DATA_FILE = '/var/lib/cloud/instance/user-data.txt' # Optional + + +def get_parser(parser=None): + """Build or extend and arg parser for collect-logs utility. + + @param parser: Optional existing ArgumentParser instance representing the + collect-logs subcommand which will be extended to support the args of + this utility. + + @returns: ArgumentParser with proper argument configuration. + """ + if not parser: + parser = argparse.ArgumentParser( + prog='collect-logs', + description='Collect and tar all cloud-init debug info') + parser.add_argument( + "--tarfile", '-t', default='cloud-init.tar.gz', + help=('The tarfile to create containing all collected logs.' + ' Default: cloud-init.tar.gz')) + parser.add_argument( + "--include-userdata", '-u', default=False, action='store_true', + dest='userdata', help=( + 'Optionally include user-data from {0} which could contain' + ' sensitive information.'.format(USER_DATA_FILE))) + return parser + + +def _write_command_output_to_file(cmd, filename): + """Helper which runs a command and writes output or error to filename.""" + try: + out, _ = subp(cmd) + except ProcessExecutionError as e: + write_file(filename, str(e)) + else: + write_file(filename, out) + + +def collect_logs(tarfile, include_userdata): + """Collect all cloud-init logs and tar them up into the provided tarfile. + + @param tarfile: The path of the tar-gzipped file to create. + @param include_userdata: Boolean, true means include user-data. + """ + tarfile = os.path.abspath(tarfile) + date = datetime.utcnow().date().strftime('%Y-%m-%d') + log_dir = 'cloud-init-logs-{0}'.format(date) + with tempdir(dir='/tmp') as tmp_dir: + log_dir = os.path.join(tmp_dir, log_dir) + _write_command_output_to_file( + ['dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'], + os.path.join(log_dir, 'version')) + _write_command_output_to_file( + ['dmesg'], os.path.join(log_dir, 'dmesg.txt')) + _write_command_output_to_file( + ['journalctl', '-o', 'short-precise'], + os.path.join(log_dir, 'journal.txt')) + for log in CLOUDINIT_LOGS: + copy(log, log_dir) + if include_userdata: + copy(USER_DATA_FILE, log_dir) + run_dir = os.path.join(log_dir, 'run') + ensure_dir(run_dir) + shutil.copytree(CLOUDINIT_RUN_DIR, os.path.join(run_dir, 'cloud-init')) + with chdir(tmp_dir): + subp(['tar', 'czvf', tarfile, log_dir.replace(tmp_dir + '/', '')]) + + +def handle_collect_logs_args(name, args): + """Handle calls to 'cloud-init collect-logs' as a subcommand.""" + collect_logs(args.tarfile, args.userdata) + + +def main(): + """Tool to collect and tar all cloud-init related logs.""" + parser = get_parser() + handle_collect_logs_args('collect-logs', parser.parse_args()) + return 0 + + +if __name__ == '__main__': + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/cmd/devel/tests/__init__.py b/cloudinit/cmd/devel/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cloudinit/cmd/devel/tests/test_logs.py b/cloudinit/cmd/devel/tests/test_logs.py new file mode 100644 index 00000000..dc4947cc --- /dev/null +++ b/cloudinit/cmd/devel/tests/test_logs.py @@ -0,0 +1,120 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.cmd.devel import logs +from cloudinit.util import ensure_dir, load_file, subp, write_file +from cloudinit.tests.helpers import FilesystemMockingTestCase, wrap_and_call +from datetime import datetime +import os + + +class TestCollectLogs(FilesystemMockingTestCase): + + def setUp(self): + super(TestCollectLogs, self).setUp() + self.new_root = self.tmp_dir() + self.run_dir = self.tmp_path('run', self.new_root) + + def test_collect_logs_creates_tarfile(self): + """collect-logs creates a tarfile with all related cloud-init info.""" + log1 = self.tmp_path('cloud-init.log', self.new_root) + write_file(log1, 'cloud-init-log') + log2 = self.tmp_path('cloud-init-output.log', self.new_root) + write_file(log2, 'cloud-init-output-log') + ensure_dir(self.run_dir) + write_file(self.tmp_path('results.json', self.run_dir), 'results') + output_tarfile = self.tmp_path('logs.tgz') + + date = datetime.utcnow().date().strftime('%Y-%m-%d') + date_logdir = 'cloud-init-logs-{0}'.format(date) + + expected_subp = { + ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'): + '0.7fake\n', + ('dmesg',): 'dmesg-out\n', + ('journalctl', '-o', 'short-precise'): 'journal-out\n', + ('tar', 'czvf', output_tarfile, date_logdir): '' + } + + def fake_subp(cmd): + cmd_tuple = tuple(cmd) + if cmd_tuple not in expected_subp: + raise AssertionError( + 'Unexpected command provided to subp: {0}'.format(cmd)) + if cmd == ['tar', 'czvf', output_tarfile, date_logdir]: + subp(cmd) # Pass through tar cmd so we can check output + return expected_subp[cmd_tuple], '' + + wrap_and_call( + 'cloudinit.cmd.devel.logs', + {'subp': {'side_effect': fake_subp}, + 'CLOUDINIT_LOGS': {'new': [log1, log2]}, + 'CLOUDINIT_RUN_DIR': {'new': self.run_dir}}, + logs.collect_logs, output_tarfile, include_userdata=False) + # unpack the tarfile and check file contents + subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root]) + out_logdir = self.tmp_path(date_logdir, self.new_root) + self.assertEqual( + '0.7fake\n', + load_file(os.path.join(out_logdir, 'version'))) + self.assertEqual( + 'cloud-init-log', + load_file(os.path.join(out_logdir, 'cloud-init.log'))) + self.assertEqual( + 'cloud-init-output-log', + load_file(os.path.join(out_logdir, 'cloud-init-output.log'))) + self.assertEqual( + 'dmesg-out\n', + load_file(os.path.join(out_logdir, 'dmesg.txt'))) + self.assertEqual( + 'journal-out\n', + load_file(os.path.join(out_logdir, 'journal.txt'))) + self.assertEqual( + 'results', + load_file( + os.path.join(out_logdir, 'run', 'cloud-init', 'results.json'))) + + def test_collect_logs_includes_optional_userdata(self): + """collect-logs include userdata when --include-userdata is set.""" + log1 = self.tmp_path('cloud-init.log', self.new_root) + write_file(log1, 'cloud-init-log') + log2 = self.tmp_path('cloud-init-output.log', self.new_root) + write_file(log2, 'cloud-init-output-log') + userdata = self.tmp_path('user-data.txt', self.new_root) + write_file(userdata, 'user-data') + ensure_dir(self.run_dir) + write_file(self.tmp_path('results.json', self.run_dir), 'results') + output_tarfile = self.tmp_path('logs.tgz') + + date = datetime.utcnow().date().strftime('%Y-%m-%d') + date_logdir = 'cloud-init-logs-{0}'.format(date) + + expected_subp = { + ('dpkg-query', '--show', "-f=${Version}\n", 'cloud-init'): + '0.7fake', + ('dmesg',): 'dmesg-out\n', + ('journalctl', '-o', 'short-precise'): 'journal-out\n', + ('tar', 'czvf', output_tarfile, date_logdir): '' + } + + def fake_subp(cmd): + cmd_tuple = tuple(cmd) + if cmd_tuple not in expected_subp: + raise AssertionError( + 'Unexpected command provided to subp: {0}'.format(cmd)) + if cmd == ['tar', 'czvf', output_tarfile, date_logdir]: + subp(cmd) # Pass through tar cmd so we can check output + return expected_subp[cmd_tuple], '' + + wrap_and_call( + 'cloudinit.cmd.devel.logs', + {'subp': {'side_effect': fake_subp}, + 'CLOUDINIT_LOGS': {'new': [log1, log2]}, + 'CLOUDINIT_RUN_DIR': {'new': self.run_dir}, + 'USER_DATA_FILE': {'new': userdata}}, + logs.collect_logs, output_tarfile, include_userdata=True) + # unpack the tarfile and check file contents + subp(['tar', 'zxvf', output_tarfile, '-C', self.new_root]) + out_logdir = self.tmp_path(date_logdir, self.new_root) + self.assertEqual( + 'user-data', + load_file(os.path.join(out_logdir, 'user-data.txt'))) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 68563e0c..6fb9d9e7 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -764,16 +764,25 @@ def main(sysv_args=None): parser_devel = subparsers.add_parser( 'devel', help='Run development tools') + parser_collect_logs = subparsers.add_parser( + 'collect-logs', help='Collect and tar all cloud-init debug info') + if sysv_args: # Only load subparsers if subcommand is specified to avoid load cost if sysv_args[0] == 'analyze': from cloudinit.analyze.__main__ import get_parser as analyze_parser # Construct analyze subcommand parser analyze_parser(parser_analyze) - if sysv_args[0] == 'devel': + elif sysv_args[0] == 'devel': from cloudinit.cmd.devel.parser import get_parser as devel_parser # Construct devel subcommand parser devel_parser(parser_devel) + elif sysv_args[0] == 'collect-logs': + from cloudinit.cmd.devel.logs import ( + get_parser as logs_parser, handle_collect_logs_args) + logs_parser(parser_collect_logs) + parser_collect_logs.set_defaults( + action=('collect-logs', handle_collect_logs_args)) args = parser.parse_args(args=sysv_args) diff --git a/packages/debian/rules.in b/packages/debian/rules.in index b87a5e84..4aa907e3 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -10,6 +10,7 @@ PYVER ?= python${pyver} override_dh_install: dh_install install -d debian/cloud-init/etc/rsyslog.d + install -d debian/cloud-init/usr/share/apport/package-hooks cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf install -D ./tools/Z99-cloud-locale-test.sh debian/cloud-init/etc/profile.d/Z99-cloud-locale-test.sh install -D ./tools/Z99-cloudinit-warnings.sh debian/cloud-init/etc/profile.d/Z99-cloudinit-warnings.sh diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 495bdc9f..258a9f08 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -72,18 +72,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): def test_conditional_subcommands_from_entry_point_sys_argv(self): """Subcommands from entry-point are properly parsed from sys.argv.""" + stdout = six.StringIO() + self.patchStdoutAndStderr(stdout=stdout) + expected_errors = [ - 'usage: cloud-init analyze', 'usage: cloud-init devel'] - conditional_subcommands = ['analyze', 'devel'] + 'usage: cloud-init analyze', 'usage: cloud-init collect-logs', + 'usage: cloud-init devel'] + conditional_subcommands = ['analyze', 'collect-logs', 'devel'] # The cloud-init entrypoint calls main without passing sys_argv for subcommand in conditional_subcommands: - with mock.patch('sys.argv', ['cloud-init', subcommand]): + with mock.patch('sys.argv', ['cloud-init', subcommand, '-h']): try: cli.main() except SystemExit as e: - self.assertEqual(2, e.code) # exit 2 on proper usage docs + self.assertEqual(0, e.code) # exit 2 on proper -h usage for error_message in expected_errors: - self.assertIn(error_message, self.stderr.getvalue()) + self.assertIn(error_message, stdout.getvalue()) def test_analyze_subcommand_parser(self): """The subcommand cloud-init analyze calls the correct subparser.""" @@ -94,6 +98,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): for subcommand in expected_subcommands: self.assertIn(subcommand, error) + def test_collect_logs_subcommand_parser(self): + """The subcommand cloud-init collect-logs calls the subparser.""" + # Provide -h param to collect-logs to avoid having to mock behavior. + stdout = six.StringIO() + self.patchStdoutAndStderr(stdout=stdout) + self._call_main(['cloud-init', 'collect-logs', '-h']) + self.assertIn('usage: cloud-init collect-log', stdout.getvalue()) + def test_devel_subcommand_parser(self): """The subcommand cloud-init devel calls the correct subparser.""" self._call_main(['cloud-init', 'devel']) -- cgit v1.2.3 From 10f067d87a2d48092c593862e686c517c57b987c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 15 Sep 2017 22:50:21 -0400 Subject: GCE: Fix usage of user-data. This regressed in the rework of GCE datasource to have a main. The fix really just stores the user-data that was read in self.userdata_raw, rather than self.userdata. That is consistent with other datasources and ulitimately how it was before the refactor. The main is updated to address the fact that user-data is binary data and may not be able to be printed. LP: #1717598 --- cloudinit/sources/DataSourceGCE.py | 26 +++++++++++++++++++------- tests/unittests/test_datasource/test_gce.py | 3 ++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 94484d60..ccae4200 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -62,7 +62,7 @@ class DataSourceGCE(sources.DataSource): LOG.debug(ret['reason']) return False self.metadata = ret['meta-data'] - self.userdata = ret['user-data'] + self.userdata_raw = ret['user-data'] return True @property @@ -80,9 +80,6 @@ class DataSourceGCE(sources.DataSource): # GCE has long FDQN's and has asked for short hostnames return self.metadata['local-hostname'].split('.')[0] - def get_userdata_raw(self): - return self.userdata - @property def availability_zone(self): return self.metadata['availability-zone'] @@ -202,6 +199,9 @@ def get_datasource_list(depends): if __name__ == "__main__": import argparse import json + import sys + + from base64 import b64encode parser = argparse.ArgumentParser(description='Query GCE Metadata Service') parser.add_argument("--endpoint", metavar="URL", @@ -211,8 +211,20 @@ if __name__ == "__main__": help="Ignore smbios platform check", action='store_false', default=True) args = parser.parse_args() - print(json.dumps( - read_md(address=args.endpoint, platform_check=args.platform_check), - indent=1, sort_keys=True, separators=(',', ': '))) + data = read_md(address=args.endpoint, platform_check=args.platform_check) + if 'user-data' in data: + # user-data is bytes not string like other things. Handle it specially. + # if it can be represented as utf-8 then do so. Otherwise print base64 + # encoded value in the key user-data-b64. + try: + data['user-data'] = data['user-data'].decode() + except UnicodeDecodeError: + sys.stderr.write("User-data cannot be decoded. " + "Writing as base64\n") + del data['user-data'] + # b64encode returns a bytes value. decode to get the string. + data['user-data-b64'] = b64encode(data['user-data']).decode() + + print(json.dumps(data, indent=1, sort_keys=True, separators=(',', ': '))) # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 50e49a10..d399ae7a 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -23,7 +23,8 @@ GCE_META = { 'instance/zone': 'foo/bar', 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', 'instance/hostname': 'server.project-foo.local', - 'instance/attributes/user-data': b'/bin/echo foo\n', + # UnicodeDecodeError below if set to ds.userdata instead of userdata_raw + 'instance/attributes/user-data': b'/bin/echo \xff\n', } GCE_META_PARTIAL = { -- cgit v1.2.3 From eaadf52b1010cf189bde2a6abb3265b890f6d36d Mon Sep 17 00:00:00 2001 From: Paul Meyer Date: Mon, 18 Sep 2017 16:07:46 -0600 Subject: Azure: wait longer for SSH pub keys to arrive. Currently the Azure data source waits up to 60 seconds. This has proven not to be sufficient to provide resiliency to unrelated transient failures in other parts of the infrastructure. Azure already has logic outside of the VM to abort hung provisioning. This changes lengthens the time out to 15 minutes. LP: #1717611 --- cloudinit/sources/DataSourceAzure.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index b5a95a1f..80c2bd12 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -317,9 +317,13 @@ class DataSourceAzure(sources.DataSource): LOG.debug("ssh authentication: " "using fingerprint from fabirc") - missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", + # wait very long for public SSH keys to arrive + # https://bugs.launchpad.net/cloud-init/+bug/1717611 + missing = util.log_time(logfunc=LOG.debug, + msg="waiting for SSH public key files", func=wait_for_files, - args=(fp_files,)) + args=(fp_files, 900)) + if len(missing): LOG.warning("Did not find files, but going on: %s", missing) @@ -656,7 +660,7 @@ def pubkeys_from_crt_files(flist): return pubkeys -def wait_for_files(flist, maxwait=60, naplen=.5, log_pre=""): +def wait_for_files(flist, maxwait, naplen=.5, log_pre=""): need = set(flist) waited = 0 while True: -- cgit v1.2.3 From 7eb3460b0d6d3e362a246958a7ea0a9ee5d91d5e Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 15 Sep 2017 20:07:11 -0600 Subject: ec2: Fix maybe_perform_dhcp_discovery to use /var/tmp as a tmpdir /run/cloud-init/tmp is on a filesystem mounted noexec, so running dchlient in Ec2Local during discovery breaks with 'Permission denied'. This branch allows us to run from a different tmp dir so we have exec rights. LP: #1717627 --- cloudinit/net/dhcp.py | 5 +- cloudinit/net/tests/test_dhcp.py | 18 ++++--- cloudinit/temp_utils.py | 22 +++++--- cloudinit/tests/test_temp_utils.py | 101 +++++++++++++++++++++++++++++++++++++ cloudinit/util.py | 2 +- 5 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 cloudinit/tests/test_temp_utils.py diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c842c839..05350639 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -48,8 +48,9 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: - return dhcp_discovery(dhclient_path, nic, tmpdir) + with temp_utils.tempdir(prefix='cloud-init-dhcp-', needs_exe=True) as tdir: + # Use /var/tmp because /run/cloud-init/tmp is mounted noexec + return dhcp_discovery(dhclient_path, nic, tdir) def parse_dhcp_lease_file(lease_file): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 4a37e98a..1324c3d0 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -8,7 +8,7 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery) from cloudinit.util import ensure_file, write_file -from cloudinit.tests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase, wrap_and_call class TestParseDHCPLeasesFile(CiTestCase): @@ -91,21 +91,27 @@ class TestDHCPDiscoveryClean(CiTestCase): 'Skip dhclient configuration: No dhclient command found.', self.logs.getvalue()) + @mock.patch('cloudinit.temp_utils.os.getuid') @mock.patch('cloudinit.net.dhcp.dhcp_discovery') @mock.patch('cloudinit.net.dhcp.util.which') @mock.patch('cloudinit.net.dhcp.find_fallback_nic') - def test_dhclient_run_with_tmpdir(self, m_fallback, m_which, m_dhcp): + def test_dhclient_run_with_tmpdir(self, m_fback, m_which, m_dhcp, m_uid): """maybe_perform_dhcp_discovery passes tmpdir to dhcp_discovery.""" - m_fallback.return_value = 'eth9' + m_uid.return_value = 0 # Fake root user for tmpdir + m_fback.return_value = 'eth9' m_which.return_value = '/sbin/dhclient' m_dhcp.return_value = {'address': '192.168.2.2'} - self.assertEqual( - {'address': '192.168.2.2'}, maybe_perform_dhcp_discovery()) + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'_TMPDIR': {'new': None}, + 'os.getuid': 0}, + maybe_perform_dhcp_discovery) + self.assertEqual({'address': '192.168.2.2'}, retval) m_dhcp.assert_called_once() call = m_dhcp.call_args_list[0] self.assertEqual('/sbin/dhclient', call[0][0]) self.assertEqual('eth9', call[0][1]) - self.assertIn('/tmp/cloud-init-dhcp-', call[0][2]) + self.assertIn('/var/tmp/cloud-init/cloud-init-dhcp-', call[0][2]) @mock.patch('cloudinit.net.dhcp.util.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp): diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py index 0355f19d..5d7adf70 100644 --- a/cloudinit/temp_utils.py +++ b/cloudinit/temp_utils.py @@ -8,9 +8,10 @@ import tempfile _TMPDIR = None _ROOT_TMPDIR = "/run/cloud-init/tmp" +_EXE_ROOT_TMPDIR = "/var/tmp/cloud-init" -def _tempfile_dir_arg(odir=None): +def _tempfile_dir_arg(odir=None, needs_exe=False): """Return the proper 'dir' argument for tempfile functions. When root, cloud-init will use /run/cloud-init/tmp to avoid @@ -20,8 +21,10 @@ def _tempfile_dir_arg(odir=None): If the caller of this function (mkdtemp or mkstemp) was provided with a 'dir' argument, then that is respected. - @param odir: original 'dir' arg to 'mkdtemp' or other.""" - + @param odir: original 'dir' arg to 'mkdtemp' or other. + @param needs_exe: Boolean specifying whether or not exe permissions are + needed for tempdir. This is needed because /run is mounted noexec. + """ if odir is not None: return odir @@ -29,7 +32,9 @@ def _tempfile_dir_arg(odir=None): if _TMPDIR: return _TMPDIR - if os.getuid() == 0: + if needs_exe: + tdir = _EXE_ROOT_TMPDIR + elif os.getuid() == 0: tdir = _ROOT_TMPDIR else: tdir = os.environ.get('TMPDIR', '/tmp') @@ -42,7 +47,8 @@ def _tempfile_dir_arg(odir=None): def ExtendedTemporaryFile(**kwargs): - kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + kwargs['dir'] = _tempfile_dir_arg( + kwargs.pop('dir', None), kwargs.pop('needs_exe', False)) fh = tempfile.NamedTemporaryFile(**kwargs) # Replace its unlink with a quiet version # that does not raise errors when the @@ -82,12 +88,14 @@ def tempdir(**kwargs): def mkdtemp(**kwargs): - kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + kwargs['dir'] = _tempfile_dir_arg( + kwargs.pop('dir', None), kwargs.pop('needs_exe', False)) return tempfile.mkdtemp(**kwargs) def mkstemp(**kwargs): - kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + kwargs['dir'] = _tempfile_dir_arg( + kwargs.pop('dir', None), kwargs.pop('needs_exe', False)) return tempfile.mkstemp(**kwargs) # vi: ts=4 expandtab diff --git a/cloudinit/tests/test_temp_utils.py b/cloudinit/tests/test_temp_utils.py new file mode 100644 index 00000000..ffbb92cd --- /dev/null +++ b/cloudinit/tests/test_temp_utils.py @@ -0,0 +1,101 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""Tests for cloudinit.temp_utils""" + +from cloudinit.temp_utils import mkdtemp, mkstemp +from cloudinit.tests.helpers import CiTestCase, wrap_and_call + + +class TestTempUtils(CiTestCase): + + def test_mkdtemp_default_non_root(self): + """mkdtemp creates a dir under /tmp for the unprivileged.""" + calls = [] + + def fake_mkdtemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 1000, + 'tempfile.mkdtemp': {'side_effect': fake_mkdtemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkdtemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/tmp'}], calls) + + def test_mkdtemp_default_non_root_needs_exe(self): + """mkdtemp creates a dir under /var/tmp/cloud-init when needs_exe.""" + calls = [] + + def fake_mkdtemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 1000, + 'tempfile.mkdtemp': {'side_effect': fake_mkdtemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkdtemp, needs_exe=True) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/var/tmp/cloud-init'}], calls) + + def test_mkdtemp_default_root(self): + """mkdtemp creates a dir under /run/cloud-init for the privileged.""" + calls = [] + + def fake_mkdtemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 0, + 'tempfile.mkdtemp': {'side_effect': fake_mkdtemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkdtemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/run/cloud-init/tmp'}], calls) + + def test_mkstemp_default_non_root(self): + """mkstemp creates secure tempfile under /tmp for the unprivileged.""" + calls = [] + + def fake_mkstemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 1000, + 'tempfile.mkstemp': {'side_effect': fake_mkstemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkstemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/tmp'}], calls) + + def test_mkstemp_default_root(self): + """mkstemp creates a secure tempfile in /run/cloud-init for root.""" + calls = [] + + def fake_mkstemp(*args, **kwargs): + calls.append(kwargs) + return '/fake/return/path' + + retval = wrap_and_call( + 'cloudinit.temp_utils', + {'os.getuid': 0, + 'tempfile.mkstemp': {'side_effect': fake_mkstemp}, + '_TMPDIR': {'new': None}, + 'os.path.isdir': True}, + mkstemp) + self.assertEqual('/fake/return/path', retval) + self.assertEqual([{'dir': '/run/cloud-init/tmp'}], calls) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 7e9d94fc..4c01f449 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1755,7 +1755,7 @@ def subp_blob_in_tempfile(blob, *args, **kwargs): args = [tuple()] # Use tmpdir over tmpfile to avoid 'text file busy' on execute - with temp_utils.tempdir() as tmpd: + with temp_utils.tempdir(needs_exe=True) as tmpd: tmpf = os.path.join(tmpd, basename) if 'args' in kwargs: kwargs['args'] = [tmpf] + list(kwargs['args']) -- cgit v1.2.3 From 7a2d4cc8bfbbc3b4386cee6333a966acd09e9c74 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 Sep 2017 14:28:39 -0400 Subject: tests: fix ds-identify unit tests to set EC2_STRICT_ID_DEFAULT. The variable DI_EC2_STRICT_ID_DEFAULT was not being set in unit tests so when 16.04 built, which changed that setting in patches the tests would unexpectedly fail. --- tests/unittests/test_ds_identify.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 92454d7c..1284e755 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -31,6 +31,7 @@ POLICY_FOUND_ONLY = "search,found=all,maybe=none,notfound=disabled" POLICY_FOUND_OR_MAYBE = "search,found=all,maybe=all,notfound=disabled" DI_DEFAULT_POLICY = "search,found=all,maybe=all,notfound=enabled" DI_DEFAULT_POLICY_NO_DMI = "search,found=all,maybe=all,notfound=disabled" +DI_EC2_STRICT_ID_DEFAULT = "true" SHELL_MOCK_TMPL = """\ %(name)s() { @@ -62,7 +63,8 @@ class TestDsIdentify(CiTestCase): def call(self, rootd=None, mocks=None, args=None, files=None, policy_dmi=DI_DEFAULT_POLICY, - policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI): + policy_no_dmi=DI_DEFAULT_POLICY_NO_DMI, + ec2_strict_id=DI_EC2_STRICT_ID_DEFAULT): if args is None: args = [] if mocks is None: @@ -89,6 +91,7 @@ class TestDsIdentify(CiTestCase): ". " + self.dsid_path, 'DI_DEFAULT_POLICY="%s"' % policy_dmi, 'DI_DEFAULT_POLICY_NO_DMI="%s"' % policy_no_dmi, + 'DI_EC2_STRICT_ID_DEFAULT="%s"' % ec2_strict_id, "" ] -- cgit v1.2.3 From 82b2da3a56680b43df00ed31837b8650b5971656 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 20 Sep 2017 15:47:04 -0600 Subject: Makefile: No longer look for yaml files in obsolete ./bin/. The bin/ dir was deleted some time ago, but the Makefile was still searching for files down it. This didn't cause any problems other Than a wierd looking error message in a build log. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9e7f4ee7..7feea400 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ PYVER ?= $(shell for p in python3 python2; do \ noseopts ?= -v -YAML_FILES=$(shell find cloudinit bin tests tools -name "*.yaml" -type f ) +YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f ) YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f ) PIP_INSTALL := pip install -- cgit v1.2.3 From d3a8777244ebc107e1124c4fab441b5e0eb75f44 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 20 Sep 2017 16:41:31 -0600 Subject: tests: Add cloudinit package to all test targets The package cloudinit was sparsely added to only the makefile's unittest target and tox's py3 target. This branch adds cloudinit package to 'make unittest3' and all tox environments. It tweaks one cloudinit unit test to use mocked_object.call_count instead of mocked_object.assert_called_once which is not defined in some python unittest versions. --- Makefile | 2 +- cloudinit/net/tests/test_dhcp.py | 3 ++- tox.ini | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7feea400..4ace2270 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ unittest: clean_pyc nosetests $(noseopts) tests/unittests cloudinit unittest3: clean_pyc - nosetests3 $(noseopts) tests/unittests + nosetests3 $(noseopts) tests/unittests cloudinit ci-deps-ubuntu: @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 1324c3d0..a38edaec 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -107,7 +107,8 @@ class TestDHCPDiscoveryClean(CiTestCase): 'os.getuid': 0}, maybe_perform_dhcp_discovery) self.assertEqual({'address': '192.168.2.2'}, retval) - m_dhcp.assert_called_once() + self.assertEqual( + 1, m_dhcp.call_count, 'dhcp_discovery not called once') call = m_dhcp.call_args_list[0] self.assertEqual('/sbin/dhclient', call[0][0]) self.assertEqual('eth9', call[0][1]) diff --git a/tox.ini b/tox.ini index 72de9830..776f4253 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py27, py3, flake8, xenial, pylint recreate = True [testenv] -commands = python -m nose {posargs:tests/unittests} +commands = python -m nose {posargs:tests/unittests cloudinit} setenv = LC_ALL = en_US.utf-8 -- cgit v1.2.3 From 27613443139578dd1b968d1f5f953bd2bc6245a4 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 20 Sep 2017 23:41:22 -0600 Subject: docs: fix sphinx module schema documentation Create a copy of each modules schema attribute when generating sphinx docs to avoid altering the actual module dict in memory. This avoids illegible rendering of module examples and distros where each character of a list was represented on a separate line by itself. Fixes ntp, resizefs, runcmd and bootcmd docs. --- cloudinit/config/schema.py | 12 +++++++----- tests/unittests/test_cli.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c17d973e..bb291ff8 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -8,6 +8,7 @@ from cloudinit.util import find_modules, read_file_or_url import argparse from collections import defaultdict +from copy import deepcopy import logging import os import re @@ -273,12 +274,13 @@ def get_schema_doc(schema): @param schema: Dict of jsonschema to render. @raise KeyError: If schema lacks an expected key. """ - schema['property_doc'] = _get_property_doc(schema) - schema['examples'] = _get_schema_examples(schema) - schema['distros'] = ', '.join(schema['distros']) + schema_copy = deepcopy(schema) + schema_copy['property_doc'] = _get_property_doc(schema) + schema_copy['examples'] = _get_schema_examples(schema) + schema_copy['distros'] = ', '.join(schema['distros']) # Need an underbar of the same length as the name - schema['title_underbar'] = re.sub(r'.', '-', schema['name']) - return SCHEMA_DOC_TMPL.format(**schema) + schema_copy['title_underbar'] = re.sub(r'.', '-', schema['name']) + return SCHEMA_DOC_TMPL.format(**schema_copy) FULL_SCHEMA = None diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 258a9f08..fccbbd23 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -125,6 +125,21 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): 'Expected either --config-file argument or --doc\n', self.stderr.getvalue()) + def test_wb_devel_schema_subcommand_doc_content(self): + """Validate that doc content is sane from known examples.""" + stdout = six.StringIO() + self.patchStdoutAndStderr(stdout=stdout) + self._call_main(['cloud-init', 'devel', 'schema', '--doc']) + expected_doc_sections = [ + '**Supported distros:** all', + '**Supported distros:** centos, debian, fedora', + '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', + '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' + ] + stdout = stdout.getvalue() + for expected in expected_doc_sections: + self.assertIn(expected, stdout) + @mock.patch('cloudinit.cmd.main.main_single') def test_single_subcommand(self, m_main_single): """The subcommand 'single' calls main_single with valid args.""" -- cgit v1.2.3 From 243ec59fb62a8710430f9ba1e2490ee964c1abc0 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Thu, 21 Sep 2017 08:07:28 -0400 Subject: suse: Copy sysvinit files from redhat with slight changes. Here we commit the SuSE provided sysvinit scripts. They are very similar to those in redhat/ directory. They differ in small but important ways. Rather than build a template system here we will just accept the copy and paste. sysvinit in both RedHat and SuSE is EOL, so we do not expect any real maintenance cost here. LP: #1718649 --- setup.py | 2 + sysvinit/suse/cloud-config | 113 ++++++++++++++++++++++++++++++++++++++++ sysvinit/suse/cloud-final | 113 ++++++++++++++++++++++++++++++++++++++++ sysvinit/suse/cloud-init | 114 +++++++++++++++++++++++++++++++++++++++++ sysvinit/suse/cloud-init-local | 113 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 455 insertions(+) create mode 100644 sysvinit/suse/cloud-config create mode 100644 sysvinit/suse/cloud-final create mode 100644 sysvinit/suse/cloud-init create mode 100644 sysvinit/suse/cloud-init-local diff --git a/setup.py b/setup.py index 91993174..bf697d7f 100755 --- a/setup.py +++ b/setup.py @@ -121,6 +121,7 @@ INITSYS_FILES = { 'sysvinit_freebsd': [f for f in glob('sysvinit/freebsd/*') if is_f(f)], 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], 'sysvinit_openrc': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], + 'sysvinit_suse': [f for f in glob('sysvinit/suse/*') if is_f(f)], 'systemd': [render_tmpl(f) for f in (glob('systemd/*.tmpl') + glob('systemd/*.service') + @@ -133,6 +134,7 @@ INITSYS_ROOTS = { 'sysvinit_freebsd': 'usr/local/etc/rc.d', 'sysvinit_deb': 'etc/init.d', 'sysvinit_openrc': 'etc/init.d', + 'sysvinit_suse': 'etc/init.d', 'systemd': pkg_config_read('systemd', 'systemdsystemunitdir'), 'systemd.generators': pkg_config_read('systemd', 'systemdsystemgeneratordir'), diff --git a/sysvinit/suse/cloud-config b/sysvinit/suse/cloud-config new file mode 100644 index 00000000..75b81512 --- /dev/null +++ b/sysvinit/suse/cloud-config @@ -0,0 +1,113 @@ +#!/bin/sh +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This file is part of cloud-init. See LICENSE file for license information. + +# See: http://wiki.debian.org/LSBInitScripts +# See: http://tiny.cc/czvbgw +# See: http://www.novell.com/coolsolutions/feature/15380.html +# Also based on dhcpd in RHEL (for comparison) + +### BEGIN INIT INFO +# Provides: cloud-config +# Required-Start: cloud-init cloud-init-local +# Should-Start: $time +# Required-Stop: $null +# Should-Stop: $null +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: The config cloud-init job +# Description: Start cloud-init and runs the config phase +# and any associated config modules as desired. +### END INIT INFO + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - generic or unspecified error +# 2 - invalid or excess argument(s) +# 3 - unimplemented feature (e.g. "reload") +# 4 - user had insufficient privileges +# 5 - program is not installed +# 6 - program is not configured +# 7 - program is not running +# 8--199 - reserved (8--99 LSB, 100--149 distrib, 150--199 appl) +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signaling is not supported) are +# considered a success. + +RETVAL=0 + +prog="cloud-init" +cloud_init="/usr/bin/cloud-init" +conf="/etc/cloud/cloud.cfg" + +# If there exist sysconfig/default variable override files use it... +[ -f /etc/sysconfig/cloud-init ] && . /etc/sysconfig/cloud-init +[ -f /etc/default/cloud-init ] && . /etc/default/cloud-init + +. /etc/rc.status +rc_reset + +start() { + [ -x $cloud_init ] || return 5 + [ -f $conf ] || return 6 + + echo -n $"Starting $prog: " + $cloud_init $CLOUDINITARGS modules --mode config + RETVAL=$? + return $RETVAL +} + +stop() { + echo -n $"Shutting down $prog: " + # No-op + RETVAL=7 + return $RETVAL +} + +case "$1" in + start) + start + RETVAL=$? + ;; + stop) + stop + RETVAL=$? + ;; + restart|try-restart|condrestart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + # + ## Note: try-restart is now part of LSB (as of 1.9). + ## RH has a similar command named condrestart. + start + RETVAL=$? + ;; + reload|force-reload) + # It does not support reload + RETVAL=3 + ;; + status) + echo -n $"Checking for service $prog:" + # Return value is slightly different for the status command: + # 0 - service up and running + # 1 - service dead, but /var/run/ pid file exists + # 2 - service dead, but /var/lock/ lock file exists + # 3 - service not running (unused) + # 4 - service status unknown :-( + # 5--199 reserved (5--99 LSB, 100--149 distro, 150--199 appl.) + RETVAL=3 + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|condrestart|restart|force-reload|reload}" + RETVAL=3 + ;; +esac + +_rc_status=$RETVAL +rc_status -v +rc_exit diff --git a/sysvinit/suse/cloud-final b/sysvinit/suse/cloud-final new file mode 100644 index 00000000..25586e1e --- /dev/null +++ b/sysvinit/suse/cloud-final @@ -0,0 +1,113 @@ +#!/bin/sh +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This file is part of cloud-init. See LICENSE file for license information. + +# See: http://wiki.debian.org/LSBInitScripts +# See: http://tiny.cc/czvbgw +# See: http://www.novell.com/coolsolutions/feature/15380.html +# Also based on dhcpd in RHEL (for comparison) + +### BEGIN INIT INFO +# Provides: cloud-final +# Required-Start: cloud-config +# Should-Start: $time +# Required-Stop: $null +# Should-Stop: $null +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: The final cloud-init job +# Description: Start cloud-init and runs the final phase +# and any associated final modules as desired. +### END INIT INFO + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - generic or unspecified error +# 2 - invalid or excess argument(s) +# 3 - unimplemented feature (e.g. "reload") +# 4 - user had insufficient privileges +# 5 - program is not installed +# 6 - program is not configured +# 7 - program is not running +# 8--199 - reserved (8--99 LSB, 100--149 distrib, 150--199 appl) +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signaling is not supported) are +# considered a success. + +RETVAL=0 + +prog="cloud-init" +cloud_init="/usr/bin/cloud-init" +conf="/etc/cloud/cloud.cfg" + +# If there exist sysconfig/default variable override files use it... +[ -f /etc/sysconfig/cloud-init ] && . /etc/sysconfig/cloud-init +[ -f /etc/default/cloud-init ] && . /etc/default/cloud-init + +. /etc/rc.status +rc_reset + +start() { + [ -x $cloud_init ] || return 5 + [ -f $conf ] || return 6 + + echo -n $"Starting $prog: " + $cloud_init $CLOUDINITARGS modules --mode final + RETVAL=$? + return $RETVAL +} + +stop() { + echo -n $"Shutting down $prog: " + # No-op + RETVAL=7 + return $RETVAL +} + +case "$1" in + start) + start + RETVAL=$? + ;; + stop) + stop + RETVAL=$? + ;; + restart|try-restart|condrestart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + # + ## Note: try-restart is now part of LSB (as of 1.9). + ## RH has a similar command named condrestart. + start + RETVAL=$? + ;; + reload|force-reload) + # It does not support reload + RETVAL=3 + ;; + status) + echo -n $"Checking for service $prog:" + # Return value is slightly different for the status command: + # 0 - service up and running + # 1 - service dead, but /var/run/ pid file exists + # 2 - service dead, but /var/lock/ lock file exists + # 3 - service not running (unused) + # 4 - service status unknown :-( + # 5--199 reserved (5--99 LSB, 100--149 distro, 150--199 appl.) + RETVAL=3 + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|condrestart|restart|force-reload|reload}" + RETVAL=3 + ;; +esac + +_rc_status=$RETVAL +rc_status -v +rc_exit diff --git a/sysvinit/suse/cloud-init b/sysvinit/suse/cloud-init new file mode 100644 index 00000000..67e8e6af --- /dev/null +++ b/sysvinit/suse/cloud-init @@ -0,0 +1,114 @@ +#!/bin/sh +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This file is part of cloud-init. See LICENSE file for license information. + +# See: http://wiki.debian.org/LSBInitScripts +# See: http://tiny.cc/czvbgw +# See: http://www.novell.com/coolsolutions/feature/15380.html +# Also based on dhcpd in RHEL (for comparison) + +### BEGIN INIT INFO +# Provides: cloud-init +# Required-Start: $local_fs $network $named $remote_fs cloud-init-local +# Should-Start: $time +# Required-Stop: $null +# Should-Stop: $null +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: The initial cloud-init job (net and fs contingent) +# Description: Start cloud-init and runs the initialization phase +# and any associated initial modules as desired. +### END INIT INFO + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - generic or unspecified error +# 2 - invalid or excess argument(s) +# 3 - unimplemented feature (e.g. "reload") +# 4 - user had insufficient privileges +# 5 - program is not installed +# 6 - program is not configured +# 7 - program is not running +# 8--199 - reserved (8--99 LSB, 100--149 distrib, 150--199 appl) +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signaling is not supported) are +# considered a success. + +RETVAL=0 + +prog="cloud-init" +cloud_init="/usr/bin/cloud-init" +conf="/etc/cloud/cloud.cfg" + +# If there exist sysconfig/default variable override files use it... +[ -f /etc/sysconfig/cloud-init ] && . /etc/sysconfig/cloud-init +[ -f /etc/default/cloud-init ] && . /etc/default/cloud-init + +. /etc/rc.status +rc_reset + +start() { + [ -x $cloud_init ] || return 5 + [ -f $conf ] || return 6 + + echo -n $"Starting $prog: " + $cloud_init $CLOUDINITARGS init + RETVAL=$? + return $RETVAL +} + +stop() { + echo -n $"Shutting down $prog: " + # No-op + RETVAL=7 + return $RETVAL +} + +case "$1" in + start) + start + RETVAL=$? + ;; + stop) + stop + RETVAL=$? + ;; + restart|try-restart|condrestart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + # + ## Note: try-restart is now part of LSB (as of 1.9). + ## RH has a similar command named condrestart. + start + RETVAL=$? + ;; + reload|force-reload) + # It does not support reload + RETVAL=3 + ;; + status) + echo -n $"Checking for service $prog:" + RETVAL=3 + [ -e /root/.ssh/authorized_keys ] && RETVAL=0 + # Return value is slightly different for the status command: + # 0 - service up and running + # 1 - service dead, but /var/run/ pid file exists + # 2 - service dead, but /var/lock/ lock file exists + # 3 - service not running (unused) + # 4 - service status unknown :-( + # 5--199 reserved (5--99 LSB, 100--149 distro, 150--199 appl.) + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|condrestart|restart|force-reload|reload}" + RETVAL=3 + ;; +esac + +_rc_status=$RETVAL +rc_status -v +rc_exit diff --git a/sysvinit/suse/cloud-init-local b/sysvinit/suse/cloud-init-local new file mode 100644 index 00000000..1370d980 --- /dev/null +++ b/sysvinit/suse/cloud-init-local @@ -0,0 +1,113 @@ +#!/bin/sh +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This file is part of cloud-init. See LICENSE file for license information. + +# See: http://wiki.debian.org/LSBInitScripts +# See: http://tiny.cc/czvbgw +# See: http://www.novell.com/coolsolutions/feature/15380.html +# Also based on dhcpd in RHEL (for comparison) + +### BEGIN INIT INFO +# Provides: cloud-init-local +# Required-Start: $local_fs $remote_fs +# Should-Start: $time +# Required-Stop: $null +# Should-Stop: $null +# Default-Start: 2 3 5 +# Default-Stop: 0 1 6 +# Short-Description: The initial cloud-init job (local fs contingent) +# Description: Start cloud-init and runs the initialization phases +# and any associated initial modules as desired. +### END INIT INFO + +# Return values acc. to LSB for all commands but status: +# 0 - success +# 1 - generic or unspecified error +# 2 - invalid or excess argument(s) +# 3 - unimplemented feature (e.g. "reload") +# 4 - user had insufficient privileges +# 5 - program is not installed +# 6 - program is not configured +# 7 - program is not running +# 8--199 - reserved (8--99 LSB, 100--149 distrib, 150--199 appl) +# +# Note that starting an already running service, stopping +# or restarting a not-running service as well as the restart +# with force-reload (in case signaling is not supported) are +# considered a success. + +RETVAL=0 + +prog="cloud-init" +cloud_init="/usr/bin/cloud-init" +conf="/etc/cloud/cloud.cfg" + +# If there exist sysconfig/default variable override files use it... +[ -f /etc/sysconfig/cloud-init ] && . /etc/sysconfig/cloud-init +[ -f /etc/default/cloud-init ] && . /etc/default/cloud-init + +. /etc/rc.status +rc_reset + +start() { + [ -x $cloud_init ] || return 5 + [ -f $conf ] || return 6 + + echo -n $"Starting $prog: " + $cloud_init $CLOUDINITARGS init --local + RETVAL=$? + return $RETVAL +} + +stop() { + echo -n $"Shutting down $prog: " + # No-op + RETVAL=7 + return $RETVAL +} + +case "$1" in + start) + start + RETVAL=$? + ;; + stop) + stop + RETVAL=$? + ;; + restart|try-restart|condrestart) + ## Stop the service and regardless of whether it was + ## running or not, start it again. + # + ## Note: try-restart is now part of LSB (as of 1.9). + ## RH has a similar command named condrestart. + start + RETVAL=$? + ;; + reload|force-reload) + # It does not support reload + RETVAL=3 + ;; + status) + echo -n $"Checking for service $prog:" + # Return value is slightly different for the status command: + # 0 - service up and running + # 1 - service dead, but /var/run/ pid file exists + # 2 - service dead, but /var/lock/ lock file exists + # 3 - service not running (unused) + # 4 - service status unknown :-( + # 5--199 reserved (5--99 LSB, 100--149 distro, 150--199 appl.) + RETVAL=3 + ;; + *) + echo "Usage: $0 {start|stop|status|try-restart|condrestart|restart|force-reload|reload}" + RETVAL=3 + ;; +esac + +_rc_status=$RETVAL +rc_status -v +rc_exit -- cgit v1.2.3 From 0451a9f60960da56e3af4f97bbcece3d98482f86 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Thu, 21 Sep 2017 07:38:48 -0400 Subject: suse: updates to templates to support openSUSE and SLES. Things done here: - identify 'suse' as a variant in util.system_info and also tools/render-cloudcfg. - update systemd and cloud.cfg templates for suse specific changes. LP: #1718640 --- cloudinit/util.py | 2 ++ config/cloud.cfg.tmpl | 8 ++++++-- systemd/cloud-init-local.service.tmpl | 6 ++++++ systemd/cloud-init.service.tmpl | 10 ++++++++++ tools/render-cloudcfg | 5 +++-- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 4c01f449..e1290aa8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -598,6 +598,8 @@ def system_info(): var = 'ubuntu' elif linux_dist == 'redhat': var = 'rhel' + elif linux_dist == 'suse': + var = 'suse' else: var = 'linux' elif system in ('windows', 'darwin', "freebsd"): diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index a537d65a..50e3bd86 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -127,7 +127,7 @@ cloud_final_modules: # (not accessible to handlers/transforms) system_info: # This will affect which distro class gets used -{% if variant in ["centos", "debian", "fedora", "rhel", "ubuntu", "freebsd"] %} +{% if variant in ["centos", "debian", "fedora", "rhel", "suse", "ubuntu", "freebsd"] %} distro: {{ variant }} {% else %} # Unknown/fallback distro. @@ -163,13 +163,17 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh -{% elif variant in ["centos", "rhel", "fedora"] %} +{% elif variant in ["centos", "rhel", "fedora", "suse"] %} # Default user name + that default users groups (if added/used) default_user: name: {{ variant }} lock_passwd: True gecos: {{ variant }} Cloud User +{% if variant == "suse" %} + groups: [cdrom, users] +{% else %} groups: [wheel, adm, systemd-journal] +{% endif %} sudo: ["ALL=(ALL) NOPASSWD:ALL"] shell: /bin/bash # Other config here will be given to the distro class and/or path classes diff --git a/systemd/cloud-init-local.service.tmpl b/systemd/cloud-init-local.service.tmpl index ff9c644d..bf6b2961 100644 --- a/systemd/cloud-init-local.service.tmpl +++ b/systemd/cloud-init-local.service.tmpl @@ -13,6 +13,12 @@ Before=shutdown.target Before=sysinit.target Conflicts=shutdown.target {% endif %} +{% if variant in ["suse"] %} +# Other distros use Before=sysinit.target. There is not a clearly identified +# reason for usage of basic.target instead. +Before=basic.target +Conflicts=shutdown.target +{% endif %} RequiresMountsFor=/var/lib/cloud [Service] diff --git a/systemd/cloud-init.service.tmpl b/systemd/cloud-init.service.tmpl index 2c71889d..b92e8abc 100644 --- a/systemd/cloud-init.service.tmpl +++ b/systemd/cloud-init.service.tmpl @@ -13,6 +13,13 @@ After=networking.service {% if variant in ["centos", "fedora", "redhat"] %} After=network.service {% endif %} +{% if variant in ["suse"] %} +Requires=wicked.service +After=wicked.service +# setting hostname via hostnamectl depends on dbus, which otherwise +# would not be guaranteed at this point. +After=dbus.service +{% endif %} Before=network-online.target Before=sshd-keygen.service Before=sshd.service @@ -20,6 +27,9 @@ Before=sshd.service Before=sysinit.target Conflicts=shutdown.target {% endif %} +{% if variant in ["suse"] %} +Conflicts=shutdown.target +{% endif %} Before=systemd-user-sessions.service [Service] diff --git a/tools/render-cloudcfg b/tools/render-cloudcfg index e624541a..8b7cb875 100755 --- a/tools/render-cloudcfg +++ b/tools/render-cloudcfg @@ -4,6 +4,8 @@ import argparse import os import sys +VARIANTS = ["bsd", "centos", "fedora", "rhel", "suse", "ubuntu", "unknown"] + if "avoid-pep8-E402-import-not-top-of-file": _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, _tdir) @@ -14,11 +16,10 @@ if "avoid-pep8-E402-import-not-top-of-file": def main(): parser = argparse.ArgumentParser() - variants = ["bsd", "centos", "fedora", "rhel", "ubuntu", "unknown"] platform = util.system_info() parser.add_argument( "--variant", default=platform['variant'], action="store", - help="define the variant.", choices=variants) + help="define the variant.", choices=VARIANTS) parser.add_argument( "template", nargs="?", action="store", default='./config/cloud.cfg.tmpl', -- cgit v1.2.3 From 99ef5adfed0b31f87f8ea56b22113737f41bba9d Mon Sep 17 00:00:00 2001 From: Arnd Hannemann Date: Mon, 21 Aug 2017 15:47:50 +0200 Subject: doc: document GCE datasource. Add some minimal documentation for GCE datasource. --- doc/rtd/topics/datasources.rst | 1 + doc/rtd/topics/datasources/gce.rst | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 doc/rtd/topics/datasources/gce.rst diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index a60f5eb7..7e2854de 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -94,5 +94,6 @@ Follow for more information. datasources/ovf.rst datasources/smartos.rst datasources/fallback.rst + datasources/gce.rst .. vi: textwidth=78 diff --git a/doc/rtd/topics/datasources/gce.rst b/doc/rtd/topics/datasources/gce.rst new file mode 100644 index 00000000..8406695c --- /dev/null +++ b/doc/rtd/topics/datasources/gce.rst @@ -0,0 +1,20 @@ +.. _datasource_gce: + +Google Compute Engine +===================== + +The GCE datasource gets its data from the internal compute metadata server. +Metadata can be queried at the URL +'``http://metadata.google.internal/computeMetadata/v1/``' +from within an instance. For more information see the `GCE metadata docs`_. + +Currently the default project and instance level metadatakeys keys +``project/attributes/sshKeys`` and ``instance/attributes/ssh-keys`` are merged +to provide ``public-keys``. + +``user-data`` and ``user-data-encoding`` can be provided to cloud-init by +setting those custom metadata keys for an *instance*. + +.. _GCE metadata docs: https://cloud.google.com/compute/docs/storing-retrieving-metadata#querying + +.. vi: textwidth=78 -- cgit v1.2.3 From bf6456fd2220e6c1ca43a373e12af0c3d8a93bab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 21 Sep 2017 14:59:07 -0400 Subject: release 17.1 Bump the version in cloudinit/version.py to be 17.1 and update ChangeLog. --- ChangeLog | 422 +++++++++++++++++++++++++++++++++++++++++++++++++++ cloudinit/version.py | 2 +- 2 files changed, 423 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 80405bc2..0260c576 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,425 @@ +17.1: + - doc: document GCE datasource. [Arnd Hannemann] + - suse: updates to templates to support openSUSE and SLES. + [Robert Schweikert] (LP: #1718640) + - suse: Copy sysvinit files from redhat with slight changes. + [Robert Schweikert] (LP: #1718649) + - docs: fix sphinx module schema documentation [Chad Smith] + - tests: Add cloudinit package to all test targets [Chad Smith] + - Makefile: No longer look for yaml files in obsolete ./bin/. + - tests: fix ds-identify unit tests to set EC2_STRICT_ID_DEFAULT. + - ec2: Fix maybe_perform_dhcp_discovery to use /var/tmp as a tmpdir + [Chad Smith] (LP: #1717627) + - Azure: wait longer for SSH pub keys to arrive. + [Paul Meyer] (LP: #1717611) + - GCE: Fix usage of user-data. (LP: #1717598) + - cmdline: add collect-logs subcommand. [Chad Smith] (LP: #1607345) + - CloudStack: consider dhclient lease files named with a hyphen. + (LP: #1717147) + - resizefs: Drop check for read-only device file, do not warn on + overlayroot. [Chad Smith] + - Do not provide systemd-fsck drop-in which could cause ordering cycles. + [Balint Reczey] (LP: #1717477) + - tests: Enable the NoCloud KVM platform [Joshua Powers] + - resizefs: pass mount point to xfs_growfs [Dusty Mabe] + - vmware: Enable nics before sending the SUCCESS event. [Sankar Tanguturi] + - cloud-config modules: honor distros definitions in each module + [Chad Smith] (LP: #1715738, #1715690) + - chef: Add option to pin chef omnibus install version + [Ethan Apodaca] (LP: #1462693) + - tests: execute: support command as string [Joshua Powers] + - schema and docs: Add jsonschema to resizefs and bootcmd modules + [Chad Smith] + - tools: Add xkvm script, wrapper around qemu-system [Joshua Powers] + - vmware customization: return network config format + [Sankar Tanguturi] (LP: #1675063) + - Ec2: only attempt to operate at local mode on known platforms. + (LP: #1715128) + - Use /run/cloud-init for tempfile operations. (LP: #1707222) + - ds-identify: Make OpenStack return maybe on arch other than intel. + (LP: #1715241) + - tests: mock missed openstack metadata uri network_data.json + [Chad Smith] (LP: #1714376) + - relocate tests/unittests/helpers.py to cloudinit/tests + [Lars Kellogg-Stedman] + - tox: add nose timer output [Joshua Powers] + - upstart: do not package upstart jobs, drop ubuntu-init-switch module. + - tests: Stop leaking calls through unmocked metadata addresses + [Chad Smith] (LP: #1714117) + - distro: allow distro to specify a default locale [Ryan Harper] + - tests: fix two recently added tests for sles distro. + - url_helper: dynamically import oauthlib import from inside oauth_headers + [Chad Smith] + - tox: make xenial environment run with python3.6 + - suse: Add support for openSUSE and return SLES to a working state. + [Robert Schweikert] + - GCE: Add a main to the GCE Datasource. + - ec2: Add IPv6 dhcp support to Ec2DataSource. [Chad Smith] (LP: #1639030) + - url_helper: fail gracefully if oauthlib is not available + [Lars Kellogg-Stedman] (LP: #1713760) + - cloud-init analyze: fix issues running under python 2. [Andrew Jorgensen] + - Configure logging module to always use UTC time. + [Ryan Harper] (LP: #1713158) + - Log a helpful message if a user script does not include shebang. + [Andrew Jorgensen] + - cli: Fix command line parsing of coniditionally loaded subcommands. + [Chad Smith] (LP: #1712676) + - doc: Explain error behavior in user data include file format. + [Jason Butz] + - cc_landscape & cc_puppet: Fix six.StringIO use in writing configs + [Chad Smith] (LP: #1699282, #1710932) + - schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema + [Chad Smith] + - Debian: Remove non-free repositories from apt sources template. + [Joonas Kylmälä] (LP: #1700091) + - tools: Add tooling for basic cloud-init performance analysis. + [Chad Smith] (LP: #1709761) + - network: add v2 passthrough and fix parsing v2 config with bonds/bridge + params [Ryan Harper] (LP: #1709180) + - doc: update capabilities with features available, link doc reference, + cli example [Ryan Harper] + - vcloud directory: Guest Customization support for passwords + [Maitreyee Saikia] + - ec2: Allow Ec2 to run in init-local using dhclient in a sandbox. + [Chad Smith] (LP: #1709772) + - cc_ntp: fallback on timesyncd configuration if ntp is not installable + [Ryan Harper] (LP: #1686485) + - net: Reduce duplicate code. Have get_interfaces_by_mac use + get_interfaces. + - tests: Fix build tree integration tests [Joshua Powers] + - sysconfig: Dont repeat header when rendering resolv.conf + [Ryan Harper] (LP: #1701420) + - archlinux: Fix bug with empty dns, do not render 'lo' devices. + (LP: #1663045, #1706593) + - cloudinit.net: add initialize_network_device function and tests + [Chad Smith] + - makefile: fix ci-deps-ubuntu target [Chad Smith] + - tests: adjust locale integration test to parse default locale. + - tests: remove 'yakkety' from releases as it is EOL. + - tests: Add initial tests for EC2 and improve a docstring. + - locale: Do not re-run locale-gen if provided locale is system default. + - archlinux: fix set hostname usage of write_file. + [Joshua Powers] (LP: #1705306) + - sysconfig: support subnet type of 'manual'. + - tools/run-centos: make running with no argument show help. + - Drop rand_str() usage in DNS redirection detection + [Bob Aman] (LP: #1088611) + - sysconfig: use MACADDR on bonds/bridges to configure mac_address + [Ryan Harper] (LP: #1701417) + - net: eni route rendering missed ipv6 default route config + [Ryan Harper] (LP: #1701097) + - sysconfig: enable mtu set per subnet, including ipv6 mtu + [Ryan Harper] (LP: #1702513) + - sysconfig: handle manual type subnets [Ryan Harper] (LP: #1687725) + - sysconfig: fix ipv6 gateway routes [Ryan Harper] (LP: #1694801) + - sysconfig: fix rendering of bond, bridge and vlan types. + [Ryan Harper] (LP: #1695092) + - Templatize systemd unit files for cross distro deltas. [Ryan Harper] + - sysconfig: ipv6 and default gateway fixes. [Ryan Harper] (LP: #1704872) + - net: fix renaming of nics to support mac addresses written in upper + case. (LP: #1705147) + - tests: fixes for issues uncovered when moving to python 3.6. + (LP: #1703697) + - sysconfig: include GATEWAY value if set in subnet + [Ryan Harper] (LP: #1686856) + - Scaleway: add datasource with user and vendor data for Scaleway. + [Julien Castets] + - Support comments in content read by load_shell_content. + - cloudinitlocal fail to run during boot [Hongjiang Zhang] + - doc: fix disk setup example table_type options + [Sandor Zeestraten] (LP: #1703789) + - tools: Fix exception handling. [Joonas Kylmälä] (LP: #1701527) + - tests: fix usage of mock in GCE test. + - test_gce: Fix invalid mock of platform_reports_gce to return False + [Chad Smith] + - test: fix incorrect keyid for apt repository. + [Joshua Powers] (LP: #1702717) + - tests: Update version of pylxd [Joshua Powers] + - write_files: Remove log from helper function signatures. + [Andrew Jorgensen] + - doc: document the cmdline options to NoCloud [Brian Candler] + - read_dmi_data: always return None when inside a container. (LP: #1701325) + - requirements.txt: remove trailing white space. + - Azure: Add network-config, Refactor net layer to handle duplicate macs. + [Ryan Harper] + - Tests: Simplify the check on ssh-import-id [Joshua Powers] + - tests: update ntp tests after sntp added [Joshua Powers] + - FreeBSD: Make freebsd a variant, fix unittests and + tools/build-on-freebsd. + - FreeBSD: fix test failure + - FreeBSD: replace ifdown/ifup with "ifconfig down" and "ifconfig up". + [Hongjiang Zhang] (LP: #1697815) + - FreeBSD: fix cdrom mounting failure if /mnt/cdrom/secure did not exist. + [Hongjiang Zhang] (LP: #1696295) + - main: Don't use templater to format the welcome message + [Andrew Jorgensen] + - docs: Automatically generate module docs form schema if present. + [Chad Smith] + - debian: fix path comment in /etc/hosts template. + [Jens Sandmann] (LP: #1606406) + - suse: add hostname and fully qualified domain to template. + [Jens Sandmann] + - write_file(s): Print permissions as octal, not decimal [Andrew Jorgensen] + - ci deps: Add --test-distro to read-dependencies to install all deps + [Chad Smith] + - tools/run-centos: cleanups and move to using read-dependencies + - pkg build ci: Add make ci-deps- target to install pkgs + [Chad Smith] + - systemd: make cloud-final.service run before apt daily services. + (LP: #1693361) + - selinux: Allow restorecon to be non-fatal. [Ryan Harper] (LP: #1686751) + - net: Allow netinfo subprocesses to return 0 or 1. + [Ryan Harper] (LP: #1686751) + - net: Allow for NetworkManager configuration [Ryan McCabe] (LP: #1693251) + - Use distro release version to determine if we use systemd in redhat spec + [Ryan Harper] + - net: normalize data in network_state object + - Integration Testing: tox env, pyxld 2.2.3, and revamp framework + [Wesley Wiedenmeier] + - Chef: Update omnibus url to chef.io, minor doc changes. [JJ Asghar] + - tools: add centos scripts to build and test [Joshua Powers] + - Drop cheetah python module as it is not needed by trunk [Ryan Harper] + - rhel/centos spec cleanups. + - cloud.cfg: move to a template. setup.py changes along the way. + - Makefile: add deb-src and srpm targets. use PYVER more places. + - makefile: fix python 2/3 detection in the Makefile [Chad Smith] + - snap: Removing snapcraft plug line [Joshua Powers] (LP: #1695333) + - RHEL/CentOS: Fix default routes for IPv4/IPv6 configuration. + [Andreas Karis] (LP: #1696176) + - test: Fix pyflakes complaint of unused import. + [Joshua Powers] (LP: #1695918) + - NoCloud: support seed of nocloud from smbios information + [Vladimir Pouzanov] (LP: #1691772) + - net: when selecting a network device, use natural sort order + [Marc-Aurèle Brothier] + - fix typos and remove whitespace in various docs [Stephan Telling] + - systemd: Fix typo in comment in cloud-init.target. [Chen-Han Hsiao] + - Tests: Skip jsonschema related unit tests when dependency is absent. + [Chad Smith] (LP: #1695318) + - azure: remove accidental duplicate line in merge. + - azure: identify platform by well known value in chassis asset tag. + [Chad Smith] (LP: #1693939) + - tools/net-convert.py: support old cloudinit versions by using kwargs. + - ntp: Add schema definition and passive schema validation. + [Chad Smith] (LP: #1692916) + - Fix eni rendering for bridge params that require repeated key for + values. [Ryan Harper] + - net: remove systemd link file writing from eni renderer [Ryan Harper] + - AliYun: Enable platform identification and enable by default. + [Junjie Wang] (LP: #1638931) + - net: fix reading and rendering addresses in cidr format. + [Dimitri John Ledkov] (LP: #1689346, #1684349) + - disk_setup: udev settle before attempting partitioning or fs creation. + (LP: #1692093) + - GCE: Update the attribute used to find instance SSH keys. + [Daniel Watkins] (LP: #1693582) + - nplan: For bonds, allow dashed or underscore names of keys. + [Dimitri John Ledkov] (LP: #1690480) + - python2.6: fix unit tests usage of assertNone and format. + - test: update docstring on test_configured_list_with_none + - fix tools/ds-identify to not write None twice. + - tox/build: do not package depend on style requirements. + - cc_ntp: Restructure cc_ntp unit tests. [Chad Smith] (LP: #1692794) + - flake8: move the pinned version of flake8 up to 3.3.0 + - tests: Apply workaround for snapd bug in test case. [Joshua Powers] + - RHEL/CentOS: Fix dual stack IPv4/IPv6 configuration. + [Andreas Karis] (LP: #1679817, #1685534, #1685532) + - disk_setup: fix several issues with gpt disk partitions. (LP: #1692087) + - function spelling & docstring update [Joshua Powers] + - Fixing wrong file name regression. [Joshua Powers] + - tox: move pylint target to 1.7.1 + - Fix get_interfaces_by_mac for empty macs (LP: #1692028) + - DigitalOcean: remove routes except for the public interface. + [Ben Howard] (LP: #1681531.) + - netplan: pass macaddress, when specified, for vlans + [Dimitri John Ledkov] (LP: #1690388) + - doc: various improvements for the docs on cc_users_groups. + [Felix Dreissig] + - cc_ntp: write template before installing and add service restart + [Ryan Harper] (LP: #1645644) + - cloudstack: fix tests to avoid accessing /var/lib/NetworkManager + [Lars Kellogg-Stedman] + - tests: fix hardcoded path to mkfs.ext4 [Joshua Powers] (LP: #1691517) + - Actually skip warnings when .skip file is present. + [Chris Brinker] (LP: #1691551) + - netplan: fix netplan render_network_state signature. + [Dimitri John Ledkov] (LP: #1685944) + - Azure: fix reformatting of ephemeral disks on resize to large types. + (LP: #1686514) + - Revert "tools/net-convert: fix argument order for render_network_state" + - make deb: Add devscripts dependency for make deb. Cleanup + packages/bddeb. [Chad Smith] (LP: #1685935) + - tools/net-convert: fix argument order for render_network_state + [Ryan Harper] (LP: #1685944) + - openstack: fix log message copy/paste typo in _get_url_settings + [Lars Kellogg-Stedman] + - unittests: fix unittests run on centos [Joshua Powers] + - Improve detection of snappy to include os-release and kernel cmdline. + (LP: #1689944) + - Add address to config entry generated by _klibc_to_config_entry. + [Julien Castets] (LP: #1691135) + - sysconfig: Raise ValueError when multiple default gateways are present. + [Chad Smith] (LP: #1687485) + - FreeBSD: improvements and fixes for use on Azure + [Hongjiang Zhang] (LP: #1636345) + - Add unit tests for ds-identify, fix Ec2 bug found. + - fs_setup: if cmd is specified, use shell interpretation. + [Paul Meyer] (LP: #1687712) + - doc: document network configuration defaults policy and formats. + [Ryan Harper] + - Fix name of "uri" key in docs for "cc_apt_configure" module + [Felix Dreissig] + - tests: Enable artful [Joshua Powers] + - nova-lxd: read product_name from environment, not platform. + (LP: #1685810) + - Fix yum repo config where keys contain array values + [Dylan Perry] (LP: #1592150) + - template: Update debian backports template [Joshua Powers] (LP: #1627293) + - rsyslog: replace ~ with stop [Joshua Powers] (LP: #1367899) + - Doc: add additional RTD examples [Joshua Powers] (LP: #1459604) + - Fix growpart for some cases when booted with root=PARTUUID. + (LP: #1684869) + - pylint: update output style to parseable [Joshua Powers] + - pylint: fix all logging warnings [Joshua Powers] + - CloudStack: Add NetworkManager to list of supported DHCP lease dirs. + [Syed] + - net: kernel lies about vlans not stealing mac addresses, when they do + [Dimitri John Ledkov] (LP: #1682871) + - ds-identify: Check correct path for "latest" config drive + [Daniel Watkins] (LP: #1673637) + - doc: Fix example for resolve.conf configuration. + [Jon Grimm] (LP: #1531582) + - Fix examples that reference upstream chef repository. + [Jon Grimm] (LP: #1678145) + - doc: correct grammar and improve clarity in merging documentation. + [David Tagatac] + - doc: Add missing doc link to snap-config module. [Ryan Harper] + - snap: allows for creating cloud-init snap [Joshua Powers] + - DigitalOcean: assign IPv4ll address to lowest indexed interface. + [Ben Howard] + - DigitalOcean: configure all NICs presented in meta-data. [Ben Howard] + - Remove (and/or fix) URL shortener references [Jon Grimm] (LP: #1669727) + - HACKING.rst: more info on filling out contributors agreement. + - util: teach write_file about copy_mode option + [Lars Kellogg-Stedman] (LP: #1644064) + - DigitalOcean: bind resolvers to loopback interface. [Ben Howard] + - tests: fix AltCloud tests to not rely on blkid (LP: #1636531) + - OpenStack: add 'dvs' to the list of physical link types. (LP: #1674946) + - Fix bug that resulted in an attempt to rename bonds or vlans. + (LP: #1669860) + - tests: update OpenNebula and Digital Ocean to not rely on host + interfaces. + - net: in netplan renderer delete known image-builtin content. + (LP: #1675576) + - doc: correct grammar in capabilities.rst [David Tagatac] + - ds-identify: fix detecting of maas datasource. (LP: #1677710) + - netplan: remove debugging prints, add debug logging [Ryan Harper] + - ds-identify: do not write None twice to datasource_list. + - support resizing partition and rootfs on system booted without + initramfs. [Steve Langasek] (LP: #1677376) + - apt_configure: run only when needed. (LP: #1675185) + - OpenStack: identify OpenStack by product 'OpenStack Compute'. + (LP: #1675349) + - GCE: Search GCE in ds-identify, consider serial number in check. + (LP: #1674861) + - Add support for setting hashed passwords [Tore S. Lonoy] (LP: #1570325) + - Fix filesystem creation when using "partition: auto" + [Jonathan Ballet] (LP: #1634678) + - ConfigDrive: support reading config drive data from /config-drive. + (LP: #1673411) + - ds-identify: fix detection of Bigstep datasource. (LP: #1674766) + - test: add running of pylint [Joshua Powers] + - ds-identify: fix bug where filename expansion was left on. + - advertise network config v2 support (NETWORK_CONFIG_V2) in features. + - Bigstep: fix bug when executing in python3. [root] + - Fix unit test when running in a system deployed with cloud-init. + - Bounce network interface for Azure when using the built-in path. + [Brent Baude] (LP: #1674685) + - cloudinit.net: add network config v2 parsing and rendering [Ryan Harper] + - net: Fix incorrect call to isfile [Joshua Powers] (LP: #1674317) + - net: add renderers for automatically selecting the renderer. + - doc: fix config drive doc with regard to unpartitioned disks. + (LP: #1673818) + - test: Adding integratiron test for password as list [Joshua Powers] + - render_network_state: switch arguments around, do not require target + - support 'loopback' as a device type. + - Integration Testing: improve testcase subclassing [Wesley Wiedenmeier] + - gitignore: adding doc/rtd_html [Joshua Powers] + - doc: add instructions for running integration tests via tox. + [Joshua Powers] + - test: avoid differences in 'date' output due to daylight savings. + - Fix chef config module in omnibus install. [Jeremy Melvin] (LP: #1583837) + - Add feature flags to cloudinit.version. [Wesley Wiedenmeier] + - tox: add a citest environment + - Further fix regression to support 'password' for default user. + - fix regression when no chpasswd/list was provided. + - Support chpasswd/list being a list in addition to a string. + [Sergio Lystopad] (LP: #1665694) + - doc: Fix configuration example for cc_set_passwords module. + [Sergio Lystopad] (LP: #1665773) + - net: support both ipv4 and ipv6 gateways in sysconfig. + [Lars Kellogg-Stedman] (LP: #1669504) + - net: do not raise exception for > 3 nameservers + [Lars Kellogg-Stedman] (LP: #1670052) + - ds-identify: report cleanups for config and exit value. (LP: #1669949) + - ds-identify: move default setting for Ec2/strict_id to a global. + - ds-identify: record not found in cloud.cfg and always add None. + - Support warning if the used datasource is not in ds-identify's list. + - tools/ds-identify: make report mode write namespaced results. + - Move warning functionality to cloudinit/warnings.py + - Add profile.d script for showing warnings on login. + - Z99-cloud-locale-test.sh: install and make consistent. + - tools/ds-identify: look at cloud.cfg when looking for ec2 strict_id. + - tools/ds-identify: disable vmware_guest_customization by default. + - tools/ds-identify: ovf identify vmware guest customization. + - Identify Brightbox as an Ec2 datasource user. (LP: #1661693) + - DatasourceEc2: add warning message when not on AWS. + - ds-identify: add reading of datasource/Ec2/strict_id + - tools/ds-identify: add support for found or maybe contributing config. + - tools/ds-identify: read the seed directory on Ec2 + - tools/ds-identify: use quotes in local declarations. + - tools/ds-identify: fix documentation of policy setting in a comment. + - ds-identify: only run once per boot unless --force is given. + - flake8: fix flake8 complaints in previous commit. + - net: correct errors in cloudinit/net/sysconfig.py + [Lars Kellogg-Stedman] (LP: #1665441) + - ec2_utils: fix MetadataLeafDecoder that returned bytes on empty + - apply the runtime configuration written by ds-identify. + - ds-identify: fix checking for filesystem label (LP: #1663735) + - ds-identify: read ds=nocloud properly (LP: #1663723) + - support nova-lxd by reading platform from environment of pid 1. + (LP: #1661797) + - ds-identify: change aarch64 to use the default for non-dmi systems. + - Remove style checking during build and add latest style checks to tox + [Joshua Powers] (LP: #1652329) + - code-style: make master pass pycodestyle (2.3.1) cleanly, currently: + [Joshua Powers] + - manual_cache_clean: When manually cleaning touch a file in instance dir. + - Add tools/ds-identify to identify datasources available. + - Fix small typo and change iso-filename for consistency [Robin Naundorf] + - Fix eni rendering of multiple IPs per interface + [Ryan Harper] (LP: #1657940) + - tools/mock-meta: support python2 or python3 and ipv6 in both. + - tests: remove executable bit on test_net, so it runs, and fix it. + - tests: No longer monkey patch httpretty for python 3.4.2 + - Add 3 ecdsa-sha2-nistp* ssh key types now that they are standardized + [Lars Kellogg-Stedman] (LP: #1658174) + - reset httppretty for each test [Lars Kellogg-Stedman] (LP: #1658200) + - build: fix running Make on a branch with tags other than master + - EC2: Do not cache security credentials on disk + [Andrew Jorgensen] (LP: #1638312) + - doc: Fix typos and clarify some aspects of the part-handler + [Erik M. Bray] + - doc: add some documentation on OpenStack datasource. + - OpenStack: Use timeout and retries from config in get_data. + [Lars Kellogg-Stedman] (LP: #1657130) + - Fixed Misc issues related to VMware customization. [Sankar Tanguturi] + - Fix minor docs typo: perserve > preserve [Jeremy Bicha] + - Use dnf instead of yum when available + [Lars Kellogg-Stedman] (LP: #1647118) + - validate-yaml: use python rather than explicitly python3 + - Get early logging logged, including failures of cmdline url. + 0.7.9: - doc: adjust headers in tests documentation for consistency. - pep8: fix issue found in zesty build with pycodestyle. diff --git a/cloudinit/version.py b/cloudinit/version.py index dff4af04..3255f399 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -4,7 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. -__VERSION__ = "0.7.9" +__VERSION__ = "17.1" FEATURES = [ # supports network config version 1 -- cgit v1.2.3 From 79ce0a234584a50b1c6e2b664b9ccf7a5d1fca58 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 21 Sep 2017 16:59:59 -0400 Subject: tests: remove a temp file used in bootcmd tests. The bootcmd test was leaving files in the tmpdir named ci-FakeExtendedTempFile.XXXXXX. This cleans those up. --- tests/unittests/test_handler/test_handler_bootcmd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index 580017ed..dbf43e0d 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -29,6 +29,7 @@ class FakeExtendedTempFile(object): def __exit__(self, exc_type, exc_value, traceback): self.handle.close() + util.del_file(self.handle.name) class TestBootcmd(CiTestCase): -- cgit v1.2.3 From da6562e21d0b17a0957adc0c5a2c9da076e0d219 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 19 Sep 2017 11:10:09 -0500 Subject: DataSourceOVF: use util.find_devs_with(TYPE=iso9660) DataSourceOVF attempts to find iso files via walking os.listdir('/dev/') which is far too wide. This approach is too invasive and can sometimes race with systemd attempting to fsck and mount devices. Instead, utilize cloudinit.util.find_devs_with to filter devices by criteria (which uses blkid under the covers). This results in fewer attempts to mount block devices which do not contain iso filesystems. Unittest changes include: - cloudinit.tests.helpers; introduce add_patch() helper - Add unittest coverage for DataSourceOVF use of transport_iso9660 LP: #1718287 --- cloudinit/sources/DataSourceOVF.py | 74 ++++++++----- cloudinit/tests/helpers.py | 10 ++ tests/unittests/test_datasource/test_ovf.py | 164 ++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 27 deletions(-) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 24b45d55..ccebf11a 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -375,26 +375,56 @@ def get_ovf_env(dirname): return (None, False) -# Transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): +def maybe_cdrom_device(devname): + """Test if devname matches known list of devices which may contain iso9660 + filesystems. - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + Be helpful in accepting either knames (with no leading /dev/) or full path + names, but do not allow paths outside of /dev/, like /dev/foo/bar/xxx. + """ + if not devname: + return False + elif not isinstance(devname, util.string_types): + raise ValueError("Unexpected input for devname: %s" % devname) + + # resolve '..' and multi '/' elements + devname = os.path.normpath(devname) - devname_regex = os.environ.get(envname, default_regex) + # drop leading '/dev/' + if devname.startswith("/dev/"): + # partition returns tuple (before, partition, after) + devname = devname.partition("/dev/")[-1] + + # ignore leading slash (/sr0), else fail on / in name (foo/bar/xvdc) + if devname.startswith("/"): + devname = devname.split("/")[-1] + elif devname.count("/") > 0: + return False + + # if empty string + if not devname: + return False + + # default_regex matches values in /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + default_regex = r"^(sr[0-9]+|hd[a-z]|xvd.*)" + devname_regex = os.environ.get("CLOUD_INIT_CDROM_DEV_REGEX", default_regex) cdmatch = re.compile(devname_regex) + return cdmatch.match(devname) is not None + + +# Transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + # Go through mounts to see if it was already mounted mounts = util.mounts() for (dev, info) in mounts.items(): fstype = info['fstype'] if fstype != "iso9660" and require_iso: continue - if cdmatch.match(dev[5:]) is None: # take off '/dev/' + if not maybe_cdrom_device(dev): continue mp = info['mountpoint'] (fname, contents) = get_ovf_env(mp) @@ -406,29 +436,19 @@ def transport_iso9660(require_iso=True): else: mtype = None - devs = os.listdir("/dev/") - devs.sort() + # generate a list of devices with mtype filesystem, filter by regex + devs = [dev for dev in + util.find_devs_with("TYPE=%s" % mtype if mtype else None) + if maybe_cdrom_device(dev)] for dev in devs: - fullp = os.path.join("/dev/", dev) - - if (fullp in mounts or - not cdmatch.match(dev) or os.path.isdir(fullp)): - continue - - try: - # See if we can read anything at all...?? - util.peek_file(fullp, 512) - except IOError: - continue - try: - (fname, contents) = util.mount_cb(fullp, get_ovf_env, mtype=mtype) + (fname, contents) = util.mount_cb(dev, get_ovf_env, mtype=mtype) except util.MountFailedError: - LOG.debug("%s not mountable as iso9660", fullp) + LOG.debug("%s not mountable as iso9660", dev) continue if contents is not False: - return (contents, fullp, fname) + return (contents, dev, fname) return (False, None, None) diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 28e26622..6f88a5b7 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -104,6 +104,16 @@ class TestCase(unittest2.TestCase): super(TestCase, self).setUp() self.reset_global_state() + def add_patch(self, target, attr, **kwargs): + """Patches specified target object and sets it as attr on test + instance also schedules cleanup""" + if 'autospec' not in kwargs: + kwargs['autospec'] = True + m = mock.patch(target, **kwargs) + p = m.start() + self.addCleanup(m.stop) + setattr(self, attr, p) + class CiTestCase(TestCase): """This is the preferred test case base class unless user diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 9dbf4dd9..700da86c 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 +from collections import OrderedDict from cloudinit.tests import helpers as test_helpers @@ -70,4 +71,167 @@ class TestReadOvfEnv(test_helpers.TestCase): self.assertEqual({'password': "passw0rd"}, cfg) self.assertIsNone(ud) + +class TestTransportIso9660(test_helpers.CiTestCase): + + def setUp(self): + super(TestTransportIso9660, self).setUp() + self.add_patch('cloudinit.util.find_devs_with', + 'm_find_devs_with') + self.add_patch('cloudinit.util.mounts', 'm_mounts') + self.add_patch('cloudinit.util.mount_cb', 'm_mount_cb') + self.add_patch('cloudinit.sources.DataSourceOVF.get_ovf_env', + 'm_get_ovf_env') + self.m_get_ovf_env.return_value = ('myfile', 'mycontent') + + def test_find_already_mounted(self): + """Check we call get_ovf_env from on matching mounted devices""" + mounts = { + '/dev/sr9': { + 'fstype': 'iso9660', + 'mountpoint': 'wark/media/sr9', + 'opts': 'ro', + } + } + self.m_mounts.return_value = mounts + + (contents, fullp, fname) = dsovf.transport_iso9660() + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/sr9", fullp) + self.assertEqual("myfile", fname) + + def test_find_already_mounted_skips_non_iso9660(self): + """Check we call get_ovf_env ignoring non iso9660""" + mounts = { + '/dev/xvdb': { + 'fstype': 'vfat', + 'mountpoint': 'wark/foobar', + 'opts': 'defaults,noatime', + }, + '/dev/xvdc': { + 'fstype': 'iso9660', + 'mountpoint': 'wark/media/sr9', + 'opts': 'ro', + } + } + # We use an OrderedDict here to ensure we check xvdb before xvdc + # as we're not mocking the regex matching, however, if we place + # an entry in the results then we can be reasonably sure that + # we're skipping an entry which fails to match. + self.m_mounts.return_value = ( + OrderedDict(sorted(mounts.items(), key=lambda t: t[0]))) + + (contents, fullp, fname) = dsovf.transport_iso9660() + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/xvdc", fullp) + self.assertEqual("myfile", fname) + + def test_find_already_mounted_matches_kname(self): + """Check we dont regex match on basename of the device""" + mounts = { + '/dev/foo/bar/xvdc': { + 'fstype': 'iso9660', + 'mountpoint': 'wark/media/sr9', + 'opts': 'ro', + } + } + # we're skipping an entry which fails to match. + self.m_mounts.return_value = mounts + + (contents, fullp, fname) = dsovf.transport_iso9660() + self.assertEqual(False, contents) + self.assertIsNone(fullp) + self.assertIsNone(fname) + + def test_mount_cb_called_on_blkdevs_with_iso9660(self): + """Check we call mount_cb on blockdevs with iso9660 only""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = ['/dev/sr0'] + self.m_mount_cb.return_value = ("myfile", "mycontent") + + (contents, fullp, fname) = dsovf.transport_iso9660() + + self.m_mount_cb.assert_called_with( + "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660") + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/sr0", fullp) + self.assertEqual("myfile", fname) + + def test_mount_cb_called_on_blkdevs_with_iso9660_check_regex(self): + """Check we call mount_cb on blockdevs with iso9660 and match regex""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = [ + '/dev/abc', '/dev/my-cdrom', '/dev/sr0'] + self.m_mount_cb.return_value = ("myfile", "mycontent") + + (contents, fullp, fname) = dsovf.transport_iso9660() + + self.m_mount_cb.assert_called_with( + "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660") + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/sr0", fullp) + self.assertEqual("myfile", fname) + + def test_mount_cb_not_called_no_matches(self): + """Check we don't call mount_cb if nothing matches""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = ['/dev/vg/myovf'] + + (contents, fullp, fname) = dsovf.transport_iso9660() + + self.assertEqual(0, self.m_mount_cb.call_count) + self.assertEqual(False, contents) + self.assertIsNone(fullp) + self.assertIsNone(fname) + + def test_mount_cb_called_require_iso_false(self): + """Check we call mount_cb on blockdevs with require_iso=False""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = ['/dev/xvdz'] + self.m_mount_cb.return_value = ("myfile", "mycontent") + + (contents, fullp, fname) = dsovf.transport_iso9660(require_iso=False) + + self.m_mount_cb.assert_called_with( + "/dev/xvdz", dsovf.get_ovf_env, mtype=None) + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/xvdz", fullp) + self.assertEqual("myfile", fname) + + def test_maybe_cdrom_device_none(self): + """Test maybe_cdrom_device returns False for none/empty input""" + self.assertFalse(dsovf.maybe_cdrom_device(None)) + self.assertFalse(dsovf.maybe_cdrom_device('')) + + def test_maybe_cdrom_device_non_string_exception(self): + """Test maybe_cdrom_device raises ValueError on non-string types""" + with self.assertRaises(ValueError): + dsovf.maybe_cdrom_device({'a': 'eleven'}) + + def test_maybe_cdrom_device_false_on_multi_dir_paths(self): + """Test maybe_cdrom_device is false on /dev[/.*]/* paths""" + self.assertFalse(dsovf.maybe_cdrom_device('/dev/foo/sr0')) + self.assertFalse(dsovf.maybe_cdrom_device('foo/sr0')) + self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0')) + self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0')) + + def test_maybe_cdrom_device_true_on_hd_partitions(self): + """Test maybe_cdrom_device is false on /dev/hd[a-z][0-9]+ paths""" + self.assertTrue(dsovf.maybe_cdrom_device('/dev/hda1')) + self.assertTrue(dsovf.maybe_cdrom_device('hdz9')) + + def test_maybe_cdrom_device_true_on_valid_relative_paths(self): + """Test maybe_cdrom_device normalizes paths""" + self.assertTrue(dsovf.maybe_cdrom_device('/dev/wark/../sr9')) + self.assertTrue(dsovf.maybe_cdrom_device('///sr0')) + self.assertTrue(dsovf.maybe_cdrom_device('/sr0')) + self.assertTrue(dsovf.maybe_cdrom_device('//dev//hda')) + + def test_maybe_cdrom_device_true_on_xvd_partitions(self): + """Test maybe_cdrom_device returns true on xvd*""" + self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda')) + self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda1')) + self.assertTrue(dsovf.maybe_cdrom_device('xvdza1')) + +# # vi: ts=4 expandtab -- cgit v1.2.3 From ad099a53d120e88719a5ad50f29d22e9f7a52bc7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 25 Sep 2017 14:29:13 -0400 Subject: AltCloud: Trust PATH for udevadm and modprobe. Previously we had hard coded paths in /sbin for the udevadm and modprobe programs invoked by AltCloud. Its more flexible to expect the PATH to be set correctly. Debian: #852564 --- cloudinit/sources/DataSourceAltCloud.py | 4 ++-- tests/unittests/test_datasource/test_altcloud.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index ed1d691a..c78ad9eb 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -28,8 +28,8 @@ LOG = logging.getLogger(__name__) CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info' # Shell command lists -CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy'] -CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle', '--timeout=5'] +CMD_PROBE_FLOPPY = ['modprobe', 'floppy'] +CMD_UDEVADM_SETTLE = ['udevadm', 'settle', '--timeout=5'] META_DATA_NOT_SUPPORTED = { 'block-device-mapping': {}, diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 3b274d90..a4dfb540 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -280,8 +280,8 @@ class TestUserDataRhevm(TestCase): pass dsac.CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info' - dsac.CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy'] - dsac.CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle', + dsac.CMD_PROBE_FLOPPY = ['modprobe', 'floppy'] + dsac.CMD_UDEVADM_SETTLE = ['udevadm', 'settle', '--quiet', '--timeout=5'] def test_mount_cb_fails(self): -- cgit v1.2.3 From fd57d50911b9d2a2012dbc2b84c64566b857ab4b Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Thu, 21 Sep 2017 10:16:00 -0700 Subject: tests: remove dependency on shlex This removes shlex and converts the subprocess commands to use a list over a string. --- tests/cloud_tests/instances/nocloudkvm.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/cloud_tests/instances/nocloudkvm.py b/tests/cloud_tests/instances/nocloudkvm.py index 7abfe737..8a0e5319 100644 --- a/tests/cloud_tests/instances/nocloudkvm.py +++ b/tests/cloud_tests/instances/nocloudkvm.py @@ -4,7 +4,6 @@ import os import paramiko -import shlex import socket import subprocess import time @@ -83,10 +82,10 @@ class NoCloudKVMInstance(base.Instance): def mount_image_callback(self, cmd): """Run mount-image-callback.""" - mic = ('sudo mount-image-callback --system-mounts --system-resolvconf ' - '%s -- chroot _MOUNTPOINT_ ' % self.name) - - out, err = c_util.subp(shlex.split(mic) + cmd) + out, err = c_util.subp(['sudo', 'mount-image-callback', + '--system-mounts', '--system-resolvconf', + self.name, '--', 'chroot', + '_MOUNTPOINT_'] + cmd) return out, err @@ -122,11 +121,11 @@ class NoCloudKVMInstance(base.Instance): if self.pid: super(NoCloudKVMInstance, self).push_file() else: - cmd = ("sudo mount-image-callback --system-mounts " - "--system-resolvconf %s -- chroot _MOUNTPOINT_ " - "/bin/sh -c 'cat - > %s'" % (self.name, remote_path)) local_file = open(local_path) - p = subprocess.Popen(shlex.split(cmd), + p = subprocess.Popen(['sudo', 'mount-image-callback', + '--system-mounts', '--system-resolvconf', + self.name, '--', 'chroot', '_MOUNTPOINT_', + '/bin/sh', '-c', 'cat - > %s' % remote_path], stdin=local_file, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -186,12 +185,14 @@ class NoCloudKVMInstance(base.Instance): self.pid_file = os.path.join(tmpdir, '%s.pid' % self.name) self.ssh_port = self.get_free_port() - cmd = ('./tools/xkvm --disk %s,cache=unsafe --disk %s,cache=unsafe ' - '--netdev user,hostfwd=tcp::%s-:22 ' - '-- -pidfile %s -vnc none -m 2G -smp 2' - % (self.name, seed, self.ssh_port, self.pid_file)) - - subprocess.Popen(shlex.split(cmd), close_fds=True, + subprocess.Popen(['./tools/xkvm', + '--disk', '%s,cache=unsafe' % self.name, + '--disk', '%s,cache=unsafe' % seed, + '--netdev', + 'user,hostfwd=tcp::%s-:22' % self.ssh_port, + '--', '-pidfile', self.pid_file, '-vnc', 'none', + '-m', '2G', '-smp', '2'], + close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -- cgit v1.2.3 From d32049993a8e719c52cb491dd8cc7935bfede2d3 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Fri, 29 Sep 2017 08:53:05 -0400 Subject: debian/copyright: dep5 updates, reorganize, add Apache 2.0 license. The copyright was updated to be lintian clean and reorganized to list the licenses at the bottom after declaring the metadata and file information. Add the MIT license to the file. LP: #1718681 --- packages/debian/copyright | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/packages/debian/copyright b/packages/debian/copyright index c9c7d231..c2236702 100644 --- a/packages/debian/copyright +++ b/packages/debian/copyright @@ -1,33 +1,32 @@ -Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?op=file&rev=135 -Name: cloud-init -Maintainer: Scott Moser +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: cloud-init +Upstream-Contact: cloud-init-dev@lists.launchpad.net Source: https://launchpad.net/cloud-init -This package was debianized by Soren Hansen on -Thu, 04 Sep 2008 12:49:15 +0200 as ec2-init. It was later renamed to -cloud-init by Scott Moser +Files: * +Copyright: 2010, Canonical Ltd. +License: GPL-3 or Apache-2.0 -Upstream Author: Scott Moser - Soren Hansen - Chuck Short +Files: cloudinit/boto_utils.py +Copyright: 2006,2007, Mitch Garnaat http://garnaat.org/ +License: MIT -Copyright: 2010, Canonical Ltd. -License: GPL-3 or Apache-2.0 License: GPL-3 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 3, as published by the Free Software Foundation. - + . This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - + . You should have received a copy of the GNU General Public License along with this program. If not, see . - + . The complete text of the GPL version 3 can be seen in /usr/share/common-licenses/GPL-3. + License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -43,3 +42,23 @@ License: Apache-2.0 . On Debian-based systems the full text of the Apache version 2.0 license can be found in `/usr/share/common-licenses/Apache-2.0'. + +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, dis- + tribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the fol- + lowing conditions: + . + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- + ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + IN THE SOFTWARE. -- cgit v1.2.3 From f010594beb75e146091db47b7d72d1fc1d763e98 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Mon, 2 Oct 2017 12:53:56 -0600 Subject: Remove prettytable dependency, introduce simpletable The first revision of this rendered tables with less decoration but there was a desire upstream to avoid possibly breaking some parsing someone might be doing, so it has been revised to render the same as prettytable for the cases cloud-init actually uses. --- cloudinit/config/cc_ssh_authkey_fingerprints.py | 4 ++-- cloudinit/netinfo.py | 8 ++++---- packages/pkg-deps.json | 3 --- requirements.txt | 3 --- tools/build-on-freebsd | 1 - tox.ini | 3 --- 6 files changed, 6 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 0066e97f..35d8c57f 100755 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -28,7 +28,7 @@ the keys can be specified, but defaults to ``md5``. import base64 import hashlib -from prettytable import PrettyTable +from cloudinit.simpletable import SimpleTable from cloudinit.distros import ug_util from cloudinit import ssh_util @@ -74,7 +74,7 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', return tbl_fields = ['Keytype', 'Fingerprint (%s)' % (hash_meth), 'Options', 'Comment'] - tbl = PrettyTable(tbl_fields) + tbl = SimpleTable(tbl_fields) for entry in key_entries: if _is_printable_key(entry): row = [] diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 39c79dee..8f99d99c 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -13,7 +13,7 @@ import re from cloudinit import log as logging from cloudinit import util -from prettytable import PrettyTable +from cloudinit.simpletable import SimpleTable LOG = logging.getLogger() @@ -170,7 +170,7 @@ def netdev_pformat(): lines.append(util.center("Net device info failed", '!', 80)) else: fields = ['Device', 'Up', 'Address', 'Mask', 'Scope', 'Hw-Address'] - tbl = PrettyTable(fields) + tbl = SimpleTable(fields) for (dev, d) in netdev.items(): tbl.add_row([dev, d["up"], d["addr"], d["mask"], ".", d["hwaddr"]]) if d.get('addr6'): @@ -194,7 +194,7 @@ def route_pformat(): if routes.get('ipv4'): fields_v4 = ['Route', 'Destination', 'Gateway', 'Genmask', 'Interface', 'Flags'] - tbl_v4 = PrettyTable(fields_v4) + tbl_v4 = SimpleTable(fields_v4) for (n, r) in enumerate(routes.get('ipv4')): route_id = str(n) tbl_v4.add_row([route_id, r['destination'], @@ -207,7 +207,7 @@ def route_pformat(): if routes.get('ipv6'): fields_v6 = ['Route', 'Proto', 'Recv-Q', 'Send-Q', 'Local Address', 'Foreign Address', 'State'] - tbl_v6 = PrettyTable(fields_v6) + tbl_v6 = SimpleTable(fields_v6) for (n, r) in enumerate(routes.get('ipv6')): route_id = str(n) tbl_v6.add_row([route_id, r['proto'], diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index 822d29d9..72409dd8 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -34,9 +34,6 @@ "jsonschema" : { "3" : "python34-jsonschema" }, - "prettytable" : { - "3" : "python34-prettytable" - }, "pyflakes" : { "2" : "pyflakes", "3" : "python34-pyflakes" diff --git a/requirements.txt b/requirements.txt index 61d1e90b..dd10d85d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,6 @@ # Used for untemplating any files or strings with parameters. jinja2 -# This is used for any pretty printing of tabular data. -PrettyTable - # This one is currently only used by the MAAS datasource. If that # datasource is removed, this is no longer needed oauthlib diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index ff9153ad..d23fde2b 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -18,7 +18,6 @@ pkgs=" py27-jsonpatch py27-jsonpointer py27-oauthlib - py27-prettytable py27-requests py27-serial py27-six diff --git a/tox.ini b/tox.ini index 776f4253..aef1f84b 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,6 @@ deps = # requirements jinja2==2.8 pyyaml==3.11 - PrettyTable==0.7.2 oauthlib==1.0.3 pyserial==3.0.1 configobj==5.0.6 @@ -89,7 +88,6 @@ deps = argparse==1.2.1 jinja2==2.2.1 pyyaml==3.10 - PrettyTable==0.7.2 oauthlib==0.6.0 configobj==4.6.0 requests==2.6.0 @@ -105,7 +103,6 @@ deps = argparse==1.3.0 jinja2==2.8 PyYAML==3.11 - PrettyTable==0.7.2 oauthlib==0.7.2 configobj==5.0.6 requests==2.11.1 -- cgit v1.2.3 From b3acdff390329c8091b880b490e1f27441a7a486 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Mon, 2 Oct 2017 13:17:39 -0600 Subject: Add missing simpletable and simpletable tests for failed merge --- cloudinit/simpletable.py | 62 ++++++++++++++++++++++ cloudinit/tests/test_simpletable.py | 100 ++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 cloudinit/simpletable.py create mode 100644 cloudinit/tests/test_simpletable.py diff --git a/cloudinit/simpletable.py b/cloudinit/simpletable.py new file mode 100644 index 00000000..90603228 --- /dev/null +++ b/cloudinit/simpletable.py @@ -0,0 +1,62 @@ +# Copyright (C) 2017 Amazon.com, Inc. or its affiliates +# +# Author: Ethan Faust +# Author: Andrew Jorgensen +# +# This file is part of cloud-init. See LICENSE file for license information. + + +class SimpleTable(object): + """A minimal implementation of PrettyTable + for distribution with cloud-init. + """ + + def __init__(self, fields): + self.fields = fields + self.rows = [] + + # initialize list of 0s the same length + # as the number of fields + self.column_widths = [0] * len(self.fields) + self.update_column_widths(fields) + + def update_column_widths(self, values): + for i, value in enumerate(values): + self.column_widths[i] = max( + len(value), + self.column_widths[i]) + + def add_row(self, values): + if len(values) > len(self.fields): + raise TypeError('too many values') + values = [str(value) for value in values] + self.rows.append(values) + self.update_column_widths(values) + + def _hdiv(self): + """Returns a horizontal divider for the table.""" + return '+' + '+'.join( + ['-' * (w + 2) for w in self.column_widths]) + '+' + + def _row(self, row): + """Returns a formatted row.""" + return '|' + '|'.join( + [col.center(self.column_widths[i] + 2) + for i, col in enumerate(row)]) + '|' + + def __str__(self): + """Returns a string representation of the table with lines around. + + +-----+-----+ + | one | two | + +-----+-----+ + | 1 | 2 | + | 01 | 10 | + +-----+-----+ + """ + lines = [self._hdiv(), self._row(self.fields), self._hdiv()] + lines += [self._row(r) for r in self.rows] + [self._hdiv()] + return '\n'.join(lines) + + def get_string(self): + return repr(self) diff --git a/cloudinit/tests/test_simpletable.py b/cloudinit/tests/test_simpletable.py new file mode 100644 index 00000000..96bc24cf --- /dev/null +++ b/cloudinit/tests/test_simpletable.py @@ -0,0 +1,100 @@ +# Copyright (C) 2017 Amazon.com, Inc. or its affiliates +# +# Author: Andrew Jorgensen +# +# This file is part of cloud-init. See LICENSE file for license information. +"""Tests that SimpleTable works just like PrettyTable for cloud-init. + +Not all possible PrettyTable cases are tested because we're not trying to +reimplement the entire library, only the minimal parts we actually use. +""" + +from cloudinit.simpletable import SimpleTable +from cloudinit.tests.helpers import CiTestCase + +# Examples rendered by cloud-init using PrettyTable +NET_DEVICE_FIELDS = ( + 'Device', 'Up', 'Address', 'Mask', 'Scope', 'Hw-Address') +NET_DEVICE_ROWS = ( + ('ens3', True, '172.31.4.203', '255.255.240.0', '.', '0a:1f:07:15:98:70'), + ('ens3', True, 'fe80::81f:7ff:fe15:9870/64', '.', 'link', + '0a:1f:07:15:98:70'), + ('lo', True, '127.0.0.1', '255.0.0.0', '.', '.'), + ('lo', True, '::1/128', '.', 'host', '.'), +) +NET_DEVICE_TABLE = """\ ++--------+------+----------------------------+---------------+-------+-------------------+ +| Device | Up | Address | Mask | Scope | Hw-Address | ++--------+------+----------------------------+---------------+-------+-------------------+ +| ens3 | True | 172.31.4.203 | 255.255.240.0 | . | 0a:1f:07:15:98:70 | +| ens3 | True | fe80::81f:7ff:fe15:9870/64 | . | link | 0a:1f:07:15:98:70 | +| lo | True | 127.0.0.1 | 255.0.0.0 | . | . | +| lo | True | ::1/128 | . | host | . | ++--------+------+----------------------------+---------------+-------+-------------------+""" # noqa: E501 +ROUTE_IPV4_FIELDS = ( + 'Route', 'Destination', 'Gateway', 'Genmask', 'Interface', 'Flags') +ROUTE_IPV4_ROWS = ( + ('0', '0.0.0.0', '172.31.0.1', '0.0.0.0', 'ens3', 'UG'), + ('1', '169.254.0.0', '0.0.0.0', '255.255.0.0', 'ens3', 'U'), + ('2', '172.31.0.0', '0.0.0.0', '255.255.240.0', 'ens3', 'U'), +) +ROUTE_IPV4_TABLE = """\ ++-------+-------------+------------+---------------+-----------+-------+ +| Route | Destination | Gateway | Genmask | Interface | Flags | ++-------+-------------+------------+---------------+-----------+-------+ +| 0 | 0.0.0.0 | 172.31.0.1 | 0.0.0.0 | ens3 | UG | +| 1 | 169.254.0.0 | 0.0.0.0 | 255.255.0.0 | ens3 | U | +| 2 | 172.31.0.0 | 0.0.0.0 | 255.255.240.0 | ens3 | U | ++-------+-------------+------------+---------------+-----------+-------+""" + +AUTHORIZED_KEYS_FIELDS = ( + 'Keytype', 'Fingerprint (md5)', 'Options', 'Comment') +AUTHORIZED_KEYS_ROWS = ( + ('ssh-rsa', '24:c7:41:49:47:12:31:a0:de:6f:62:79:9b:13:06:36', '-', + 'ajorgens'), +) +AUTHORIZED_KEYS_TABLE = """\ ++---------+-------------------------------------------------+---------+----------+ +| Keytype | Fingerprint (md5) | Options | Comment | ++---------+-------------------------------------------------+---------+----------+ +| ssh-rsa | 24:c7:41:49:47:12:31:a0:de:6f:62:79:9b:13:06:36 | - | ajorgens | ++---------+-------------------------------------------------+---------+----------+""" # noqa: E501 + +# from prettytable import PrettyTable +# pt = PrettyTable(('HEADER',)) +# print(pt) +NO_ROWS_FIELDS = ('HEADER',) +NO_ROWS_TABLE = """\ ++--------+ +| HEADER | ++--------+ ++--------+""" + + +class TestSimpleTable(CiTestCase): + + def test_no_rows(self): + """An empty table is rendered as PrettyTable would have done it.""" + table = SimpleTable(NO_ROWS_FIELDS) + self.assertEqual(str(table), NO_ROWS_TABLE) + + def test_net_dev(self): + """Net device info is rendered as it was with PrettyTable.""" + table = SimpleTable(NET_DEVICE_FIELDS) + for row in NET_DEVICE_ROWS: + table.add_row(row) + self.assertEqual(str(table), NET_DEVICE_TABLE) + + def test_route_ipv4(self): + """Route IPv4 info is rendered as it was with PrettyTable.""" + table = SimpleTable(ROUTE_IPV4_FIELDS) + for row in ROUTE_IPV4_ROWS: + table.add_row(row) + self.assertEqual(str(table), ROUTE_IPV4_TABLE) + + def test_authorized_keys(self): + """SSH authorized keys are rendered as they were with PrettyTable.""" + table = SimpleTable(AUTHORIZED_KEYS_FIELDS) + for row in AUTHORIZED_KEYS_ROWS: + table.add_row(row) + self.assertEqual(str(table), AUTHORIZED_KEYS_TABLE) -- cgit v1.2.3 From 6f2aaf7b0fa9fd9376561e9b6233b4d36de51da1 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Sun, 24 Sep 2017 15:10:36 -0400 Subject: systemd: only mention Before=apt-daily.service on debian based distros. Ordering on apt service should only be set up on Debian based distributions. This changes is really a net-zero in runtime result. But, mentioning apt on a rpm based distro could be confusing. --- systemd/cloud-final.service.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/systemd/cloud-final.service.tmpl b/systemd/cloud-final.service.tmpl index fc01b891..e2b91255 100644 --- a/systemd/cloud-final.service.tmpl +++ b/systemd/cloud-final.service.tmpl @@ -4,9 +4,10 @@ Description=Execute cloud user/final scripts After=network-online.target cloud-config.service rc-local.service {% if variant in ["ubuntu", "unknown", "debian"] %} After=multi-user.target +Before=apt-daily.service {% endif %} Wants=network-online.target cloud-config.service -Before=apt-daily.service + [Service] Type=oneshot -- cgit v1.2.3 From 946232bb9eda2f4bc66c4464db9e72d3edfd9900 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 2 Oct 2017 15:20:10 -0400 Subject: packages/debian/copyright: remove mention of boto and MIT license boto_utils.py had been removed some time ago, and the current cloudinit/ec2_utils.py is not based on what was in boto_utils. We just failed to remove the mention of it from the upstream debian/copyright. And then put it back in everywhere in recent changes to get upstream and ubuntu in sync. --- packages/debian/copyright | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/packages/debian/copyright b/packages/debian/copyright index c2236702..598cda14 100644 --- a/packages/debian/copyright +++ b/packages/debian/copyright @@ -7,10 +7,6 @@ Files: * Copyright: 2010, Canonical Ltd. License: GPL-3 or Apache-2.0 -Files: cloudinit/boto_utils.py -Copyright: 2006,2007, Mitch Garnaat http://garnaat.org/ -License: MIT - License: GPL-3 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 3, as @@ -42,23 +38,3 @@ License: Apache-2.0 . On Debian-based systems the full text of the Apache version 2.0 license can be found in `/usr/share/common-licenses/Apache-2.0'. - -License: MIT - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, dis- - tribute, sublicense, and/or sell copies of the Software, and to permit - persons to whom the Software is furnished to do so, subject to the fol- - lowing conditions: - . - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- - ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT - SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - IN THE SOFTWARE. -- cgit v1.2.3 From 9d2a87dc386b7aed1a8243d599676e78ed358749 Mon Sep 17 00:00:00 2001 From: Dimitri John Ledkov Date: Sat, 30 Sep 2017 00:00:18 -0400 Subject: Azure, CloudStack: Support reading dhcp options from systemd-networkd. Systems that used systemd-networkd's dhcp client would not be able to get information on the Azure endpoint (placed in Option 245) or the CloudStack server (in 'server_address'). The change here supports reading these files in /run/systemd/netif/leases. The files declare that "This is private data. Do not parse.", but at this point we do not have another option. LP: #1718029 --- cloudinit/net/dhcp.py | 42 ++++++ cloudinit/net/tests/test_dhcp.py | 113 +++++++++++++++- cloudinit/sources/DataSourceCloudStack.py | 17 ++- cloudinit/sources/helpers/azure.py | 20 ++- .../unittests/test_datasource/test_azure_helper.py | 143 ++++++++++++++------- tests/unittests/test_datasource/test_cloudstack.py | 11 +- 6 files changed, 282 insertions(+), 64 deletions(-) diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 05350639..0cba7032 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -4,6 +4,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import configobj import logging import os import re @@ -11,9 +12,12 @@ import re from cloudinit.net import find_fallback_nic, get_devicelist from cloudinit import temp_utils from cloudinit import util +from six import StringIO LOG = logging.getLogger(__name__) +NETWORKD_LEASES_DIR = '/run/systemd/netif/leases' + class InvalidDHCPLeaseFileError(Exception): """Raised when parsing an empty or invalid dhcp.leases file. @@ -118,4 +122,42 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): return parse_dhcp_lease_file(lease_file) +def networkd_parse_lease(content): + """Parse a systemd lease file content as in /run/systemd/netif/leases/ + + Parse this (almost) ini style file even though it says: + # This is private data. Do not parse. + + Simply return a dictionary of key/values.""" + + return dict(configobj.ConfigObj(StringIO(content), list_values=False)) + + +def networkd_load_leases(leases_d=None): + """Return a dictionary of dictionaries representing each lease + found in lease_d.i + + The top level key will be the filename, which is typically the ifindex.""" + + if leases_d is None: + leases_d = NETWORKD_LEASES_DIR + + ret = {} + if not os.path.isdir(leases_d): + return ret + for lfile in os.listdir(leases_d): + ret[lfile] = networkd_parse_lease( + util.load_file(os.path.join(leases_d, lfile))) + return ret + + +def networkd_get_option_from_leases(keyname, leases_d=None): + if leases_d is None: + leases_d = NETWORKD_LEASES_DIR + leases = networkd_load_leases(leases_d=leases_d) + for ifindex, data in sorted(leases.items()): + if data.get(keyname): + return data[keyname] + return None + # vi: ts=4 expandtab diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index a38edaec..1c1f504a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -6,9 +6,9 @@ from textwrap import dedent from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, - parse_dhcp_lease_file, dhcp_discovery) + parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases) from cloudinit.util import ensure_file, write_file -from cloudinit.tests.helpers import CiTestCase, wrap_and_call +from cloudinit.tests.helpers import CiTestCase, wrap_and_call, populate_dir class TestParseDHCPLeasesFile(CiTestCase): @@ -149,3 +149,112 @@ class TestDHCPDiscoveryClean(CiTestCase): [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf', lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), 'eth9', '-sf', '/bin/true'], capture=True)]) + + +class TestSystemdParseLeases(CiTestCase): + + lxd_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.75.205.242 + NETMASK=255.255.255.0 + ROUTER=10.75.205.1 + SERVER_ADDRESS=10.75.205.1 + NEXT_SERVER=10.75.205.1 + BROADCAST=10.75.205.255 + T1=1580 + T2=2930 + LIFETIME=3600 + DNS=10.75.205.1 + DOMAINNAME=lxd + HOSTNAME=a1 + CLIENTID=ffe617693400020000ab110c65a6a0866931c2 + """) + + lxd_parsed = { + 'ADDRESS': '10.75.205.242', + 'NETMASK': '255.255.255.0', + 'ROUTER': '10.75.205.1', + 'SERVER_ADDRESS': '10.75.205.1', + 'NEXT_SERVER': '10.75.205.1', + 'BROADCAST': '10.75.205.255', + 'T1': '1580', + 'T2': '2930', + 'LIFETIME': '3600', + 'DNS': '10.75.205.1', + 'DOMAINNAME': 'lxd', + 'HOSTNAME': 'a1', + 'CLIENTID': 'ffe617693400020000ab110c65a6a0866931c2', + } + + azure_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.132.0.5 + NETMASK=255.255.255.255 + ROUTER=10.132.0.1 + SERVER_ADDRESS=169.254.169.254 + NEXT_SERVER=10.132.0.1 + MTU=1460 + T1=43200 + T2=75600 + LIFETIME=86400 + DNS=169.254.169.254 + NTP=169.254.169.254 + DOMAINNAME=c.ubuntu-foundations.internal + DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal + HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal + ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 + CLIENTID=ff405663a200020000ab11332859494d7a8b4c + OPTION_245=624c3620 + """) + + azure_parsed = { + 'ADDRESS': '10.132.0.5', + 'NETMASK': '255.255.255.255', + 'ROUTER': '10.132.0.1', + 'SERVER_ADDRESS': '169.254.169.254', + 'NEXT_SERVER': '10.132.0.1', + 'MTU': '1460', + 'T1': '43200', + 'T2': '75600', + 'LIFETIME': '86400', + 'DNS': '169.254.169.254', + 'NTP': '169.254.169.254', + 'DOMAINNAME': 'c.ubuntu-foundations.internal', + 'DOMAIN_SEARCH_LIST': 'c.ubuntu-foundations.internal google.internal', + 'HOSTNAME': 'tribaal-test-171002-1349.c.ubuntu-foundations.internal', + 'ROUTES': '10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1', + 'CLIENTID': 'ff405663a200020000ab11332859494d7a8b4c', + 'OPTION_245': '624c3620'} + + def setUp(self): + super(TestSystemdParseLeases, self).setUp() + self.lease_d = self.tmp_dir() + + def test_no_leases_returns_empty_dict(self): + """A leases dir with no lease files should return empty dictionary.""" + self.assertEqual({}, networkd_load_leases(self.lease_d)) + + def test_no_leases_dir_returns_empty_dict(self): + """A non-existing leases dir should return empty dict.""" + enodir = os.path.join(self.lease_d, 'does-not-exist') + self.assertEqual({}, networkd_load_leases(enodir)) + + def test_single_leases_file(self): + """A leases dir with one leases file.""" + populate_dir(self.lease_d, {'2': self.lxd_lease}) + self.assertEqual( + {'2': self.lxd_parsed}, networkd_load_leases(self.lease_d)) + + def test_single_azure_leases_file(self): + """On Azure, option 245 should be present, verify it specifically.""" + populate_dir(self.lease_d, {'1': self.azure_lease}) + self.assertEqual( + {'1': self.azure_parsed}, networkd_load_leases(self.lease_d)) + + def test_multiple_files(self): + """Multiple leases files on azure with one found return that value.""" + self.maxDiff = None + populate_dir(self.lease_d, {'1': self.azure_lease, + '9': self.lxd_lease}) + self.assertEqual({'1': self.azure_parsed, '9': self.lxd_parsed}, + networkd_load_leases(self.lease_d)) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7e0f9bb8..9dc473fc 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -19,6 +19,7 @@ import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit.net import dhcp from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util @@ -224,20 +225,28 @@ def get_vr_address(): # Get the address of the virtual router via dhcp leases # If no virtual router is detected, fallback on default gateway. # See http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/4.8/virtual_machines/user-data.html # noqa + + # Try networkd first... + latest_address = dhcp.networkd_get_option_from_leases('SERVER_ADDRESS') + if latest_address: + LOG.debug("Found SERVER_ADDRESS '%s' via networkd_leases", + latest_address) + return latest_address + + # Try dhcp lease files next... lease_file = get_latest_lease() if not lease_file: LOG.debug("No lease file found, using default gateway") return get_default_gateway() - latest_address = None with open(lease_file, "r") as fd: for line in fd: if "dhcp-server-identifier" in line: words = line.strip(" ;\r\n").split(" ") if len(words) > 2: - dhcp = words[2] - LOG.debug("Found DHCP identifier %s", dhcp) - latest_address = dhcp + dhcptok = words[2] + LOG.debug("Found DHCP identifier %s", dhcptok) + latest_address = dhcptok if not latest_address: # No virtual router found, fallback on default gateway LOG.debug("No DHCP found, using default gateway") diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 28ed0ae2..959b1bda 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -8,6 +8,7 @@ import socket import struct import time +from cloudinit.net import dhcp from cloudinit import stages from cloudinit import temp_utils from contextlib import contextmanager @@ -15,7 +16,6 @@ from xml.etree import ElementTree from cloudinit import util - LOG = logging.getLogger(__name__) @@ -238,6 +238,11 @@ class WALinuxAgentShim(object): packed_bytes = unescaped_value.encode('utf-8') return socket.inet_ntoa(packed_bytes) + @staticmethod + def _networkd_get_value_from_leases(leases_d=None): + return dhcp.networkd_get_option_from_leases( + 'OPTION_245', leases_d=leases_d) + @staticmethod def _get_value_from_leases_file(fallback_lease_file): leases = [] @@ -287,12 +292,15 @@ class WALinuxAgentShim(object): @staticmethod def find_endpoint(fallback_lease_file=None): - LOG.debug('Finding Azure endpoint...') value = None - # Option-245 stored in /run/cloud-init/dhclient.hooks/.json - # a dhclient exit hook that calls cloud-init-dhclient-hook - dhcp_options = WALinuxAgentShim._load_dhclient_json() - value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) + LOG.debug('Finding Azure endpoint from networkd...') + value = WALinuxAgentShim._networkd_get_value_from_leases() + if value is None: + # Option-245 stored in /run/cloud-init/dhclient.hooks/.json + # a dhclient exit hook that calls cloud-init-dhclient-hook + LOG.debug('Finding Azure endpoint from hook json...') + dhcp_options = WALinuxAgentShim._load_dhclient_json() + value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) if value is None: # Fallback and check the leases file if unsuccessful LOG.debug("Unable to find endpoint in dhclient logs. " diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 44b99eca..b42b073f 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,10 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. import os +from textwrap import dedent from cloudinit.sources.helpers import azure as azure_helper -from cloudinit.tests.helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, populate_dir +from cloudinit.sources.helpers.azure import WALinuxAgentShim as wa_shim GOAL_STATE_TEMPLATE = """\ @@ -45,7 +47,7 @@ GOAL_STATE_TEMPLATE = """\ """ -class TestFindEndpoint(TestCase): +class TestFindEndpoint(CiTestCase): def setUp(self): super(TestFindEndpoint, self).setUp() @@ -56,18 +58,19 @@ class TestFindEndpoint(TestCase): mock.patch.object(azure_helper.util, 'load_file')) self.dhcp_options = patches.enter_context( - mock.patch.object(azure_helper.WALinuxAgentShim, - '_load_dhclient_json')) + mock.patch.object(wa_shim, '_load_dhclient_json')) + + self.networkd_leases = patches.enter_context( + mock.patch.object(wa_shim, '_networkd_get_value_from_leases')) + self.networkd_leases.return_value = None def test_missing_file(self): - self.assertRaises(ValueError, - azure_helper.WALinuxAgentShim.find_endpoint) + self.assertRaises(ValueError, wa_shim.find_endpoint) def test_missing_special_azure_line(self): self.load_file.return_value = '' self.dhcp_options.return_value = {'eth0': {'key': 'value'}} - self.assertRaises(ValueError, - azure_helper.WALinuxAgentShim.find_endpoint) + self.assertRaises(ValueError, wa_shim.find_endpoint) @staticmethod def _build_lease_content(encoded_address): @@ -80,8 +83,7 @@ class TestFindEndpoint(TestCase): def test_from_dhcp_client(self): self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}} - self.assertEqual('5.4.3.2', - azure_helper.WALinuxAgentShim.find_endpoint(None)) + self.assertEqual('5.4.3.2', wa_shim.find_endpoint(None)) def test_latest_lease_used(self): encoded_addresses = ['5:4:3:2', '4:3:2:1'] @@ -89,53 +91,38 @@ class TestFindEndpoint(TestCase): for encoded_address in encoded_addresses]) self.load_file.return_value = file_content self.assertEqual(encoded_addresses[-1].replace(':', '.'), - azure_helper.WALinuxAgentShim.find_endpoint("foobar")) + wa_shim.find_endpoint("foobar")) -class TestExtractIpAddressFromLeaseValue(TestCase): +class TestExtractIpAddressFromLeaseValue(CiTestCase): def test_hex_string(self): ip_address, encoded_address = '98.76.54.32', '62:4c:36:20' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_hex_string_with_single_character_part(self): ip_address, encoded_address = '4.3.2.1', '4:3:2:1' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string(self): ip_address, encoded_address = '98.76.54.32', 'bL6 ' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string_with_escaped_quote(self): ip_address, encoded_address = '100.72.34.108', 'dH\\"l' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) def test_packed_string_containing_a_colon(self): ip_address, encoded_address = '100.72.58.108', 'dH:l' self.assertEqual( - ip_address, - azure_helper.WALinuxAgentShim.get_ip_from_lease_value( - encoded_address - )) + ip_address, wa_shim.get_ip_from_lease_value(encoded_address)) -class TestGoalStateParsing(TestCase): +class TestGoalStateParsing(CiTestCase): default_parameters = { 'incarnation': 1, @@ -195,7 +182,7 @@ class TestGoalStateParsing(TestCase): self.assertIsNone(certificates_xml) -class TestAzureEndpointHttpClient(TestCase): +class TestAzureEndpointHttpClient(CiTestCase): regular_headers = { 'x-ms-agent-name': 'WALinuxAgent', @@ -258,7 +245,7 @@ class TestAzureEndpointHttpClient(TestCase): self.read_file_or_url.call_args) -class TestOpenSSLManager(TestCase): +class TestOpenSSLManager(CiTestCase): def setUp(self): super(TestOpenSSLManager, self).setUp() @@ -300,7 +287,7 @@ class TestOpenSSLManager(TestCase): self.assertEqual([mock.call(manager.tmpdir)], del_dir.call_args_list) -class TestWALinuxAgentShim(TestCase): +class TestWALinuxAgentShim(CiTestCase): def setUp(self): super(TestWALinuxAgentShim, self).setUp() @@ -310,8 +297,7 @@ class TestWALinuxAgentShim(TestCase): self.AzureEndpointHttpClient = patches.enter_context( mock.patch.object(azure_helper, 'AzureEndpointHttpClient')) self.find_endpoint = patches.enter_context( - mock.patch.object( - azure_helper.WALinuxAgentShim, 'find_endpoint')) + mock.patch.object(wa_shim, 'find_endpoint')) self.GoalState = patches.enter_context( mock.patch.object(azure_helper, 'GoalState')) self.OpenSSLManager = patches.enter_context( @@ -320,7 +306,7 @@ class TestWALinuxAgentShim(TestCase): mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) def test_http_client_uses_certificate(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.OpenSSLManager.return_value.certificate)], @@ -328,7 +314,7 @@ class TestWALinuxAgentShim(TestCase): def test_correct_url_used_for_goalstate(self): self.find_endpoint.return_value = 'test_endpoint' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() get = self.AzureEndpointHttpClient.return_value.get self.assertEqual( @@ -340,7 +326,7 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.call_args_list) def test_certificates_used_to_determine_public_keys(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() data = shim.register_with_azure_and_fetch_data() self.assertEqual( [mock.call(self.GoalState.return_value.certificates_xml)], @@ -351,13 +337,13 @@ class TestWALinuxAgentShim(TestCase): def test_absent_certificates_produces_empty_public_keys(self): self.GoalState.return_value.certificates_xml = None - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() data = shim.register_with_azure_and_fetch_data() self.assertEqual([], data['public-keys']) def test_correct_url_used_for_report_ready(self): self.find_endpoint.return_value = 'test_endpoint' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() expected_url = 'http://test_endpoint/machine?comp=health' self.assertEqual( @@ -368,7 +354,7 @@ class TestWALinuxAgentShim(TestCase): self.GoalState.return_value.incarnation = 'TestIncarnation' self.GoalState.return_value.container_id = 'TestContainerId' self.GoalState.return_value.instance_id = 'TestInstanceId' - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() posted_document = ( self.AzureEndpointHttpClient.return_value.post.call_args[1]['data'] @@ -378,11 +364,11 @@ class TestWALinuxAgentShim(TestCase): self.assertIn('TestInstanceId', posted_document) def test_clean_up_can_be_called_at_any_time(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.clean_up() def test_clean_up_will_clean_up_openssl_manager_if_instantiated(self): - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() shim.register_with_azure_and_fetch_data() shim.clean_up() self.assertEqual( @@ -393,12 +379,12 @@ class TestWALinuxAgentShim(TestCase): pass self.AzureEndpointHttpClient.return_value.get.side_effect = ( SentinelException) - shim = azure_helper.WALinuxAgentShim() + shim = wa_shim() self.assertRaises(SentinelException, shim.register_with_azure_and_fetch_data) -class TestGetMetadataFromFabric(TestCase): +class TestGetMetadataFromFabric(CiTestCase): @mock.patch.object(azure_helper, 'WALinuxAgentShim') def test_data_from_shim_returned(self, shim): @@ -422,4 +408,65 @@ class TestGetMetadataFromFabric(TestCase): azure_helper.get_metadata_from_fabric) self.assertEqual(1, shim.return_value.clean_up.call_count) + +class TestExtractIpAddressFromNetworkd(CiTestCase): + + azure_lease = dedent("""\ + # This is private data. Do not parse. + ADDRESS=10.132.0.5 + NETMASK=255.255.255.255 + ROUTER=10.132.0.1 + SERVER_ADDRESS=169.254.169.254 + NEXT_SERVER=10.132.0.1 + MTU=1460 + T1=43200 + T2=75600 + LIFETIME=86400 + DNS=169.254.169.254 + NTP=169.254.169.254 + DOMAINNAME=c.ubuntu-foundations.internal + DOMAIN_SEARCH_LIST=c.ubuntu-foundations.internal google.internal + HOSTNAME=tribaal-test-171002-1349.c.ubuntu-foundations.internal + ROUTES=10.132.0.1/32,0.0.0.0 0.0.0.0/0,10.132.0.1 + CLIENTID=ff405663a200020000ab11332859494d7a8b4c + OPTION_245=624c3620 + """) + + def setUp(self): + super(TestExtractIpAddressFromNetworkd, self).setUp() + self.lease_d = self.tmp_dir() + + def test_no_valid_leases_is_none(self): + """No valid leases should return None.""" + self.assertIsNone( + wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_option_245_is_found_in_single(self): + """A single valid lease with 245 option should return it.""" + populate_dir(self.lease_d, {'9': self.azure_lease}) + self.assertEqual( + '624c3620', wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_option_245_not_found_returns_None(self): + """A valid lease, but no option 245 should return None.""" + populate_dir( + self.lease_d, + {'9': self.azure_lease.replace("OPTION_245", "OPTION_999")}) + self.assertIsNone( + wa_shim._networkd_get_value_from_leases(self.lease_d)) + + def test_multiple_returns_first(self): + """Somewhat arbitrarily return the first address when multiple. + + Most important at the moment is that this is consistent behavior + rather than changing randomly as in order of a dictionary.""" + myval = "624c3601" + populate_dir( + self.lease_d, + {'9': self.azure_lease, + '2': self.azure_lease.replace("624c3620", myval)}) + self.assertEqual( + myval, wa_shim._networkd_get_value_from_leases(self.lease_d)) + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 8e98e1bb..96144b64 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -23,13 +23,16 @@ class TestCloudStackPasswordFetching(CiTestCase): default_gw = "192.201.20.0" get_latest_lease = mock.MagicMock(return_value=None) self.patches.enter_context(mock.patch( - 'cloudinit.sources.DataSourceCloudStack.get_latest_lease', - get_latest_lease)) + mod_name + '.get_latest_lease', get_latest_lease)) get_default_gw = mock.MagicMock(return_value=default_gw) self.patches.enter_context(mock.patch( - 'cloudinit.sources.DataSourceCloudStack.get_default_gateway', - get_default_gw)) + mod_name + '.get_default_gateway', get_default_gw)) + + get_networkd_server_address = mock.MagicMock(return_value=None) + self.patches.enter_context(mock.patch( + mod_name + '.dhcp.networkd_get_option_from_leases', + get_networkd_server_address)) def _set_password_server_response(self, response_string): subp = mock.MagicMock(return_value=(response_string, '')) -- cgit v1.2.3 From 0ee829f91322ae1788ee6fb2a164cf06cdfff7db Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 2 Oct 2017 14:32:48 -0700 Subject: tests: Combine integration configs and testcases Combine the configs and testcases directories, so all files are together in one place. Update the test config location as well. --- tests/cloud_tests/__init__.py | 2 +- tests/cloud_tests/configs/bugs/README.md | 13 --- tests/cloud_tests/configs/bugs/lp1511485.yaml | 11 --- tests/cloud_tests/configs/bugs/lp1611074.yaml | 8 -- tests/cloud_tests/configs/bugs/lp1628337.yaml | 23 ----- tests/cloud_tests/configs/examples/README.md | 12 --- tests/cloud_tests/configs/examples/TODO.md | 15 --- .../configs/examples/add_apt_repositories.yaml | 23 ----- .../configs/examples/alter_completion_message.yaml | 16 ---- ...configure_instance_trusted_ca_certificates.yaml | 41 -------- .../examples/configure_instances_ssh_keys.yaml | 63 ------------- .../configs/examples/including_user_groups.yaml | 53 ----------- .../examples/install_arbitrary_packages.yaml | 20 ---- .../configs/examples/install_run_chef_recipes.yaml | 103 -------------------- .../configs/examples/run_apt_upgrade.yaml | 11 --- .../cloud_tests/configs/examples/run_commands.yaml | 16 ---- .../configs/examples/run_commands_first_boot.yaml | 16 ---- .../configs/examples/setup_run_puppet.yaml | 55 ----------- .../examples/writing_out_arbitrary_files.yaml | 45 --------- tests/cloud_tests/configs/main/README.md | 11 --- .../configs/main/command_output_simple.yaml | 13 --- tests/cloud_tests/configs/modules/README.md | 12 --- tests/cloud_tests/configs/modules/TODO.md | 98 ------------------- .../configs/modules/apt_configure_conf.yaml | 21 ----- .../modules/apt_configure_disable_suites.yaml | 20 ---- .../configs/modules/apt_configure_primary.yaml | 26 ----- .../configs/modules/apt_configure_proxy.yaml | 18 ---- .../configs/modules/apt_configure_security.yaml | 18 ---- .../configs/modules/apt_configure_sources_key.yaml | 50 ---------- .../modules/apt_configure_sources_keyserver.yaml | 23 ----- .../modules/apt_configure_sources_list.yaml | 22 ----- .../configs/modules/apt_configure_sources_ppa.yaml | 29 ------ .../configs/modules/apt_pipelining_disable.yaml | 15 --- .../configs/modules/apt_pipelining_os.yaml | 15 --- tests/cloud_tests/configs/modules/bootcmd.yaml | 13 --- tests/cloud_tests/configs/modules/byobu.yaml | 20 ---- tests/cloud_tests/configs/modules/ca_certs.yaml | 52 ---------- .../cloud_tests/configs/modules/debug_disable.yaml | 9 -- .../cloud_tests/configs/modules/debug_enable.yaml | 9 -- .../cloud_tests/configs/modules/final_message.yaml | 13 --- .../configs/modules/keys_to_console.yaml | 15 --- tests/cloud_tests/configs/modules/landscape.yaml | 28 ------ tests/cloud_tests/configs/modules/locale.yaml | 22 ----- tests/cloud_tests/configs/modules/lxd_bridge.yaml | 32 ------- tests/cloud_tests/configs/modules/lxd_dir.yaml | 19 ---- tests/cloud_tests/configs/modules/ntp.yaml | 21 ----- tests/cloud_tests/configs/modules/ntp_pools.yaml | 31 ------ tests/cloud_tests/configs/modules/ntp_servers.yaml | 27 ------ .../modules/package_update_upgrade_install.yaml | 33 ------- tests/cloud_tests/configs/modules/runcmd.yaml | 13 --- tests/cloud_tests/configs/modules/salt_minion.yaml | 34 ------- .../configs/modules/seed_random_command.yaml | 18 ---- .../configs/modules/seed_random_data.yaml | 15 --- .../cloud_tests/configs/modules/set_hostname.yaml | 20 ---- .../configs/modules/set_hostname_fqdn.yaml | 22 ----- .../cloud_tests/configs/modules/set_password.yaml | 19 ---- .../configs/modules/set_password_expire.yaml | 30 ------ .../configs/modules/set_password_list.yaml | 40 -------- .../configs/modules/set_password_list_string.yaml | 40 -------- tests/cloud_tests/configs/modules/snappy.yaml | 15 --- .../modules/ssh_auth_key_fingerprints_disable.yaml | 15 --- .../modules/ssh_auth_key_fingerprints_enable.yaml | 21 ----- .../cloud_tests/configs/modules/ssh_import_id.yaml | 17 ---- .../configs/modules/ssh_keys_generate.yaml | 44 --------- .../configs/modules/ssh_keys_provided.yaml | 105 --------------------- tests/cloud_tests/configs/modules/timezone.yaml | 16 ---- tests/cloud_tests/configs/modules/user_groups.yaml | 52 ---------- tests/cloud_tests/configs/modules/write_files.yaml | 46 --------- tests/cloud_tests/testcases/bugs/README.md | 13 +++ tests/cloud_tests/testcases/bugs/lp1511485.yaml | 11 +++ tests/cloud_tests/testcases/bugs/lp1611074.yaml | 8 ++ tests/cloud_tests/testcases/bugs/lp1628337.yaml | 23 +++++ tests/cloud_tests/testcases/examples/README.md | 12 +++ tests/cloud_tests/testcases/examples/TODO.md | 15 +++ .../testcases/examples/add_apt_repositories.yaml | 23 +++++ .../examples/alter_completion_message.yaml | 16 ++++ ...configure_instance_trusted_ca_certificates.yaml | 41 ++++++++ .../examples/configure_instances_ssh_keys.yaml | 63 +++++++++++++ .../testcases/examples/including_user_groups.yaml | 53 +++++++++++ .../examples/install_arbitrary_packages.yaml | 20 ++++ .../examples/install_run_chef_recipes.yaml | 103 ++++++++++++++++++++ .../testcases/examples/run_apt_upgrade.yaml | 11 +++ .../testcases/examples/run_commands.yaml | 16 ++++ .../examples/run_commands_first_boot.yaml | 16 ++++ .../testcases/examples/setup_run_puppet.yaml | 55 +++++++++++ .../examples/writing_out_arbitrary_files.yaml | 45 +++++++++ tests/cloud_tests/testcases/main/README.md | 11 +++ .../testcases/main/command_output_simple.yaml | 13 +++ tests/cloud_tests/testcases/modules/README.md | 12 +++ tests/cloud_tests/testcases/modules/TODO.md | 98 +++++++++++++++++++ .../testcases/modules/apt_configure_conf.yaml | 21 +++++ .../modules/apt_configure_disable_suites.yaml | 20 ++++ .../testcases/modules/apt_configure_primary.yaml | 26 +++++ .../testcases/modules/apt_configure_proxy.yaml | 18 ++++ .../testcases/modules/apt_configure_security.yaml | 18 ++++ .../modules/apt_configure_sources_key.yaml | 50 ++++++++++ .../modules/apt_configure_sources_keyserver.yaml | 23 +++++ .../modules/apt_configure_sources_list.yaml | 22 +++++ .../modules/apt_configure_sources_ppa.yaml | 29 ++++++ .../testcases/modules/apt_pipelining_disable.yaml | 15 +++ .../testcases/modules/apt_pipelining_os.yaml | 15 +++ tests/cloud_tests/testcases/modules/bootcmd.yaml | 13 +++ tests/cloud_tests/testcases/modules/byobu.yaml | 20 ++++ tests/cloud_tests/testcases/modules/ca_certs.yaml | 52 ++++++++++ .../testcases/modules/debug_disable.yaml | 9 ++ .../testcases/modules/debug_enable.yaml | 9 ++ .../testcases/modules/final_message.yaml | 13 +++ .../testcases/modules/keys_to_console.yaml | 15 +++ tests/cloud_tests/testcases/modules/landscape.yaml | 28 ++++++ tests/cloud_tests/testcases/modules/locale.yaml | 22 +++++ .../cloud_tests/testcases/modules/lxd_bridge.yaml | 32 +++++++ tests/cloud_tests/testcases/modules/lxd_dir.yaml | 19 ++++ tests/cloud_tests/testcases/modules/ntp.yaml | 21 +++++ tests/cloud_tests/testcases/modules/ntp_pools.yaml | 31 ++++++ .../cloud_tests/testcases/modules/ntp_servers.yaml | 27 ++++++ .../modules/package_update_upgrade_install.yaml | 33 +++++++ tests/cloud_tests/testcases/modules/runcmd.yaml | 13 +++ .../cloud_tests/testcases/modules/salt_minion.yaml | 34 +++++++ .../testcases/modules/seed_random_command.yaml | 18 ++++ .../testcases/modules/seed_random_data.yaml | 15 +++ .../testcases/modules/set_hostname.yaml | 20 ++++ .../testcases/modules/set_hostname_fqdn.yaml | 22 +++++ .../testcases/modules/set_password.yaml | 19 ++++ .../testcases/modules/set_password_expire.yaml | 30 ++++++ .../testcases/modules/set_password_list.yaml | 40 ++++++++ .../modules/set_password_list_string.yaml | 40 ++++++++ tests/cloud_tests/testcases/modules/snappy.yaml | 15 +++ .../modules/ssh_auth_key_fingerprints_disable.yaml | 15 +++ .../modules/ssh_auth_key_fingerprints_enable.yaml | 21 +++++ .../testcases/modules/ssh_import_id.yaml | 17 ++++ .../testcases/modules/ssh_keys_generate.yaml | 44 +++++++++ .../testcases/modules/ssh_keys_provided.yaml | 105 +++++++++++++++++++++ tests/cloud_tests/testcases/modules/timezone.yaml | 16 ++++ .../cloud_tests/testcases/modules/user_groups.yaml | 52 ++++++++++ .../cloud_tests/testcases/modules/write_files.yaml | 46 +++++++++ 135 files changed, 1862 insertions(+), 1862 deletions(-) delete mode 100644 tests/cloud_tests/configs/bugs/README.md delete mode 100644 tests/cloud_tests/configs/bugs/lp1511485.yaml delete mode 100644 tests/cloud_tests/configs/bugs/lp1611074.yaml delete mode 100644 tests/cloud_tests/configs/bugs/lp1628337.yaml delete mode 100644 tests/cloud_tests/configs/examples/README.md delete mode 100644 tests/cloud_tests/configs/examples/TODO.md delete mode 100644 tests/cloud_tests/configs/examples/add_apt_repositories.yaml delete mode 100644 tests/cloud_tests/configs/examples/alter_completion_message.yaml delete mode 100644 tests/cloud_tests/configs/examples/configure_instance_trusted_ca_certificates.yaml delete mode 100644 tests/cloud_tests/configs/examples/configure_instances_ssh_keys.yaml delete mode 100644 tests/cloud_tests/configs/examples/including_user_groups.yaml delete mode 100644 tests/cloud_tests/configs/examples/install_arbitrary_packages.yaml delete mode 100644 tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml delete mode 100644 tests/cloud_tests/configs/examples/run_apt_upgrade.yaml delete mode 100644 tests/cloud_tests/configs/examples/run_commands.yaml delete mode 100644 tests/cloud_tests/configs/examples/run_commands_first_boot.yaml delete mode 100644 tests/cloud_tests/configs/examples/setup_run_puppet.yaml delete mode 100644 tests/cloud_tests/configs/examples/writing_out_arbitrary_files.yaml delete mode 100644 tests/cloud_tests/configs/main/README.md delete mode 100644 tests/cloud_tests/configs/main/command_output_simple.yaml delete mode 100644 tests/cloud_tests/configs/modules/README.md delete mode 100644 tests/cloud_tests/configs/modules/TODO.md delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_conf.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_primary.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_proxy.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_security.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml delete mode 100644 tests/cloud_tests/configs/modules/apt_pipelining_os.yaml delete mode 100644 tests/cloud_tests/configs/modules/bootcmd.yaml delete mode 100644 tests/cloud_tests/configs/modules/byobu.yaml delete mode 100644 tests/cloud_tests/configs/modules/ca_certs.yaml delete mode 100644 tests/cloud_tests/configs/modules/debug_disable.yaml delete mode 100644 tests/cloud_tests/configs/modules/debug_enable.yaml delete mode 100644 tests/cloud_tests/configs/modules/final_message.yaml delete mode 100644 tests/cloud_tests/configs/modules/keys_to_console.yaml delete mode 100644 tests/cloud_tests/configs/modules/landscape.yaml delete mode 100644 tests/cloud_tests/configs/modules/locale.yaml delete mode 100644 tests/cloud_tests/configs/modules/lxd_bridge.yaml delete mode 100644 tests/cloud_tests/configs/modules/lxd_dir.yaml delete mode 100644 tests/cloud_tests/configs/modules/ntp.yaml delete mode 100644 tests/cloud_tests/configs/modules/ntp_pools.yaml delete mode 100644 tests/cloud_tests/configs/modules/ntp_servers.yaml delete mode 100644 tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml delete mode 100644 tests/cloud_tests/configs/modules/runcmd.yaml delete mode 100644 tests/cloud_tests/configs/modules/salt_minion.yaml delete mode 100644 tests/cloud_tests/configs/modules/seed_random_command.yaml delete mode 100644 tests/cloud_tests/configs/modules/seed_random_data.yaml delete mode 100644 tests/cloud_tests/configs/modules/set_hostname.yaml delete mode 100644 tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml delete mode 100644 tests/cloud_tests/configs/modules/set_password.yaml delete mode 100644 tests/cloud_tests/configs/modules/set_password_expire.yaml delete mode 100644 tests/cloud_tests/configs/modules/set_password_list.yaml delete mode 100644 tests/cloud_tests/configs/modules/set_password_list_string.yaml delete mode 100644 tests/cloud_tests/configs/modules/snappy.yaml delete mode 100644 tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml delete mode 100644 tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml delete mode 100644 tests/cloud_tests/configs/modules/ssh_import_id.yaml delete mode 100644 tests/cloud_tests/configs/modules/ssh_keys_generate.yaml delete mode 100644 tests/cloud_tests/configs/modules/ssh_keys_provided.yaml delete mode 100644 tests/cloud_tests/configs/modules/timezone.yaml delete mode 100644 tests/cloud_tests/configs/modules/user_groups.yaml delete mode 100644 tests/cloud_tests/configs/modules/write_files.yaml create mode 100644 tests/cloud_tests/testcases/bugs/README.md create mode 100644 tests/cloud_tests/testcases/bugs/lp1511485.yaml create mode 100644 tests/cloud_tests/testcases/bugs/lp1611074.yaml create mode 100644 tests/cloud_tests/testcases/bugs/lp1628337.yaml create mode 100644 tests/cloud_tests/testcases/examples/README.md create mode 100644 tests/cloud_tests/testcases/examples/TODO.md create mode 100644 tests/cloud_tests/testcases/examples/add_apt_repositories.yaml create mode 100644 tests/cloud_tests/testcases/examples/alter_completion_message.yaml create mode 100644 tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.yaml create mode 100644 tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.yaml create mode 100644 tests/cloud_tests/testcases/examples/including_user_groups.yaml create mode 100644 tests/cloud_tests/testcases/examples/install_arbitrary_packages.yaml create mode 100644 tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml create mode 100644 tests/cloud_tests/testcases/examples/run_apt_upgrade.yaml create mode 100644 tests/cloud_tests/testcases/examples/run_commands.yaml create mode 100644 tests/cloud_tests/testcases/examples/run_commands_first_boot.yaml create mode 100644 tests/cloud_tests/testcases/examples/setup_run_puppet.yaml create mode 100644 tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.yaml create mode 100644 tests/cloud_tests/testcases/main/README.md create mode 100644 tests/cloud_tests/testcases/main/command_output_simple.yaml create mode 100644 tests/cloud_tests/testcases/modules/README.md create mode 100644 tests/cloud_tests/testcases/modules/TODO.md create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_conf.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_disable_suites.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_primary.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_proxy.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_security.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_sources_key.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml create mode 100644 tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml create mode 100644 tests/cloud_tests/testcases/modules/bootcmd.yaml create mode 100644 tests/cloud_tests/testcases/modules/byobu.yaml create mode 100644 tests/cloud_tests/testcases/modules/ca_certs.yaml create mode 100644 tests/cloud_tests/testcases/modules/debug_disable.yaml create mode 100644 tests/cloud_tests/testcases/modules/debug_enable.yaml create mode 100644 tests/cloud_tests/testcases/modules/final_message.yaml create mode 100644 tests/cloud_tests/testcases/modules/keys_to_console.yaml create mode 100644 tests/cloud_tests/testcases/modules/landscape.yaml create mode 100644 tests/cloud_tests/testcases/modules/locale.yaml create mode 100644 tests/cloud_tests/testcases/modules/lxd_bridge.yaml create mode 100644 tests/cloud_tests/testcases/modules/lxd_dir.yaml create mode 100644 tests/cloud_tests/testcases/modules/ntp.yaml create mode 100644 tests/cloud_tests/testcases/modules/ntp_pools.yaml create mode 100644 tests/cloud_tests/testcases/modules/ntp_servers.yaml create mode 100644 tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml create mode 100644 tests/cloud_tests/testcases/modules/runcmd.yaml create mode 100644 tests/cloud_tests/testcases/modules/salt_minion.yaml create mode 100644 tests/cloud_tests/testcases/modules/seed_random_command.yaml create mode 100644 tests/cloud_tests/testcases/modules/seed_random_data.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_hostname.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_password.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_password_expire.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_password_list.yaml create mode 100644 tests/cloud_tests/testcases/modules/set_password_list_string.yaml create mode 100644 tests/cloud_tests/testcases/modules/snappy.yaml create mode 100644 tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml create mode 100644 tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.yaml create mode 100644 tests/cloud_tests/testcases/modules/ssh_import_id.yaml create mode 100644 tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml create mode 100644 tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml create mode 100644 tests/cloud_tests/testcases/modules/timezone.yaml create mode 100644 tests/cloud_tests/testcases/modules/user_groups.yaml create mode 100644 tests/cloud_tests/testcases/modules/write_files.yaml diff --git a/tests/cloud_tests/__init__.py b/tests/cloud_tests/__init__.py index 07148c12..98c1d6c7 100644 --- a/tests/cloud_tests/__init__.py +++ b/tests/cloud_tests/__init__.py @@ -7,7 +7,7 @@ import os BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TESTCASES_DIR = os.path.join(BASE_DIR, 'testcases') -TEST_CONF_DIR = os.path.join(BASE_DIR, 'configs') +TEST_CONF_DIR = os.path.join(BASE_DIR, 'testcases') TREE_BASE = os.sep.join(BASE_DIR.split(os.sep)[:-2]) diff --git a/tests/cloud_tests/configs/bugs/README.md b/tests/cloud_tests/configs/bugs/README.md deleted file mode 100644 index 09ce0765..00000000 --- a/tests/cloud_tests/configs/bugs/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Bug Test Configs - -## purpose -Configs that reproduce bugs filed against cloud-init. Having test configs for -cloud-init bugs ensures that the fixes do not break in the future, and makes it -easy to see how many systems and platforms are effected by a new bug. - -## structure -Should have one test config for most bugs filed. The name of the test should -contain ``lp`` followed by the bug number. It may also be useful to add a -comment to each bug config with a summary copied from the bug report. - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/bugs/lp1511485.yaml b/tests/cloud_tests/configs/bugs/lp1511485.yaml deleted file mode 100644 index ebf9763f..00000000 --- a/tests/cloud_tests/configs/bugs/lp1511485.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# -# LP Bug 1511485: final_message is silent on ubuntu-12.04.5 / cloud-init 0.6.3 -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - final_message: "Final message from cloud-config" - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/bugs/lp1611074.yaml b/tests/cloud_tests/configs/bugs/lp1611074.yaml deleted file mode 100644 index 960679d5..00000000 --- a/tests/cloud_tests/configs/bugs/lp1611074.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# -# LP Bug 1611074: Reformatting of ephemeral drive fails on resize of Azure VM -# -# 2016-11-18: Disabled until test written -# -enabled: False - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/bugs/lp1628337.yaml b/tests/cloud_tests/configs/bugs/lp1628337.yaml deleted file mode 100644 index e39b3cd8..00000000 --- a/tests/cloud_tests/configs/bugs/lp1628337.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# -# LP Bug 1628337: cloud-init tries to install NTP before even configuring the archives -# -required_features: - - apt - - lsb_release -cloud_config: | - #cloud-config - ntp: - servers: ['ntp.ubuntu.com'] - apt: - primary: - - arches: [default] - uri: http://us.archive.ubuntu.com/ubuntu/ -collect_sciprts: - ntp.conf: | - #!/bin/bash - cat /etc/ntp.conf - sources.list: | - #!/bin/bash - cat /etc/apt/sources.list - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/README.md b/tests/cloud_tests/configs/examples/README.md deleted file mode 100644 index 110a223b..00000000 --- a/tests/cloud_tests/configs/examples/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Example Test Configs - -## Purpose -This folder contains example cloud configs found on -[cloudinit.readthedocs.io](https://cloudinit.readthedocs.io/en/latest/topics/examples.html). -Examples covered by other tests, like modules, are excluded from tests here -to prevent duplication and reduce test time. - -## Structure -One test per example test config on cloudinit.readthedocs.io - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/TODO.md b/tests/cloud_tests/configs/examples/TODO.md deleted file mode 100644 index 8db0e98e..00000000 --- a/tests/cloud_tests/configs/examples/TODO.md +++ /dev/null @@ -1,15 +0,0 @@ -# Missing Examples - -Below lists each of the issing examples and why it is not currently added. - - - Chef (takes > 60 seconds to run) - - Puppet (takes > 60 seconds to run) - - Manage resolve.conf (lxd backend overrides changes) - - Adding a yum repository (need centos system) - - Register RedHat Subscription (need centos system + subscription) - - Adjust mount points mounted (need multiple disks) - - Call a url when finished (need end point) - - Reboot/poweroff when finished (how to test) - - Disk setup (need multiple disks) - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/add_apt_repositories.yaml b/tests/cloud_tests/configs/examples/add_apt_repositories.yaml deleted file mode 100644 index 4b8575f7..00000000 --- a/tests/cloud_tests/configs/examples/add_apt_repositories.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -required_features: - - apt -cloud_config: | - #cloud-config - apt: - primary: - - arches: [default] - uri: "http://www.gtlib.gatech.edu/pub/ubuntu-releases/" -collect_scripts: - ubuntu.sources.list: | - #!/bin/bash - cat /etc/apt/sources.list | grep -v '^#' | sed '/^\s*$/d' | grep archive.ubuntu.com | wc -l - gatech.sources.list: | - #!/bin/bash - cat /etc/apt/sources.list | grep -v '^#' | sed '/^\s*$/d' | grep gtlib.gatech.edu | wc -l - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/alter_completion_message.yaml b/tests/cloud_tests/configs/examples/alter_completion_message.yaml deleted file mode 100644 index 9e154f80..00000000 --- a/tests/cloud_tests/configs/examples/alter_completion_message.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - final_message: | - This is my final message! - $version - $timestamp - $datasource - $uptime - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/configure_instance_trusted_ca_certificates.yaml b/tests/cloud_tests/configs/examples/configure_instance_trusted_ca_certificates.yaml deleted file mode 100644 index ad32b088..00000000 --- a/tests/cloud_tests/configs/examples/configure_instance_trusted_ca_certificates.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - ca-certs: - # If present and set to True, the 'remove-defaults' parameter will remove - # all the default trusted CA certificates that are normally shipped with - # Ubuntu. - # This is mainly for paranoid admins - most users will not need this - # functionality. - remove-defaults: true - - # If present, the 'trusted' parameter should contain a certificate (or list - # of certificates) to add to the system as trusted CA certificates. - # Pay close attention to the YAML multiline list syntax. The example shown - # here is for a list of multiline certificates. - trusted: - - | - -----BEGIN CERTIFICATE----- - YOUR-ORGS-TRUSTED-CA-CERT-HERE - -----END CERTIFICATE----- - - | - -----BEGIN CERTIFICATE----- - YOUR-ORGS-TRUSTED-CA-CERT-HERE - -----END CERTIFICATE----- -collect_scripts: - cloudinit_certs: | - #!/bin/bash - cat /etc/ssl/certs/cloud-init-ca-certs.pem - cert_count_ca: | - #!/bin/bash - wc -l /etc/ssl/certs/ca-certificates.crt - cert_count_cloudinit: | - #!/bin/bash - wc -l /etc/ssl/certs/cloud-init-ca-certs.pem - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/configure_instances_ssh_keys.yaml b/tests/cloud_tests/configs/examples/configure_instances_ssh_keys.yaml deleted file mode 100644 index f3eaf3ce..00000000 --- a/tests/cloud_tests/configs/examples/configure_instances_ssh_keys.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUUk8EEAnnkhXlukKoUPND/RRClWz2s5TCzIkd3Ou5+Cyz71X0XmazM3l5WgeErvtIwQMyT1KjNoMhoJMrJnWqQPOt5Q8zWd9qG7PBl9+eiH5qV7NZ mykey@host - - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw== smoser@brickies - - # Send pre-generated ssh private keys to the server - # If these are present, they will be written to /etc/ssh and - # new random keys will not be generated - # in addition to 'rsa' and 'dsa' as shown below, 'ecdsa' is also supported - ssh_keys: - rsa_private: | - -----BEGIN RSA PRIVATE KEY----- - MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qcon2LZS/x - 1cydPZ4pQpfjEha6WxZ6o8ci/Ea/w0n+0HGPwaxlEG2Z9inNtj3pgFrYcRztfECb - 1j6HCibZbAzYtwIBIwJgO8h72WjcmvcpZ8OvHSvTwAguO2TkR6mPgHsgSaKy6GJo - PUJnaZRWuba/HX0KGyhz19nPzLpzG5f0fYahlMJAyc13FV7K6kMBPXTRR6FxgHEg - L0MPC7cdqAwOVNcPY6A7AjEA1bNaIjOzFN2sfZX0j7OMhQuc4zP7r80zaGc5oy6W - p58hRAncFKEvnEq2CeL3vtuZAjEAwNBHpbNsBYTRPCHM7rZuG/iBtwp8Rxhc9I5w - ixvzMgi+HpGLWzUIBS+P/XhekIjPAjA285rVmEP+DR255Ls65QbgYhJmTzIXQ2T9 - luLvcmFBC6l35Uc4gTgg4ALsmXLn71MCMGMpSWspEvuGInayTCL+vEjmNBT+FAdO - W7D4zCpI43jRS9U06JVOeSc9CDk2lwiA3wIwCTB/6uc8Cq85D9YqpM10FuHjKpnP - REPPOyrAspdeOAV+6VKRavstea7+2DZmSUgE - -----END RSA PRIVATE KEY----- - - rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7XdewmZ3h8eIXJD7TRHtVW7aJX1ByifYtlL/HVzJ09nilCl+MSFrpbFnqjxyL8Rr/DSf7QcY/BrGUQbZn2Kc22PemAWthxHO18QJvWPocKJtlsDNi3 smoser@localhost - - dsa_private: | - -----BEGIN DSA PRIVATE KEY----- - MIIBuwIBAAKBgQDP2HLu7pTExL89USyM0264RCyWX/CMLmukxX0Jdbm29ax8FBJT - pLrO8TIXVY5rPAJm1dTHnpuyJhOvU9G7M8tPUABtzSJh4GVSHlwaCfycwcpLv9TX - DgWIpSj+6EiHCyaRlB1/CBp9RiaB+10QcFbm+lapuET+/Au6vSDp9IRtlQIVAIMR - 8KucvUYbOEI+yv+5LW9u3z/BAoGBAI0q6JP+JvJmwZFaeCMMVxXUbqiSko/P1lsa - LNNBHZ5/8MOUIm8rB2FC6ziidfueJpqTMqeQmSAlEBCwnwreUnGfRrKoJpyPNENY - d15MG6N5J+z81sEcHFeprryZ+D3Ge9VjPq3Tf3NhKKwCDQ0240aPezbnjPeFm4mH - bYxxcZ9GAoGAXmLIFSQgiAPu459rCKxT46tHJtM0QfnNiEnQLbFluefZ/yiI4DI3 - 8UzTCOXLhUA7ybmZha+D/csj15Y9/BNFuO7unzVhikCQV9DTeXX46pG4s1o23JKC - /QaYWNMZ7kTRv+wWow9MhGiVdML4ZN4XnifuO5krqAybngIy66PMEoQCFEIsKKWv - 99iziAH0KBMVbxy03Trz - -----END DSA PRIVATE KEY----- - - dsa_public: ssh-dsa AAAAB3NzaC1kc3MAAACBAM/Ycu7ulMTEvz1RLIzTbrhELJZf8Iwua6TFfQl1ubb1rHwUElOkus7xMhdVjms8AmbV1Meem7ImE69T0bszy09QAG3NImHgZVIeXBoJ/JzByku/1NcOBYilKP7oSIcLJpGUHX8IGn1GJoH7XRBwVub6Vqm4RP78C7q9IOn0hG2VAAAAFQCDEfCrnL1GGzhCPsr/uS1vbt8/wQAAAIEAjSrok/4m8mbBkVp4IwxXFdRuqJKSj8/WWxos00Ednn/ww5QibysHYULrOKJ1+54mmpMyp5CZICUQELCfCt5ScZ9GsqgmnI80Q1h3Xkwbo3kn7PzWwRwcV6muvJn4PcZ71WM+rdN/c2EorAINDTbjRo97NueM94WbiYdtjHFxn0YAAACAXmLIFSQgiAPu459rCKxT46tHJtM0QfnNiEnQLbFluefZ/yiI4DI38UzTCOXLhUA7ybmZha+D/csj15Y9/BNFuO7unzVhikCQV9DTeXX46pG4s1o23JKC/QaYWNMZ7kTRv+wWow9MhGiVdML4ZN4XnifuO5krqAybngIy66PMEoQ= smoser@localhost -collect_scripts: - cert_count: | - #!/bin/bash - ls | wc -l - dsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_dsa_key.pub - rsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_rsa_key.pub - auth_keys: | - #!/bin/bash - cat /home/ubuntu/.ssh/authorized_keys - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/including_user_groups.yaml b/tests/cloud_tests/configs/examples/including_user_groups.yaml deleted file mode 100644 index 0aa7ad21..00000000 --- a/tests/cloud_tests/configs/examples/including_user_groups.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - # Add groups to the system - groups: - - secret: [foobar,barfoo] - - cloud-users - - # Add users to the system. Users are added after groups are added. - users: - - default - - name: foobar - gecos: Foo B. Bar - primary-group: foobar - groups: users - expiredate: 2038-01-19 - lock_passwd: false - passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - - name: barfoo - gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL - groups: cloud-users - lock_passwd: true - - name: cloudy - gecos: Magic Cloud App Daemon User - inactive: true - system: true -collect_scripts: - group_ubuntu: | - #!/bin/bash - getent group ubuntu - group_cloud_users: | - #!/bin/bash - getent group cloud-users - user_ubuntu: | - #!/bin/bash - getent passwd ubuntu - user_foobar: | - #!/bin/bash - getent passwd foobar - user_barfoo: | - #!/bin/bash - getent passwd barfoo - user_cloudy: | - #!/bin/bash - getent passwd cloudy - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/install_arbitrary_packages.yaml b/tests/cloud_tests/configs/examples/install_arbitrary_packages.yaml deleted file mode 100644 index d3980228..00000000 --- a/tests/cloud_tests/configs/examples/install_arbitrary_packages.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - packages: - - htop - - tree -collect_scripts: - htop: | - #!/bin/bash - dpkg -l | grep htop | wc -l - tree: | - #!/bin/bash - dpkg -l | grep tree | wc -l - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml deleted file mode 100644 index 0bec305e..00000000 --- a/tests/cloud_tests/configs/examples/install_run_chef_recipes.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2017-03-31: Disabled as depends on third party apt repository -# -enabled: False -cloud_config: | - #cloud-config - # Key from https://packages.chef.io/chef.asc - apt: - source1: - source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.12 (Darwin) - Comment: GPGTools - http://gpgtools.org - - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg - PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK - CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid - AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd - Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz - SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK - OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ - Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY - IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu - twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 - DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE - WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS - 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA - dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC - MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD - 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K - zA== - =IxPr - -----END PGP PUBLIC KEY BLOCK----- - - chef: - - # Valid values are 'gems' and 'packages' and 'omnibus' - install_type: "packages" - - # Boolean: run 'install_type' code even if chef-client - # appears already installed. - force_install: false - - # Chef settings - server_url: "https://chef.yourorg.com:4000" - - # Node Name - # Defaults to the instance-id if not present - node_name: "your-node-name" - - # Environment - # Defaults to '_default' if not present - environment: "production" - - # Default validation name is chef-validator - validation_name: "yourorg-validator" - # if validation_cert's value is "system" then it is expected - # that the file already exists on the system. - validation_cert: | - -----BEGIN RSA PRIVATE KEY----- - YOUR-ORGS-VALIDATION-KEY-HERE - -----END RSA PRIVATE KEY----- - - # A run list for a first boot json - run_list: - - "recipe[apache2]" - - "role[db]" - - # Specify a list of initial attributes used by the cookbooks - initial_attributes: - apache: - prefork: - maxclients: 100 - keepalive: "off" - - # if install_type is 'omnibus', change the url to download - omnibus_url: "https://www.opscode.com/chef/install.sh" - - - # Capture all subprocess output into a logfile - # Useful for troubleshooting cloud-init issues - output: {all: '| tee -a /var/log/cloud-init-output.log'} - -collect_scripts: - chef_installed: | - #!/bin/sh - dpkg-query -W -f '${Status}\n' chef - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/run_apt_upgrade.yaml b/tests/cloud_tests/configs/examples/run_apt_upgrade.yaml deleted file mode 100644 index 2b7eae4c..00000000 --- a/tests/cloud_tests/configs/examples/run_apt_upgrade.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - package_upgrade: true - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/run_commands.yaml b/tests/cloud_tests/configs/examples/run_commands.yaml deleted file mode 100644 index b0e311ba..00000000 --- a/tests/cloud_tests/configs/examples/run_commands.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - runcmd: - - echo cloud-init run cmd test > /tmp/run_cmd -collect_scripts: - run_cmd: | - #!/bin/bash - cat /tmp/run_cmd - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/run_commands_first_boot.yaml b/tests/cloud_tests/configs/examples/run_commands_first_boot.yaml deleted file mode 100644 index 7bd803db..00000000 --- a/tests/cloud_tests/configs/examples/run_commands_first_boot.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - bootcmd: - - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts -collect_scripts: - hosts: | - #!/bin/bash - cat /etc/hosts - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/setup_run_puppet.yaml b/tests/cloud_tests/configs/examples/setup_run_puppet.yaml deleted file mode 100644 index e366c042..00000000 --- a/tests/cloud_tests/configs/examples/setup_run_puppet.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as test suite fails this long running test currently -# -enabled: False -cloud_config: | - #cloud-config - puppet: - # Every key present in the conf object will be added to puppet.conf: - # [name] - # subkey=value - # - # For example the configuration below will have the following section - # added to puppet.conf: - # [puppetd] - # server=puppetmaster.example.org - # certname=i-0123456.ip-X-Y-Z.cloud.internal - # - # The puppmaster ca certificate will be available in - # /var/lib/puppet/ssl/certs/ca.pem - conf: - agent: - server: "puppetmaster.example.org" - # certname supports substitutions at runtime: - # %i: instanceid - # Example: i-0123456 - # %f: fqdn of the machine - # Example: ip-X-Y-Z.cloud.internal - # - # NB: the certname will automatically be lowercased as required by puppet - certname: "%i.%f" - # ca_cert is a special case. It won't be added to puppet.conf. - # It holds the puppetmaster certificate in pem format. - # It should be a multi-line string (using the | yaml notation for - # multi-line strings). - # The puppetmaster certificate is located in - # /var/lib/puppet/ssl/ca/ca_crt.pem on the puppetmaster host. - # - ca_cert: | - -----BEGIN CERTIFICATE----- - MIICCTCCAXKgAwIBAgIBATANBgkqhkiG9w0BAQUFADANMQswCQYDVQQDDAJjYTAe - Fw0xMDAyMTUxNzI5MjFaFw0xNTAyMTQxNzI5MjFaMA0xCzAJBgNVBAMMAmNhMIGf - MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCu7Q40sm47/E1Pf+r8AYb/V/FWGPgc - b014OmNoX7dgCxTDvps/h8Vw555PdAFsW5+QhsGr31IJNI3kSYprFQcYf7A8tNWu - 1MASW2CfaEiOEi9F1R3R4Qlz4ix+iNoHiUDTjazw/tZwEdxaQXQVLwgTGRwVa+aA - qbutJKi93MILLwIDAQABo3kwdzA4BglghkgBhvhCAQ0EKxYpUHVwcGV0IFJ1Ynkv - T3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwDwYDVR0TAQH/BAUwAwEB/zAd - BgNVHQ4EFgQUu4+jHB+GYE5Vxo+ol1OAhevspjAwCwYDVR0PBAQDAgEGMA0GCSqG - SIb3DQEBBQUAA4GBAH/rxlUIjwNb3n7TXJcDJ6MMHUlwjr03BDJXKb34Ulndkpaf - +GAlzPXWa7bO908M9I8RnPfvtKnteLbvgTK+h+zX1XCty+S2EQWk29i2AdoqOTxb - hppiGMp0tT5Havu4aceCXiy2crVcudj3NFciy8X66SoECemW9UYDCb9T5D0d - -----END CERTIFICATE----- - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/examples/writing_out_arbitrary_files.yaml b/tests/cloud_tests/configs/examples/writing_out_arbitrary_files.yaml deleted file mode 100644 index 6f78f994..00000000 --- a/tests/cloud_tests/configs/examples/writing_out_arbitrary_files.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# -# From cloud config examples on cloudinit.readthedocs.io -# -# 2016-11-17: Disabled as covered by module based tests -# -enabled: False -cloud_config: | - #cloud-config - write_files: - - encoding: b64 - content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4 - owner: root:root - path: /root/file_b64 - permissions: '0644' - - content: | - # My new /root/file_text - - SMBDOPTIONS="-D" - path: /root/file_text - - content: !!binary | - f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI - AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA - AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA - path: /root/file_binary - permissions: '0555' - - encoding: gzip - content: !!binary | - H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= - path: /root/file_gzip - permissions: '0755' -collect_scripts: - file_b64: | - #!/bin/bash - file /root/file_b64 - file_text: | - #!/bin/bash - file /root/file_text - file_binary: | - #!/bin/bash - file /root/file_binary - file_gzip: | - #!/bin/bash - file /root/file_gzip - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/main/README.md b/tests/cloud_tests/configs/main/README.md deleted file mode 100644 index 60346063..00000000 --- a/tests/cloud_tests/configs/main/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Main Functionality Test Configs - -## purpose -Test main features and config options of cloud-init such as logging, output -redirection, early init and integration with init system - -## structure -Should have one or more test configs for all main cloud-init output and logging -options, and basic functionality test cases - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/main/command_output_simple.yaml b/tests/cloud_tests/configs/main/command_output_simple.yaml deleted file mode 100644 index 08ca8940..00000000 --- a/tests/cloud_tests/configs/main/command_output_simple.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Test functionality of simple output redirection -# -cloud_config: | - #cloud-config - output: { all: "| tee -a /var/log/cloud-init-test-output" } - final_message: "should be last line in cloud-init-test-output file" -collect_scripts: - cloud-init-test-output: | - #!/bin/bash - cat /var/log/cloud-init-test-output - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/README.md b/tests/cloud_tests/configs/modules/README.md deleted file mode 100644 index d66101f2..00000000 --- a/tests/cloud_tests/configs/modules/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Module Test Configs - -## Purpose -Test functionality of cloud config modules. See -[here](https://cloudinit.readthedocs.io/en/latest/topics/modules.html) for -a full list. - -## Structure -Should have one or more test configs for each module in cloudinit/config/. The -name of the test should indicate which module the config is verifying. - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/TODO.md b/tests/cloud_tests/configs/modules/TODO.md deleted file mode 100644 index 0b933b3b..00000000 --- a/tests/cloud_tests/configs/modules/TODO.md +++ /dev/null @@ -1,98 +0,0 @@ -# TODO - -The following lists complete or partially misisng modules. If a module is -listed with nothing below it indicates that no work is completed on that -module. If there is a list below the module name that is the remainig -identified work. - -## apt_configure - - * apt_get_wrapper - * What does this do? How to use it? - * apt_get_command - * To specify a different 'apt-get' command, set 'apt_get_command'. - This must be a list, and the subcommand (update, upgrade) is appended to it. - * Modify default and verify the options got passed correctly. - * preserve sources - * TBD - -## chef -2016-11-17: Tests took > 60 seconds and test framework times out currently. - -## disable EC2 metadata - -## disk setup - -## emit upstart - -## fan - -## growpart - -## grub dpkg - -## landscape -2016-11-17: Module is not working - -## lxd -2016-11-17: Need a zfs backed test written - -## mcollective - -## migrator - -## mounts - -## phone home - -## power state change - -## puppet -2016-11-17: Tests took > 60 seconds and test framework times out currently. - -## resizefs - -## resolv conf -2016-11-17: Issues with changing resolv.conf and lxc backend. - -## redhat subscription -2016-11-17: Need RH support in test framework. - -## rightscale userdata -2016-11-17: Specific to RightScale cloud enviornment. - -## rsyslog - -## scripts per boot -Not applicable to write a test for this as it specifies when something should be run. - -## scripts per instance -Not applicable to write a test for this as it specifies when something should be run. - -## scripts per once -Not applicable to write a test for this as it specifies when something should be run. - -## scripts user -Not applicable to write a test for this as it specifies when something should be run. - -## scripts vendor -Not applicable to write a test for this as it specifies when something should be run. - -## snappy -2016-11-17: Need test to install snaps from store - -## snap-config -2016-11-17: Need to investigate - -## spacewalk - -## ssh authkey fingerprints -The authkey_hash key does not appear to work. In fact the default claims to be md5, however syslog only shows sha256 - -## update etc hosts -2016-11-17: Issues with changing /etc/hosts and lxc backend. - -## yum add repo -2016-11-17: Need RH support in test framework. - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_conf.yaml b/tests/cloud_tests/configs/modules/apt_configure_conf.yaml deleted file mode 100644 index de453000..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_conf.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Provide a configuration for APT -# -required_features: - - apt -cloud_config: | - #cloud-config - apt: - conf: | - APT { - Get { - Assume-Yes "true"; - Fix-Broken "true"; - } - } -collect_scripts: - 94cloud-init-config: | - #!/bin/bash - cat /etc/apt/apt.conf.d/94cloud-init-config - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml b/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml deleted file mode 100644 index 98800673..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_disable_suites.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Disables everything in sources.list -# -required_features: - - apt - - lsb_release -cloud_config: | - #cloud-config - apt: - disable_suites: - - $RELEASE - - $RELEASE-updates - - $RELEASE-backports - - $RELEASE-security -collect_scripts: - sources.list: | - #!/bin/bash - grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_primary.yaml b/tests/cloud_tests/configs/modules/apt_configure_primary.yaml deleted file mode 100644 index 41bcf2fd..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_primary.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# -# Setup a custome primary sources.list -# -required_features: - - apt - - apt_src_cont -cloud_config: | - #cloud-config - apt: - primary: - - arches: - - default - uri: "http://www.gtlib.gatech.edu/pub/ubuntu-releases/" -collect_scripts: - ubuntu.sources.list: | - #!/bin/bash - grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' | grep -c archive.ubuntu.com - gatech.sources.list: | - #!/bin/bash - grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' | grep -c gtlib.gatech.edu - - sources.list: | - #!/bin/bash - cat /etc/apt/sources.list - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml b/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml deleted file mode 100644 index be6c6f81..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_proxy.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Set apt proxy -# -required_features: - - apt -cloud_config: | - #cloud-config - apt: - proxy: "http://squid.internal:3128" - http_proxy: "http://squid.internal:3128" - ftp_proxy: "ftp://squid.internal:3128" - https_proxy: "https://squid.internal:3128" -collect_scripts: - 90cloud-init-aptproxy: | - #!/bin/bash - cat /etc/apt/apt.conf.d/90cloud-init-aptproxy - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_security.yaml b/tests/cloud_tests/configs/modules/apt_configure_security.yaml deleted file mode 100644 index 83dd51df..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_security.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Add security to sources.list -# -required_features: - - apt - - ubuntu_repos -cloud_config: | - #cloud-config - apt: - security: - - arches: - - default -collect_scripts: - sources.list: | - #!/bin/bash - grep -c security.ubuntu.com /etc/apt/sources.list - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml deleted file mode 100644 index bde9398a..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_key.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# -# Add a sources.list entry with a given key (Debian Jessie) -# -required_features: - - apt - - lsb_release -cloud_config: | - #cloud-config - apt: - sources: - source1: - source: "deb http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu $RELEASE main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: SKS 1.1.6 - Comment: Hostname: keyserver.ubuntu.com - - mQINBFbZRUIBEAC+A0PIKYBP9kLC4hQtRrffRS11uLo8/BdtmOdrlW0hpPHzCfKnjR3tvSEI - lqPHG1QrrjAXKZDnZMRz+h/px7lUztvytGzHPSJd5ARUzAyjyRezUhoJ3VSCxrPqx62avuWf - RfoJaIeHfDehL5/dTVkyiWxfVZ369ZX6JN2AgLsQTeybTQ75+2z0xPrrhnGmgh6g0qTYcAaq - M5ONOGiqeSBX/Smjh6ALy5XkhUiFGLsI7Yluf6XSICY/x7gd6RAfgSIQrUTNMoS1sqhT4aot - +xvOfQy8ySkfAK4NddXql6E/+ZqTmBY/Lr0YklFBy8jGT+UysfiIznPMIwbmgq5Li7BtDDtX - b8Uyi4edPpjtextezfXYn4NVIpPL5dPZS/FXh4HpzyH0pYCfrH4QDGA7i52AGmhpiOFjJMo6 - N33sdjZHOH/2Vyp+QZaQnsdUAi1N4M6c33tQbpIScn1SY+El8z5JDA4PBzkw8HpLCi1gGoa6 - V4kfbWqXXbGAJFkLkP/vc4+pY9axOlmCkJg7xCPwhI75y1cONgovhz+BEXOzolh5KZuGbGbj - xe0wva5DLBeIg7EQFf+99pOS7Syby3Xpm6ZbswEFV0cllK4jf/QMjtfInxobuMoI0GV0bE5l - WlRtPCK5FnbHwxi0wPNzB/5fwzJ77r6HgPrR0OkT0lWmbUyoOQARAQABtC1MYXVuY2hwYWQg - UFBBIGZvciBjbG91ZCBpbml0IGRldmVsb3BtZW50IHRlYW2JAjgEEwECACIFAlbZRUICGwMG - CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEAg9Bvvk0wTfHfcP/REK5N2s1JYc69qEa9ZN - o6oi+A7l6AYw+ZY88O5TJe7F9otv5VXCIKSUT0Vsepjgf0mtXAgf/sb2lsJn/jp7tzgov3YH - vSrkTkRydz8xcA87gwQKePuvTLxQpftF4flrBxgSueIn5O/tPrBOxLz7EVYBc78SKg9aj9L2 - yUp+YuNevlwfZCTYeBb9r3FHaab2HcgkwqYch66+nKYfwiLuQ9NzXXm0Wn0JcEQ6pWvJscbj - C9BdawWovfvMK5/YLfI6Btm7F4mIpQBdhSOUp/YXKmdvHpmwxMCN2QhqYK49SM7qE9aUDbJL - arppSEBtlCLWhRBZYLTUna+BkuQ1bHz4St++XTR49Qd7vDERALpApDjB2dxPfMiBzCMwQQyq - uy13exU8o2ETLg+dZSLfDTzrBNsBFmXlw8WW17nTISYdKeGKL+QdlUjpzdwUMMzHhAO8SmMH - zjeSlDSRMXBJFAFSbCl7EwmMKa3yVX0zInT91fNllZ3iatAmtVdqVH/BFQfTIMH2ET7A8WzJ - ZzVSuMRhqoKdr5AMcHuJGPUoVkVJHQA+NNvEiXSysF3faL7jmKapmUwrhpYYX2H8pf+VMu2e - cLflKTI28dl+ZQ4Pl/aVsxrti/pzhdYy05Sn5ddtySyIkvo8L1cU5MWpbvSlFPkTstBUDLBf - pb0uBy+g0oxJQg15 - =uy53 - -----END PGP PUBLIC KEY BLOCK----- -collect_scripts: - sources.list: | - #!/bin/bash - cat /etc/apt/sources.list.d/source1.list - apt_key_list: | - #!/bin/bash - apt-key finger - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml deleted file mode 100644 index 25088135..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_keyserver.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# -# Add a sources.list entry with a key from a keyserver -# -required_features: - - apt - - lsb_release -cloud_config: | - #cloud-config - apt: - sources: - source1: - keyid: 1FF0D8535EF7E719E5C81B9C083D06FBE4D304DF - keyserver: keyserver.ubuntu.com - source: "deb http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu $RELEASE main" -collect_scripts: - sources.list: | - #!/bin/bash - cat /etc/apt/sources.list.d/source1.list - apt_key_list: | - #!/bin/bash - apt-key finger - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml deleted file mode 100644 index 143cb080..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_list.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Generate a sources.list -# -required_features: - - apt - - lsb_release -cloud_config: | - #cloud-config - apt: - sources_list: | - deb $MIRROR $RELEASE main restricted - deb-src $MIRROR $RELEASE main restricted - deb $PRIMARY $RELEASE universe restricted - deb-src $PRIMARY $RELEASE universe restricted - deb $SECURITY $RELEASE-security multiverse - deb-src $SECURITY $RELEASE-security multiverse -collect_scripts: - sources.list: | - #/bin/bash - cat /etc/apt/sources.list - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml b/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml deleted file mode 100644 index 9efdae52..00000000 --- a/tests/cloud_tests/configs/modules/apt_configure_sources_ppa.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Add a PPA to source.list -# -# NOTE: on older ubuntu releases the sources file added is named -# 'curtin-dev-test-archive-trusty', without 'ubuntu' in the middle -required_features: - - apt - - ppa - - ppa_file_name -cloud_config: | - #cloud-config - apt: - sources: - source1: - keyid: 0165013E - keyserver: keyserver.ubuntu.com - source: "ppa:curtin-dev/test-archive" -collect_scripts: - sources.list: | - #!/bin/bash - cat /etc/apt/sources.list.d/curtin-dev-ubuntu-test-archive-*.list - apt-key: | - #!/bin/bash - apt-key finger - sources_full: | - #!/bin/bash - cat /etc/apt/sources.list - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml b/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml deleted file mode 100644 index bd9b5d08..00000000 --- a/tests/cloud_tests/configs/modules/apt_pipelining_disable.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Disable apt pipelining value -# -required_features: - - apt -cloud_config: | - #cloud-config - apt: - apt_pipelining: false -collect_scripts: - 90cloud-init-pipelining: | - #!/bin/bash - cat /etc/apt/apt.conf.d/90cloud-init-pipelining - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml b/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml deleted file mode 100644 index cbed3ba3..00000000 --- a/tests/cloud_tests/configs/modules/apt_pipelining_os.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Set apt pipelining value to OS -# -required_features: - - apt -cloud_config: | - #cloud-config - apt: - apt_pipelining: os -collect_scripts: - 90cloud-init-pipelining: | - #!/bin/bash - cat /etc/apt/apt.conf.d/90cloud-init-pipelining - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/bootcmd.yaml b/tests/cloud_tests/configs/modules/bootcmd.yaml deleted file mode 100644 index 3a73994e..00000000 --- a/tests/cloud_tests/configs/modules/bootcmd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Early boot command -# -cloud_config: | - #cloud-config - bootcmd: - - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts -collect_scripts: - hosts: | - #!/bin/bash - cat /etc/hosts - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/byobu.yaml b/tests/cloud_tests/configs/modules/byobu.yaml deleted file mode 100644 index a9aa1f3f..00000000 --- a/tests/cloud_tests/configs/modules/byobu.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Install and enable byobu system wide and default user -# -required_features: - - byobu -cloud_config: | - #cloud-config - byobu_by_default: enable -collect_scripts: - byobu_installed: | - #!/bin/bash - which byobu - byobu_profile_enabled: | - #!/bin/bash - ls /etc/profile.d/Z97-byobu.sh - byobu_launch_exists: | - #!/bin/bash - which /usr/bin/byobu-launch - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ca_certs.yaml b/tests/cloud_tests/configs/modules/ca_certs.yaml deleted file mode 100644 index d939f435..00000000 --- a/tests/cloud_tests/configs/modules/ca_certs.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# -# Remove existing ca_certs and install custom ca-cert -# -cloud_config: | - #cloud-config - ca-certs: - remove-defaults: true - trusted: - - | - -----BEGIN CERTIFICATE----- - MIIGJzCCBA+gAwIBAgIBATANBgkqhkiG9w0BAQUFADCBsjELMAkGA1UEBhMCRlIx - DzANBgNVBAgMBkFsc2FjZTETMBEGA1UEBwwKU3RyYXNib3VyZzEYMBYGA1UECgwP - d3d3LmZyZWVsYW4ub3JnMRAwDgYDVQQLDAdmcmVlbGFuMS0wKwYDVQQDDCRGcmVl - bGFuIFNhbXBsZSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxIjAgBgkqhkiG9w0BCQEW - E2NvbnRhY3RAZnJlZWxhbi5vcmcwHhcNMTIwNDI3MTAzMTE4WhcNMjIwNDI1MTAz - MTE4WjB+MQswCQYDVQQGEwJGUjEPMA0GA1UECAwGQWxzYWNlMRgwFgYDVQQKDA93 - d3cuZnJlZWxhbi5vcmcxEDAOBgNVBAsMB2ZyZWVsYW4xDjAMBgNVBAMMBWFsaWNl - MSIwIAYJKoZIhvcNAQkBFhNjb250YWN0QGZyZWVsYW4ub3JnMIICIjANBgkqhkiG - 9w0BAQEFAAOCAg8AMIICCgKCAgEA3W29+ID6194bH6ejLrIC4hb2Ugo8v6ZC+Mrc - k2dNYMNPjcOKABvxxEtBamnSaeU/IY7FC/giN622LEtV/3oDcrua0+yWuVafyxmZ - yTKUb4/GUgafRQPf/eiX9urWurtIK7XgNGFNUjYPq4dSJQPPhwCHE/LKAykWnZBX - RrX0Dq4XyApNku0IpjIjEXH+8ixE12wH8wt7DEvdO7T3N3CfUbaITl1qBX+Nm2Z6 - q4Ag/u5rl8NJfXg71ZmXA3XOj7zFvpyapRIZcPmkvZYn7SMCp8dXyXHPdpSiIWL2 - uB3KiO4JrUYvt2GzLBUThp+lNSZaZ/Q3yOaAAUkOx+1h08285Pi+P8lO+H2Xic4S - vMq1xtLg2bNoPC5KnbRfuFPuUD2/3dSiiragJ6uYDLOyWJDivKGt/72OVTEPAL9o - 6T2pGZrwbQuiFGrGTMZOvWMSpQtNl+tCCXlT4mWqJDRwuMGrI4DnnGzt3IKqNwS4 - Qyo9KqjMIPwnXZAmWPm3FOKe4sFwc5fpawKO01JZewDsYTDxVj+cwXwFxbE2yBiF - z2FAHwfopwaH35p3C6lkcgP2k/zgAlnBluzACUI+MKJ/G0gv/uAhj1OHJQ3L6kn1 - SpvQ41/ueBjlunExqQSYD7GtZ1Kg8uOcq2r+WISE3Qc9MpQFFkUVllmgWGwYDuN3 - Zsez95kCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNT - TCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFFlfyRO6G8y5qEFKikl5 - ajb2fT7XMB8GA1UdIwQYMBaAFCNsLT0+KV14uGw+quK7Lh5sh/JTMA0GCSqGSIb3 - DQEBBQUAA4ICAQAT5wJFPqervbja5+90iKxi1d0QVtVGB+z6aoAMuWK+qgi0vgvr - mu9ot2lvTSCSnRhjeiP0SIdqFMORmBtOCFk/kYDp9M/91b+vS+S9eAlxrNCB5VOf - PqxEPp/wv1rBcE4GBO/c6HcFon3F+oBYCsUQbZDKSSZxhDm3mj7pb67FNbZbJIzJ - 70HDsRe2O04oiTx+h6g6pW3cOQMgIAvFgKN5Ex727K4230B0NIdGkzuj4KSML0NM - slSAcXZ41OoSKNjy44BVEZv0ZdxTDrRM4EwJtNyggFzmtTuV02nkUj1bYYYC5f0L - ADr6s0XMyaNk8twlWYlYDZ5uKDpVRVBfiGcq0uJIzIvemhuTrofh8pBQQNkPRDFT - Rq1iTo1Ihhl3/Fl1kXk1WR3jTjNb4jHX7lIoXwpwp767HAPKGhjQ9cFbnHMEtkro - RlJYdtRq5mccDtwT0GFyoJLLBZdHHMHJz0F9H7FNk2tTQQMhK5MVYwg+LIaee586 - CQVqfbscp7evlgjLW98H+5zylRHAgoH2G79aHljNKMp9BOuq6SnEglEsiWGVtu2l - hnx8SB3sVJZHeer8f/UQQwqbAO+Kdy70NmbSaqaVtp8jOxLiidWkwSyRTsuU6D8i - DiH5uEqBXExjrj0FslxcVKdVj5glVcSmkLwZKbEU1OKwleT/iXFhvooWhQ== - -----END CERTIFICATE----- -collect_scripts: - cert_count: | - #!/bin/bash - ls -l /etc/ssl/certs | wc -l - cert: | - #!/bin/bash - md5sum /etc/ssl/certs/ca-certificates.crt -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/debug_disable.yaml b/tests/cloud_tests/configs/modules/debug_disable.yaml deleted file mode 100644 index 63218b18..00000000 --- a/tests/cloud_tests/configs/modules/debug_disable.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# -# Do not run in debug mode -# -cloud_config: | - #cloud-config - debug: - verbose: False - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/debug_enable.yaml b/tests/cloud_tests/configs/modules/debug_enable.yaml deleted file mode 100644 index d44147db..00000000 --- a/tests/cloud_tests/configs/modules/debug_enable.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# -# Run in debug mode -# -cloud_config: | - #cloud-config - debug: - verbose: True - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/final_message.yaml b/tests/cloud_tests/configs/modules/final_message.yaml deleted file mode 100644 index c9ed6118..00000000 --- a/tests/cloud_tests/configs/modules/final_message.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Print a final message with various predefined variables -# -cloud_config: | - #cloud-config - final_message: | - This is my final message! - $version - $timestamp - $datasource - $uptime - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/keys_to_console.yaml b/tests/cloud_tests/configs/modules/keys_to_console.yaml deleted file mode 100644 index 5d86e739..00000000 --- a/tests/cloud_tests/configs/modules/keys_to_console.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Hide printing of ssh key and fingerprints for specific keys -# -required_features: - - syslog -cloud_config: | - #cloud-config - ssh_fp_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] - ssh_key_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] -collect_scripts: - syslog: | - #!/bin/bash - cat /var/log/syslog - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/landscape.yaml b/tests/cloud_tests/configs/modules/landscape.yaml deleted file mode 100644 index ed2c37c4..00000000 --- a/tests/cloud_tests/configs/modules/landscape.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# -# Setup landscape client settings -# -# 2016-11-17: Disabled due to this not working -# -enabled: false -required_features: - - landscape -cloud_config: | - #cloud-conifg - landscape: - client: - log_level: "info" - url: "https://landscape.canonical.com/message-system" - ping_url: "http://landscape.canonical.com/ping" - data_path: "/var/lib/landscape/client" - http_proxy: "http://my.proxy.com/foobar" - https_proxy: "https://my.proxy.com/foobar" - tags: "server,cloud" - computer_title: "footitle" - registration_key: "fookey" - account_name: "fooaccount" -collect_scripts: - client.conf: | - #!/bin/bash - cat /etc/landscape/client.conf - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/locale.yaml b/tests/cloud_tests/configs/modules/locale.yaml deleted file mode 100644 index e01518a1..00000000 --- a/tests/cloud_tests/configs/modules/locale.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Set locale to non-default option and verify -# -required_features: - - engb_locale - - locale_gen -cloud_config: | - #cloud-config - locale: en_GB.UTF-8 - locale_configfile: /etc/default/locale -collect_scripts: - locale_default: | - #!/bin/bash - cat /etc/default/locale - locale_a: | - #!/bin/bash - locale -a - locale_gen: | - #!/bin/bash - cat /etc/locale.gen | grep -v '^#' | uniq - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/lxd_bridge.yaml b/tests/cloud_tests/configs/modules/lxd_bridge.yaml deleted file mode 100644 index e6b7e76a..00000000 --- a/tests/cloud_tests/configs/modules/lxd_bridge.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# -# LXD configured with directory backend and IPv4 bridge -# -required_features: - - lxd -cloud_config: | - #cloud-config - lxd: - init: - storage_backend: dir - bridge: - mode: new - name: lxdbr0 - ipv4_address: 10.100.100.1 - ipv4_netmask: 24 - ipv4_dhcp_first: 10.100.100.100 - ipv4_dhcp_last: 10.100.100.200 - ipv4_nat: true - domain: lxd -collect_scripts: - lxc: | - #!/bin/bash - which lxc - lxd: | - #!/bin/bash - which lxd - lxc-bridge: | - #!/bin/bash - ip addr show lxdbr0 - cat /etc/default/lxd-bridge 2>/dev/null | grep -v ^# | sort -u - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/lxd_dir.yaml b/tests/cloud_tests/configs/modules/lxd_dir.yaml deleted file mode 100644 index f93a3fa7..00000000 --- a/tests/cloud_tests/configs/modules/lxd_dir.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# -# LXD configured with directory backend -# -required_features: - - lxd -cloud_config: | - #cloud-config - lxd: - init: - storage_backend: dir -collect_scripts: - lxc: | - #!/bin/bash - which lxc - lxd: | - #!/bin/bash - which lxd - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ntp.yaml b/tests/cloud_tests/configs/modules/ntp.yaml deleted file mode 100644 index fbef431b..00000000 --- a/tests/cloud_tests/configs/modules/ntp.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Emtpy NTP config to setup using defaults -# -cloud_config: | - #cloud-config - ntp: - pools: {} - servers: {} -collect_scripts: - ntp_installed: | - #!/bin/bash - ntpd --version > /dev/null 2>&1 - echo $? - ntp_conf_dist_empty: | - #!/bin/bash - ls /etc/ntp.conf.dist | wc -l - ntp_conf_pool_list: | - #!/bin/bash - grep 'pool.ntp.org' /etc/ntp.conf | grep -v ^# - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ntp_pools.yaml b/tests/cloud_tests/configs/modules/ntp_pools.yaml deleted file mode 100644 index 3a93faa2..00000000 --- a/tests/cloud_tests/configs/modules/ntp_pools.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# -# NTP config using specific pools -# -# NOTE: lsb_release listed here because with recent cloud-init deb with -# (LP: 1628337) resolved, cloud-init will attempt to configure archives. -# this fails without lsb_release as UNAVAILABLE is used for $RELEASE -required_features: - - lsb_release -cloud_config: | - #cloud-config - ntp: - pools: - - 0.cloud-init.mypool - - 1.cloud-init.mypool - - 172.16.15.14 -collect_scripts: - ntp_installed_pools: | - #!/bin/bash - ntpd --version > /dev/null 2>&1 - echo $? - ntp_conf_dist_pools: | - #!/bin/bash - ls /etc/ntp.conf.dist | wc -l - ntp_conf_pools: | - #!/bin/bash - grep '^pool' /etc/ntp.conf - ntpq_servers: | - #!/bin/sh - ntpq -p -w - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ntp_servers.yaml b/tests/cloud_tests/configs/modules/ntp_servers.yaml deleted file mode 100644 index d59d45a8..00000000 --- a/tests/cloud_tests/configs/modules/ntp_servers.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# -# NTP config using specific servers -# -required_features: - - lsb_release -cloud_config: | - #cloud-config - ntp: - servers: - - 172.16.15.14 - - 172.16.17.18 -collect_scripts: - ntp_installed_servers: | - #!/bin/sh - ntpd --version > /dev/null 2>&1 - echo $? - ntp_conf_dist_servers: | - #!/bin/sh - cat /etc/ntp.conf.dist | wc -l - ntp_conf_servers: | - #!/bin/sh - grep '^server' /etc/ntp.conf - ntpq_servers: | - #!/bin/sh - ntpq -p -w - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml b/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml deleted file mode 100644 index 71d24b83..00000000 --- a/tests/cloud_tests/configs/modules/package_update_upgrade_install.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# -# Update/upgrade via apt and then install a pair of packages -# -# NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' -# NOTE: the testcase for this looks for the command in history.log as -# /usr/bin/apt-get..., which is not how it always appears. it should -# instead look for just apt-get... -# NOTE: this testcase should not require 'apt_up_out', and should look for a -# call to 'apt-get upgrade' or 'apt-get dist-upgrade' in cloud-init.log -# rather than 'Calculating upgrade...' in output -required_features: - - apt - - apt_hist_fmt - - apt_up_out -cloud_config: | - #cloud-config - packages: - - htop - - tree - package_update: true - package_upgrade: true -collect_scripts: - apt_history_cmdline: | - #!/bin/bash - grep ^Commandline: /var/log/apt/history.log - dpkg_htop: | - #!/bin/bash - dpkg -l | grep htop | wc -l - dpkg_tree: | - #!/bin/bash - dpkg -l | grep tree | wc -l - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/runcmd.yaml b/tests/cloud_tests/configs/modules/runcmd.yaml deleted file mode 100644 index 04e5a050..00000000 --- a/tests/cloud_tests/configs/modules/runcmd.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# -# Run a simple command -# -cloud_config: | - #cloud-config - runcmd: - - echo cloud-init run cmd test > /tmp/run_cmd -collect_scripts: - run_cmd: | - #!/bin/bash - cat /tmp/run_cmd - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/salt_minion.yaml b/tests/cloud_tests/configs/modules/salt_minion.yaml deleted file mode 100644 index f20d24f0..00000000 --- a/tests/cloud_tests/configs/modules/salt_minion.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Create config for a salt minion -# -# 2016-11-17: Currently takes >60 seconds results in test failure -# -enabled: False -cloud_config: | - #cloud-config - salt_minion: - conf: - master: salt.mydomain.com - public_key: | - ------BEGIN PUBLIC KEY------- - - ------END PUBLIC KEY------- - private_key: | - ------BEGIN PRIVATE KEY------ - - ------END PRIVATE KEY------- -collect_scripts: - minion: | - #!/bin/bash - cat /etc/salt/minion - minion_id: | - #!/bin/bash - cat /etc/salt/minion_id - minion.pem: | - #!/bin/bash - cat /etc/salt/pki/minion/minion.pem - minion.pub: | - #!/bin/bash - cat /etc/salt/pki/minion/minion.pub - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/seed_random_command.yaml b/tests/cloud_tests/configs/modules/seed_random_command.yaml deleted file mode 100644 index 6a9157eb..00000000 --- a/tests/cloud_tests/configs/modules/seed_random_command.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# -# Use uuid to create a random string -# -# 2016-11-15 Disabled as this is not working currently -# -enabled: False -cloud_config: | - #cloud-config - random_seed: - command: ["cat", "/proc/sys/kernel/random/uuid"] - command_required: true - file: /root/seed -collect_scripts: - seed_data: | - #!/bin/bash - cat /root/seed - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/seed_random_data.yaml b/tests/cloud_tests/configs/modules/seed_random_data.yaml deleted file mode 100644 index a9b2c885..00000000 --- a/tests/cloud_tests/configs/modules/seed_random_data.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Push in random raw string to set as seed -# -cloud_config: | - #cloud-config - random_seed: - data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' - encoding: raw - file: /root/seed -collect_scripts: - seed_data: | - #!/bin/bash - cat /root/seed - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/set_hostname.yaml b/tests/cloud_tests/configs/modules/set_hostname.yaml deleted file mode 100644 index c96344cf..00000000 --- a/tests/cloud_tests/configs/modules/set_hostname.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Set the hostname and update /etc/hosts -# -required_features: - - hostname -cloud_config: | - #cloud-config - hostname: myhostname -collect_scripts: - hosts: | - #!/bin/bash - grep ^127 /etc/hosts - hostname: | - #!/bin/bash - hostname - fqdn: | - #!/bin/bash - hostname --fqdn - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml b/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml deleted file mode 100644 index daf75931..00000000 --- a/tests/cloud_tests/configs/modules/set_hostname_fqdn.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -# Set the hostname and update /etc/hosts -# -required_features: - - hostname -cloud_config: | - #cloud-config - manage_etc_hosts: true - hostname: myhostname - fqdn: host.myorg.com -collect_scripts: - hosts: | - #!/bin/bash - grep ^127 /etc/hosts - hostname: | - #!/bin/bash - hostname - fqdn: | - #!/bin/bash - hostname --fqdn - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/set_password.yaml b/tests/cloud_tests/configs/modules/set_password.yaml deleted file mode 100644 index 04d7c58a..00000000 --- a/tests/cloud_tests/configs/modules/set_password.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# -# Set password of default user -# -required_features: - - ubuntu_user -cloud_config: | - #cloud-config - password: password - chpasswd: { expire: False } - ssh_pwauth: True -collect_scripts: - shadow: | - #!/bin/bash - cat /etc/shadow - sshd_config: | - #!/bin/bash - grep '^PasswordAuth' /etc/ssh/sshd_config - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/set_password_expire.yaml b/tests/cloud_tests/configs/modules/set_password_expire.yaml deleted file mode 100644 index 789604b0..00000000 --- a/tests/cloud_tests/configs/modules/set_password_expire.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Expire password for all users -# -required_features: - - sshd -cloud_config: | - #cloud-config - chpasswd: { expire: True } - users: - - name: tom - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. - lock_passwd: false - - name: dick - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. - lock_passwd: false - - name: harry - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. - lock_passwd: false - - name: jane - password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. - lock_passwd: false -collect_scripts: - shadow: | - #!/bin/bash - cat /etc/shadow - sshd_config: | - #!/bin/bash - grep '^PasswordAuth' /etc/ssh/sshd_config - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/set_password_list.yaml b/tests/cloud_tests/configs/modules/set_password_list.yaml deleted file mode 100644 index a2a89c9d..00000000 --- a/tests/cloud_tests/configs/modules/set_password_list.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Set password of list of users -# -cloud_config: | - #cloud-config - ssh_pwauth: yes - users: - - name: tom - # md5 gotomgo - passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" - lock_passwd: false - - name: dick - # md5 gocubsgo - passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" - lock_passwd: false - - name: harry - # sha512 goharrygo - passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/" - lock_passwd: false - - name: jane - # sha256 gojanego - passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." - lock_passwd: false - - name: "mikey" - lock_passwd: false - chpasswd: - list: - - tom:mypassword123! - - dick:RANDOM - - harry:RANDOM - - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 -collect_scripts: - shadow: | - #!/bin/bash - cat /etc/shadow - sshd_config: | - #!/bin/bash - grep '^PasswordAuth' /etc/ssh/sshd_config - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/set_password_list_string.yaml b/tests/cloud_tests/configs/modules/set_password_list_string.yaml deleted file mode 100644 index c2a0f631..00000000 --- a/tests/cloud_tests/configs/modules/set_password_list_string.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Set password of list of users as a string -# -cloud_config: | - #cloud-config - ssh_pwauth: yes - users: - - name: tom - # md5 gotomgo - passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" - lock_passwd: false - - name: dick - # md5 gocubsgo - passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" - lock_passwd: false - - name: harry - # sha512 goharrygo - passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/" - lock_passwd: false - - name: jane - # sha256 gojanego - passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." - lock_passwd: false - - name: "mikey" - lock_passwd: false - chpasswd: - list: | - tom:mypassword123! - dick:RANDOM - harry:RANDOM - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 -collect_scripts: - shadow: | - #!/bin/bash - cat /etc/shadow - sshd_config: | - #!/bin/bash - grep '^PasswordAuth' /etc/ssh/sshd_config - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/snappy.yaml b/tests/cloud_tests/configs/modules/snappy.yaml deleted file mode 100644 index 43f93295..00000000 --- a/tests/cloud_tests/configs/modules/snappy.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Install snappy -# -required_features: - - snap -cloud_config: | - #cloud-config - snappy: - system_snappy: auto -collect_scripts: - snapd: | - #!/bin/bash - dpkg -s snapd - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml b/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml deleted file mode 100644 index 746653ec..00000000 --- a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_disable.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# -# Disable fingerprint printing -# -required_features: - - syslog -cloud_config: | - #cloud-config - ssh_genkeytypes: [] - no_ssh_fingerprints: true -collect_scripts: - syslog: | - #!/bin/bash - cat /var/log/syslog - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml b/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml deleted file mode 100644 index 9f5dc34a..00000000 --- a/tests/cloud_tests/configs/modules/ssh_auth_key_fingerprints_enable.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Print auth keys with different hash than md5 -# -# NOTE: testcase checks for '256 SHA256:.*(ECDSA)' on output line on trusty -# this fails as line in output reads '256:.*(ECDSA)' -required_features: - - syslog - - ssh_key_fmt -cloud_config: | - #cloud-config - ssh_genkeytypes: - - ecdsa - - ed25519 - ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXW9Gg5H7ehjdSc6qDzwNtgCy94XYHhEYlXZMO2+FJrH3wfHGiMfCwOHxcOMt2QiXItULthdeQWS9QjBSSjVRXf6731igFrqPFyS9qBlOQ5D29C4HBXFnQggGVpBNJ82IRJv7szbbe/vpgLBP4kttUza9Dr4e1YM1ln4PRnjfXea6T0m+m1ixNb5432pTXlqYOnNOxSIm1gHgMLxPuDrJvQERDKrSiKSjIdyC9Jd8t2e1tkNLY0stmckVRbhShmcJvlyofHWbc2Ca1mmtP7MlS1VQnfLkvU1IrFwkmaQmaggX6WR6coRJ6XFXdWcq/AI2K6GjSnl1dnnCxE8VCEXBlXgFzad+PMSG4yiL5j8Oo1ZVpkTdgBnw4okGqTYCXyZg6X00As9IBNQfZMFlQXlIo4FiWgj3CO5QHQOyOX6FuEumaU13GnERrSSdp9tCs1Qm3/DG2RSCQBWTfcgMcStIvKqvJ3IjFn0vGLvI3Ampnq9q1SHwmmzAPSdzcMA76HyMUA5VWaBvWHlUxzIM6unxZASnwvuCzpywSEB5J2OF+p6H+cStJwQ32XwmOG8pLp1srlVWpqZI58Du/lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs506oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w== -collect_scripts: - syslog: | - #!/bin/bash - cat /var/log/syslog - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ssh_import_id.yaml b/tests/cloud_tests/configs/modules/ssh_import_id.yaml deleted file mode 100644 index b62d3f69..00000000 --- a/tests/cloud_tests/configs/modules/ssh_import_id.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# -# Import a user's ssh key via gh or lp -# -required_features: - - ubuntu_user - - sudo -cloud_config: | - #cloud-config - ssh_import_id: - - gh:powersj - - lp:smoser -collect_scripts: - auth_keys_ubuntu: | - #!/bin/bash - cat /home/ubuntu/.ssh/authorized_keys - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml b/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml deleted file mode 100644 index 659fd939..00000000 --- a/tests/cloud_tests/configs/modules/ssh_keys_generate.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# -# SSH keys generated using cloud-init -# -required_features: - - ubuntu_user -cloud_config: | - #cloud-config - ssh_genkeytypes: - - ecdsa - - ed25519 - authkey_hash: sha512 -collect_scripts: - auth_keys_root: | - #!/bin/bash - cat /root/.ssh/authorized_keys - auth_keys_ubuntu: | - #!/bin/bash - cat /home/ubuntu/ssh/authorized_keys - dsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_dsa_key.pub - dsa_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_dsa_key - rsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_rsa_key.pub - rsa_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_rsa_key - ecdsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_ecdsa_key.pub - ecdsa_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_ecdsa_key - ed25519_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_ed25519_key.pub - ed25519_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_ed25519_key - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml b/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml deleted file mode 100644 index 5ceb3623..00000000 --- a/tests/cloud_tests/configs/modules/ssh_keys_provided.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# -# SSH keys provided via cloud config -# -enabled: False -required_features: - - ubuntu_user - - sudo -cloud_config: | - #cloud-config - disable_root: false - ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXW9Gg5H7ehjdSc6qDzwNtgCy94XYHhEYlXZMO2+FJrH3wfHGiMfCwOHxcOMt2QiXItULthdeQWS9QjBSSjVRXf6731igFrqPFyS9qBlOQ5D29C4HBXFnQggGVpBNJ82IRJv7szbbe/vpgLBP4kttUza9Dr4e1YM1ln4PRnjfXea6T0m+m1ixNb5432pTXlqYOnNOxSIm1gHgMLxPuDrJvQERDKrSiKSjIdyC9Jd8t2e1tkNLY0stmckVRbhShmcJvlyofHWbc2Ca1mmtP7MlS1VQnfLkvU1IrFwkmaQmaggX6WR6coRJ6XFXdWcq/AI2K6GjSnl1dnnCxE8VCEXBlXgFzad+PMSG4yiL5j8Oo1ZVpkTdgBnw4okGqTYCXyZg6X00As9IBNQfZMFlQXlIo4FiWgj3CO5QHQOyOX6FuEumaU13GnERrSSdp9tCs1Qm3/DG2RSCQBWTfcgMcStIvKqvJ3IjFn0vGLvI3Ampnq9q1SHwmmzAPSdzcMA76HyMUA5VWaBvWHlUxzIM6unxZASnwvuCzpywSEB5J2OF+p6H+cStJwQ32XwmOG8pLp1srlVWpqZI58Du/lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs506oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w== - ssh_keys: - rsa_private: | - -----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAtPx6PqN3iSEsnTtibyIEy52Tra8T5fn0ryXyg46Di2NBwdnj - o8trNv9jenfV/UhmePl58lXjT43wV8OCMl6KsYXyBdegM35NNtono4I4mLLKFMR9 - 9TOtDn6iYcaNenVhF3ZCj9Z2nNOlTrdc0uchHqKMrxLjCRCUrL91Uf+xioTF901Y - RM+ZqC5lT92yAL76F4qPF+Lq1QtUfNfUIwwvOp5ccDZLPxij0YvyBzubYye9hJHu - yjbJv78R4JHV+L2WhzSoX3W/6WrxVzeXqFGqH894ccOaC/7tnqSP6V8lIQ6fE2+c - DurJcpM3CJRgkndGHjtU55Y71YkcdLksSMvezQIDAQABAoIBAQCrU4IJP8dNeaj5 - IpkY6NQvR/jfZqfogYi+MKb1IHin/4rlDfUvPcY9pt8ttLlObjYK+OcWn3Vx/sRw - 4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2unRQvLZpMRdywBm - lq95OrCghnG03aUsFJUZPpi5ydnwbA12ma+KHkG0EzaVlhA7X9N6z0K6U+zue2gl - goMLt/MH0rsYawkHrwiwXaIFQeyV4MJP0vmrZLbFk1bycu9X/xPtTYotWyWo4eKA - cb05uu04qwexkKHDM0KXtT0JecbTo2rOefFo8Uuab6uJY+fEHNocZ+v1vLA4aOxJ - ovp1JuXlAoGBAOWYNgKrlTfy5n0sKsNk+1RuL2jHJZJ3HMd0EIt7/fFQN3Fi08Hu - jtntqD30Wj+DJK8b8Lrt66FruxyEJm5VhVmwkukrLR5ige2f6ftZnoFCmdyy+0zP - dnPZSUe2H5ZPHa+qthJgHLn+al2P04tGh+1fGHC2PbP+e0Co+/ZRIOxrAoGBAMnN - IEen9/FRsqvnDd36I8XnJGskVRTZNjylxBmbKcuMWm+gNhOI7gsCAcqzD4BYZjjW - pLhrt/u9p+l4MOJy6OUUdM/okg12SnJEGryysOcVBcXyrvOfklWnANG4EAH5jt1N - ftTb1XTxzvWVuR/WJK0B5MZNYM71cumBdUDtPi+nAoGAYmoIXMSnxb+8xNL10aOr - h9ljQQp8NHgSQfyiSufvRk0YNuYh1vMnEIsqnsPrG2Zfhx/25GmvoxXGssaCorDN - 5FAn6QK06F1ZTD5L0Y3sv4OI6G1gAuC66ZWuL6sFhyyKkQ4f1WiVZ7SCa3CHQSAO - i9VDaKz1bf4bXvAQcNj9v9kCgYACSOZCqW4vN0OUmqsXhkt9ZB6Pb/veno70pNPR - jmYsvcwQU3oJQpWfXkhy6RAV3epaXmPDCsUsfns2M3wqNC7a2R5xdCqjKGGzZX4A - AO3rz9se4J6Gd5oKijeCKFlWDGNHsibrdgm2pz42nZlY+O21X74dWKbt8O16I1MW - hxkbJQKBgAXfuen/srVkJgPuqywUYag90VWCpHsuxdn+fZJa50SyZADr+RbiDfH2 - vek8Uo8ap8AEsv4Rfs9opUcUZevLp3g2741eOaidHVLm0l4iLIVl03otGOqvSzs+ - A3tFPEOxauXpzCt8f8eXsz0WQXAgIKW2h8zu5QHjomioU3i27mtE - -----END RSA PRIVATE KEY----- - rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97N root@xenial-lxd - dsa_private: | - -----BEGIN DSA PRIVATE KEY----- - MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXrhOVAfzZ6+jklP - 55mzvC7jO53PWWC31hq10xBoWdev0WtcNF9Tv+4bAa1263y51Rqo4GI7xx+xic1d - mLqqfYijBT9k48J/1tV0cs1Wjs6FP/IJTD/kYVC930JjYQMi722lBnUxsQIVAL7i - z3fTGKTvSzvW0wQlwnYpS2QFAoGANp+KdyS9V93HgxGQEN1rlj/TSv/a3EVdCKtE - nQf55aPHxDAVDVw5JtRh4pZbbRV4oGRPc9KOdjo5BU28vSM3Lmhkb+UaaDXwHkgI - nK193o74DKjADWZxuLyyiKHiMOhxozoxDfjWxs8nz6uqvSW0pr521EwIY6RajbED - nZ2a3GkCgYEAyoUomNRB6bmpsIfzt8zdtqLP5umIj2uhr9MVPL8/QdbxmJ72Z7pf - Q2z1B7QAdIBGOlqJXtlau7ABhWK29Efe+99ObyTSSdDc6RCDeAwUmBAiPRQhDH2E - wExw3doDSCUb28L1B50wBzQ8mC3KXp6C7IkBXWspb16DLHUHFSI8bkICFA5kVUcW - nCPOXEQsayANi8+Cb7BH - -----END DSA PRIVATE KEY----- - dsa_public: ssh-dss AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4RZS8cNM4ZpeuE5UB/Nnr6OSU/nmbO8LuM7nc9ZYLfWGrXTEGhZ16/Ra1w0X1O/7hsBrXbrfLnVGqjgYjvHH7GJzV2Yuqp9iKMFP2Tjwn/W1XRyzVaOzoU/8glMP+RhUL3fQmNhAyLvbaUGdTGxAAAAFQC+4s930xik70s71tMEJcJ2KUtkBQAAAIA2n4p3JL1X3ceDEZAQ3WuWP9NK/9rcRV0Iq0SdB/nlo8fEMBUNXDkm1GHillttFXigZE9z0o52OjkFTby9IzcuaGRv5RpoNfAeSAicrX3ejvgMqMANZnG4vLKIoeIw6HGjOjEN+NbGzyfPq6q9JbSmvnbUTAhjpFqNsQOdnZrcaQAAAIEAyoUomNRB6bmpsIfzt8zdtqLP5umIj2uhr9MVPL8/QdbxmJ72Z7pfQ2z1B7QAdIBGOlqJXtlau7ABhWK29Efe+99ObyTSSdDc6RCDeAwUmBAiPRQhDH2EwExw3doDSCUb28L1B50wBzQ8mC3KXp6C7IkBXWspb16DLHUHFSI8bkI= root@xenial-lxd - ed25519_private: | - -----BEGIN OPENSSH PRIVATE KEY----- - b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW - QyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNTOhteXao0Nl5DVThJ2+QAAAJgwt+lcMLfp - XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNTOhteXao0Nl5DVThJ2+Q - AAAEDQlFZpz9q8+/YJHS9+jPAqy2ZT6cGEv8HTB6RZtTjd/dudAZSu4vjZpVWzId5pXmZg - 1M6G15dqjQ2XkNVOEnb5AAAAD3Jvb3RAeGVuaWFsLWx4ZAECAwQFBg== - -----END OPENSSH PRIVATE KEY----- - ed25519_public: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6G15dqjQ2XkNVOEnb5 root@xenial-lxd - ecdsa_private: | - -----BEGIN EC PRIVATE KEY----- - MHcCAQEEIDuK+QFc1wmyJY8uDqQVa1qHte30Rk/fdLxGIBkwJAyOoAoGCCqGSM49 - AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY5mpZqxgX4vcgb - 7f/CtXuM6s2svcDJqAeXr6Wk8OJJcMxylA== - -----END EC PRIVATE KEY----- - ecdsa_public: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFsS5Tvky/IC/dXhE/afxxUG6kdQOvdQJCYGZN42OZqWasYF+L3IG+3/wrV7jOrNrL3AyagHl6+lpPDiSXDMcpQ= root@xenial-lxd -collect_scripts: - auth_keys_root: | - #!/bin/bash - cat /root/.ssh/authorized_keys - auth_keys_ubuntu: | - #!/bin/bash - cat /home/ubuntu/ssh/authorized_keys - dsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_dsa_key.pub - dsa_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_dsa_key - rsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_rsa_key.pub - rsa_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_rsa_key - ecdsa_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_ecdsa_key.pub - ecdsa_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_ecdsa_key - ed25519_public: | - #!/bin/bash - cat /etc/ssh/ssh_host_ed25519_key.pub - ed25519_private: | - #!/bin/bash - cat /etc/ssh/ssh_host_ed25519_key - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/timezone.yaml b/tests/cloud_tests/configs/modules/timezone.yaml deleted file mode 100644 index 5112aa9f..00000000 --- a/tests/cloud_tests/configs/modules/timezone.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# -# Set system timezone -# -required_features: - - daylight_time -cloud_config: | - #cloud-config - timezone: US/Aleutian -collect_scripts: - timezone: | - #!/bin/bash - # date will convert this to system's configured time zone. - # use a static date to avoid dealing with daylight savings. - date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400" - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/user_groups.yaml b/tests/cloud_tests/configs/modules/user_groups.yaml deleted file mode 100644 index 71cc9da3..00000000 --- a/tests/cloud_tests/configs/modules/user_groups.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# -# Create groups and users with various options -# -required_features: - - ubuntu_user -cloud_config: | - #cloud-config - # Add groups to the system - groups: - - secret: [foobar,barfoo] - - cloud-users - - # Add users to the system. Users are added after groups are added. - users: - - default - - name: foobar - gecos: Foo B. Bar - primary-group: foobar - groups: users - expiredate: 2038-01-19 - lock_passwd: false - passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - - name: barfoo - gecos: Bar B. Foo - sudo: ALL=(ALL) NOPASSWD:ALL - groups: cloud-users - lock_passwd: true - - name: cloudy - gecos: Magic Cloud App Daemon User - inactive: true - system: true -collect_scripts: - group_ubuntu: | - #!/bin/bash - getent group ubuntu - group_cloud_users: | - #!/bin/bash - getent group cloud-users - user_ubuntu: | - #!/bin/bash - getent passwd ubuntu - user_foobar: | - #!/bin/bash - getent passwd foobar - user_barfoo: | - #!/bin/bash - getent passwd barfoo - user_cloudy: | - #!/bin/bash - getent passwd cloudy - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/configs/modules/write_files.yaml b/tests/cloud_tests/configs/modules/write_files.yaml deleted file mode 100644 index ce936b7b..00000000 --- a/tests/cloud_tests/configs/modules/write_files.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Write various file types -# -# NOTE: on trusty 'file' has an output formatting error for binary files and -# has 2 spaces in 'LSB executable', which causes a failure here -required_features: - - no_file_fmt_e -cloud_config: | - #cloud-config - write_files: - - encoding: b64 - content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4 - owner: root:root - path: /root/file_b64 - permissions: '0644' - - content: | - # My new /root/file_text - - SMBDOPTIONS="-D" - path: /root/file_text - - content: !!binary | - f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI - AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA - AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA - path: /root/file_binary - permissions: '0555' - - encoding: gzip - content: !!binary | - H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= - path: /root/file_gzip - permissions: '0755' -collect_scripts: - file_b64: | - #!/bin/bash - file /root/file_b64 - file_text: | - #!/bin/bash - file /root/file_text - file_binary: | - #!/bin/bash - file /root/file_binary - file_gzip: | - #!/bin/bash - file /root/file_gzip - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/bugs/README.md b/tests/cloud_tests/testcases/bugs/README.md new file mode 100644 index 00000000..09ce0765 --- /dev/null +++ b/tests/cloud_tests/testcases/bugs/README.md @@ -0,0 +1,13 @@ +# Bug Test Configs + +## purpose +Configs that reproduce bugs filed against cloud-init. Having test configs for +cloud-init bugs ensures that the fixes do not break in the future, and makes it +easy to see how many systems and platforms are effected by a new bug. + +## structure +Should have one test config for most bugs filed. The name of the test should +contain ``lp`` followed by the bug number. It may also be useful to add a +comment to each bug config with a summary copied from the bug report. + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/bugs/lp1511485.yaml b/tests/cloud_tests/testcases/bugs/lp1511485.yaml new file mode 100644 index 00000000..ebf9763f --- /dev/null +++ b/tests/cloud_tests/testcases/bugs/lp1511485.yaml @@ -0,0 +1,11 @@ +# +# LP Bug 1511485: final_message is silent on ubuntu-12.04.5 / cloud-init 0.6.3 +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + final_message: "Final message from cloud-config" + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/bugs/lp1611074.yaml b/tests/cloud_tests/testcases/bugs/lp1611074.yaml new file mode 100644 index 00000000..960679d5 --- /dev/null +++ b/tests/cloud_tests/testcases/bugs/lp1611074.yaml @@ -0,0 +1,8 @@ +# +# LP Bug 1611074: Reformatting of ephemeral drive fails on resize of Azure VM +# +# 2016-11-18: Disabled until test written +# +enabled: False + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/bugs/lp1628337.yaml b/tests/cloud_tests/testcases/bugs/lp1628337.yaml new file mode 100644 index 00000000..e39b3cd8 --- /dev/null +++ b/tests/cloud_tests/testcases/bugs/lp1628337.yaml @@ -0,0 +1,23 @@ +# +# LP Bug 1628337: cloud-init tries to install NTP before even configuring the archives +# +required_features: + - apt + - lsb_release +cloud_config: | + #cloud-config + ntp: + servers: ['ntp.ubuntu.com'] + apt: + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu/ +collect_sciprts: + ntp.conf: | + #!/bin/bash + cat /etc/ntp.conf + sources.list: | + #!/bin/bash + cat /etc/apt/sources.list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/README.md b/tests/cloud_tests/testcases/examples/README.md new file mode 100644 index 00000000..110a223b --- /dev/null +++ b/tests/cloud_tests/testcases/examples/README.md @@ -0,0 +1,12 @@ +# Example Test Configs + +## Purpose +This folder contains example cloud configs found on +[cloudinit.readthedocs.io](https://cloudinit.readthedocs.io/en/latest/topics/examples.html). +Examples covered by other tests, like modules, are excluded from tests here +to prevent duplication and reduce test time. + +## Structure +One test per example test config on cloudinit.readthedocs.io + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/TODO.md b/tests/cloud_tests/testcases/examples/TODO.md new file mode 100644 index 00000000..8db0e98e --- /dev/null +++ b/tests/cloud_tests/testcases/examples/TODO.md @@ -0,0 +1,15 @@ +# Missing Examples + +Below lists each of the issing examples and why it is not currently added. + + - Chef (takes > 60 seconds to run) + - Puppet (takes > 60 seconds to run) + - Manage resolve.conf (lxd backend overrides changes) + - Adding a yum repository (need centos system) + - Register RedHat Subscription (need centos system + subscription) + - Adjust mount points mounted (need multiple disks) + - Call a url when finished (need end point) + - Reboot/poweroff when finished (how to test) + - Disk setup (need multiple disks) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/add_apt_repositories.yaml b/tests/cloud_tests/testcases/examples/add_apt_repositories.yaml new file mode 100644 index 00000000..4b8575f7 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/add_apt_repositories.yaml @@ -0,0 +1,23 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +required_features: + - apt +cloud_config: | + #cloud-config + apt: + primary: + - arches: [default] + uri: "http://www.gtlib.gatech.edu/pub/ubuntu-releases/" +collect_scripts: + ubuntu.sources.list: | + #!/bin/bash + cat /etc/apt/sources.list | grep -v '^#' | sed '/^\s*$/d' | grep archive.ubuntu.com | wc -l + gatech.sources.list: | + #!/bin/bash + cat /etc/apt/sources.list | grep -v '^#' | sed '/^\s*$/d' | grep gtlib.gatech.edu | wc -l + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/alter_completion_message.yaml b/tests/cloud_tests/testcases/examples/alter_completion_message.yaml new file mode 100644 index 00000000..9e154f80 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/alter_completion_message.yaml @@ -0,0 +1,16 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + final_message: | + This is my final message! + $version + $timestamp + $datasource + $uptime + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.yaml b/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.yaml new file mode 100644 index 00000000..ad32b088 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/configure_instance_trusted_ca_certificates.yaml @@ -0,0 +1,41 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + ca-certs: + # If present and set to True, the 'remove-defaults' parameter will remove + # all the default trusted CA certificates that are normally shipped with + # Ubuntu. + # This is mainly for paranoid admins - most users will not need this + # functionality. + remove-defaults: true + + # If present, the 'trusted' parameter should contain a certificate (or list + # of certificates) to add to the system as trusted CA certificates. + # Pay close attention to the YAML multiline list syntax. The example shown + # here is for a list of multiline certificates. + trusted: + - | + -----BEGIN CERTIFICATE----- + YOUR-ORGS-TRUSTED-CA-CERT-HERE + -----END CERTIFICATE----- + - | + -----BEGIN CERTIFICATE----- + YOUR-ORGS-TRUSTED-CA-CERT-HERE + -----END CERTIFICATE----- +collect_scripts: + cloudinit_certs: | + #!/bin/bash + cat /etc/ssl/certs/cloud-init-ca-certs.pem + cert_count_ca: | + #!/bin/bash + wc -l /etc/ssl/certs/ca-certificates.crt + cert_count_cloudinit: | + #!/bin/bash + wc -l /etc/ssl/certs/cloud-init-ca-certs.pem + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.yaml b/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.yaml new file mode 100644 index 00000000..f3eaf3ce --- /dev/null +++ b/tests/cloud_tests/testcases/examples/configure_instances_ssh_keys.yaml @@ -0,0 +1,63 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUUk8EEAnnkhXlukKoUPND/RRClWz2s5TCzIkd3Ou5+Cyz71X0XmazM3l5WgeErvtIwQMyT1KjNoMhoJMrJnWqQPOt5Q8zWd9qG7PBl9+eiH5qV7NZ mykey@host + - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZdQueUq5ozemNSj8T7enqKHOEaFoU2VoPgGEWC9RyzSQVeyD6s7APMcE82EtmW4skVEgEGSbDc1pvxzxtchBj78hJP6Cf5TCMFSXw+Fz5rF1dR23QDbN1mkHs7adr8GW4kSWqU7Q7NDwfIrJJtO7Hi42GyXtvEONHbiRPOe8stqUly7MvUoN+5kfjBM8Qqpfl2+FNhTYWpMfYdPUnE7u536WqzFmsaqJctz3gBxH9Ex7dFtrxR4qiqEr9Qtlu3xGn7Bw07/+i1D+ey3ONkZLN+LQ714cgj8fRS4Hj29SCmXp5Kt5/82cD/VN3NtHw== smoser@brickies + + # Send pre-generated ssh private keys to the server + # If these are present, they will be written to /etc/ssh and + # new random keys will not be generated + # in addition to 'rsa' and 'dsa' as shown below, 'ecdsa' is also supported + ssh_keys: + rsa_private: | + -----BEGIN RSA PRIVATE KEY----- + MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qcon2LZS/x + 1cydPZ4pQpfjEha6WxZ6o8ci/Ea/w0n+0HGPwaxlEG2Z9inNtj3pgFrYcRztfECb + 1j6HCibZbAzYtwIBIwJgO8h72WjcmvcpZ8OvHSvTwAguO2TkR6mPgHsgSaKy6GJo + PUJnaZRWuba/HX0KGyhz19nPzLpzG5f0fYahlMJAyc13FV7K6kMBPXTRR6FxgHEg + L0MPC7cdqAwOVNcPY6A7AjEA1bNaIjOzFN2sfZX0j7OMhQuc4zP7r80zaGc5oy6W + p58hRAncFKEvnEq2CeL3vtuZAjEAwNBHpbNsBYTRPCHM7rZuG/iBtwp8Rxhc9I5w + ixvzMgi+HpGLWzUIBS+P/XhekIjPAjA285rVmEP+DR255Ls65QbgYhJmTzIXQ2T9 + luLvcmFBC6l35Uc4gTgg4ALsmXLn71MCMGMpSWspEvuGInayTCL+vEjmNBT+FAdO + W7D4zCpI43jRS9U06JVOeSc9CDk2lwiA3wIwCTB/6uc8Cq85D9YqpM10FuHjKpnP + REPPOyrAspdeOAV+6VKRavstea7+2DZmSUgE + -----END RSA PRIVATE KEY----- + + rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7XdewmZ3h8eIXJD7TRHtVW7aJX1ByifYtlL/HVzJ09nilCl+MSFrpbFnqjxyL8Rr/DSf7QcY/BrGUQbZn2Kc22PemAWthxHO18QJvWPocKJtlsDNi3 smoser@localhost + + dsa_private: | + -----BEGIN DSA PRIVATE KEY----- + MIIBuwIBAAKBgQDP2HLu7pTExL89USyM0264RCyWX/CMLmukxX0Jdbm29ax8FBJT + pLrO8TIXVY5rPAJm1dTHnpuyJhOvU9G7M8tPUABtzSJh4GVSHlwaCfycwcpLv9TX + DgWIpSj+6EiHCyaRlB1/CBp9RiaB+10QcFbm+lapuET+/Au6vSDp9IRtlQIVAIMR + 8KucvUYbOEI+yv+5LW9u3z/BAoGBAI0q6JP+JvJmwZFaeCMMVxXUbqiSko/P1lsa + LNNBHZ5/8MOUIm8rB2FC6ziidfueJpqTMqeQmSAlEBCwnwreUnGfRrKoJpyPNENY + d15MG6N5J+z81sEcHFeprryZ+D3Ge9VjPq3Tf3NhKKwCDQ0240aPezbnjPeFm4mH + bYxxcZ9GAoGAXmLIFSQgiAPu459rCKxT46tHJtM0QfnNiEnQLbFluefZ/yiI4DI3 + 8UzTCOXLhUA7ybmZha+D/csj15Y9/BNFuO7unzVhikCQV9DTeXX46pG4s1o23JKC + /QaYWNMZ7kTRv+wWow9MhGiVdML4ZN4XnifuO5krqAybngIy66PMEoQCFEIsKKWv + 99iziAH0KBMVbxy03Trz + -----END DSA PRIVATE KEY----- + + dsa_public: ssh-dsa AAAAB3NzaC1kc3MAAACBAM/Ycu7ulMTEvz1RLIzTbrhELJZf8Iwua6TFfQl1ubb1rHwUElOkus7xMhdVjms8AmbV1Meem7ImE69T0bszy09QAG3NImHgZVIeXBoJ/JzByku/1NcOBYilKP7oSIcLJpGUHX8IGn1GJoH7XRBwVub6Vqm4RP78C7q9IOn0hG2VAAAAFQCDEfCrnL1GGzhCPsr/uS1vbt8/wQAAAIEAjSrok/4m8mbBkVp4IwxXFdRuqJKSj8/WWxos00Ednn/ww5QibysHYULrOKJ1+54mmpMyp5CZICUQELCfCt5ScZ9GsqgmnI80Q1h3Xkwbo3kn7PzWwRwcV6muvJn4PcZ71WM+rdN/c2EorAINDTbjRo97NueM94WbiYdtjHFxn0YAAACAXmLIFSQgiAPu459rCKxT46tHJtM0QfnNiEnQLbFluefZ/yiI4DI38UzTCOXLhUA7ybmZha+D/csj15Y9/BNFuO7unzVhikCQV9DTeXX46pG4s1o23JKC/QaYWNMZ7kTRv+wWow9MhGiVdML4ZN4XnifuO5krqAybngIy66PMEoQ= smoser@localhost +collect_scripts: + cert_count: | + #!/bin/bash + ls | wc -l + dsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_dsa_key.pub + rsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_rsa_key.pub + auth_keys: | + #!/bin/bash + cat /home/ubuntu/.ssh/authorized_keys + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.yaml b/tests/cloud_tests/testcases/examples/including_user_groups.yaml new file mode 100644 index 00000000..0aa7ad21 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/including_user_groups.yaml @@ -0,0 +1,53 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + # Add groups to the system + groups: + - secret: [foobar,barfoo] + - cloud-users + + # Add users to the system. Users are added after groups are added. + users: + - default + - name: foobar + gecos: Foo B. Bar + primary-group: foobar + groups: users + expiredate: 2038-01-19 + lock_passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: cloud-users + lock_passwd: true + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true +collect_scripts: + group_ubuntu: | + #!/bin/bash + getent group ubuntu + group_cloud_users: | + #!/bin/bash + getent group cloud-users + user_ubuntu: | + #!/bin/bash + getent passwd ubuntu + user_foobar: | + #!/bin/bash + getent passwd foobar + user_barfoo: | + #!/bin/bash + getent passwd barfoo + user_cloudy: | + #!/bin/bash + getent passwd cloudy + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/install_arbitrary_packages.yaml b/tests/cloud_tests/testcases/examples/install_arbitrary_packages.yaml new file mode 100644 index 00000000..d3980228 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/install_arbitrary_packages.yaml @@ -0,0 +1,20 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + packages: + - htop + - tree +collect_scripts: + htop: | + #!/bin/bash + dpkg -l | grep htop | wc -l + tree: | + #!/bin/bash + dpkg -l | grep tree | wc -l + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml new file mode 100644 index 00000000..0bec305e --- /dev/null +++ b/tests/cloud_tests/testcases/examples/install_run_chef_recipes.yaml @@ -0,0 +1,103 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2017-03-31: Disabled as depends on third party apt repository +# +enabled: False +cloud_config: | + #cloud-config + # Key from https://packages.chef.io/chef.asc + apt: + source1: + source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.12 (Darwin) + Comment: GPGTools - http://gpgtools.org + + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu0IENIRUYgUGFja2FnZXMg + PHBhY2thZ2VzQGNoZWYuaW8+iGIEExECACIFAlQwYFECGwMGCwkIBwMCBhUIAgkK + CwQWAgMBAh4BAheAAAoJEClAq6mD74JqX94An26z99XOHWpLN8ahzm7cp13t4Xid + AJ9wVcgoUBzvgg91lKfv/34cmemZn7kCDQRKaQu0EAgAg7ZLCVGVTmLqBM6njZEd + Zbv+mZbvwLBSomdiqddE6u3eH0X3GuwaQfQWHUVG2yedyDMiG+EMtCdEeeRebTCz + SNXQ8Xvi22hRPoEsBSwWLZI8/XNg0n0f1+GEr+mOKO0BxDB2DG7DA0nnEISxwFkK + OFJFebR3fRsrWjj0KjDxkhse2ddU/jVz1BY7Nf8toZmwpBmdozETMOTx3LJy1HZ/ + Te9FJXJMUaB2lRyluv15MVWCKQJro4MQG/7QGcIfrIZNfAGJ32DDSjV7/YO+IpRY + IL4CUBQ65suY4gYUG4jhRH6u7H1p99sdwsg5OIpBe/v2Vbc/tbwAB+eJJAp89Zeu + twADBQf/ZcGoPhTGFuzbkcNRSIz+boaeWPoSxK2DyfScyCAuG41CY9+g0HIw9Sq8 + DuxQvJ+vrEJjNvNE3EAEdKl/zkXMZDb1EXjGwDi845TxEMhhD1dDw2qpHqnJ2mtE + WpZ7juGwA3sGhi6FapO04tIGacCfNNHmlRGipyq5ZiKIRq9mLEndlECr8cwaKgkS + 0wWu+xmMZe7N5/t/TK19HXNh4tVacv0F3fYK54GUjt2FjCQV75USnmNY4KPTYLXA + dzC364hEMlXpN21siIFgB04w+TXn5UF3B4FfAy5hevvr4DtV4MvMiGLu0oWjpaLC + MpmrR3Ny2wkmO0h+vgri9uIP06ODWIhJBBgRAgAJBQJKaQu0AhsMAAoJEClAq6mD + 74Jq4hIAoJ5KrYS8kCwj26SAGzglwggpvt3CAJ0bekyky56vNqoegB+y4PQVDv4K + zA== + =IxPr + -----END PGP PUBLIC KEY BLOCK----- + + chef: + + # Valid values are 'gems' and 'packages' and 'omnibus' + install_type: "packages" + + # Boolean: run 'install_type' code even if chef-client + # appears already installed. + force_install: false + + # Chef settings + server_url: "https://chef.yourorg.com:4000" + + # Node Name + # Defaults to the instance-id if not present + node_name: "your-node-name" + + # Environment + # Defaults to '_default' if not present + environment: "production" + + # Default validation name is chef-validator + validation_name: "yourorg-validator" + # if validation_cert's value is "system" then it is expected + # that the file already exists on the system. + validation_cert: | + -----BEGIN RSA PRIVATE KEY----- + YOUR-ORGS-VALIDATION-KEY-HERE + -----END RSA PRIVATE KEY----- + + # A run list for a first boot json + run_list: + - "recipe[apache2]" + - "role[db]" + + # Specify a list of initial attributes used by the cookbooks + initial_attributes: + apache: + prefork: + maxclients: 100 + keepalive: "off" + + # if install_type is 'omnibus', change the url to download + omnibus_url: "https://www.opscode.com/chef/install.sh" + + + # Capture all subprocess output into a logfile + # Useful for troubleshooting cloud-init issues + output: {all: '| tee -a /var/log/cloud-init-output.log'} + +collect_scripts: + chef_installed: | + #!/bin/sh + dpkg-query -W -f '${Status}\n' chef + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/run_apt_upgrade.yaml b/tests/cloud_tests/testcases/examples/run_apt_upgrade.yaml new file mode 100644 index 00000000..2b7eae4c --- /dev/null +++ b/tests/cloud_tests/testcases/examples/run_apt_upgrade.yaml @@ -0,0 +1,11 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + package_upgrade: true + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/run_commands.yaml b/tests/cloud_tests/testcases/examples/run_commands.yaml new file mode 100644 index 00000000..b0e311ba --- /dev/null +++ b/tests/cloud_tests/testcases/examples/run_commands.yaml @@ -0,0 +1,16 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + runcmd: + - echo cloud-init run cmd test > /tmp/run_cmd +collect_scripts: + run_cmd: | + #!/bin/bash + cat /tmp/run_cmd + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/run_commands_first_boot.yaml b/tests/cloud_tests/testcases/examples/run_commands_first_boot.yaml new file mode 100644 index 00000000..7bd803db --- /dev/null +++ b/tests/cloud_tests/testcases/examples/run_commands_first_boot.yaml @@ -0,0 +1,16 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + bootcmd: + - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts +collect_scripts: + hosts: | + #!/bin/bash + cat /etc/hosts + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/setup_run_puppet.yaml b/tests/cloud_tests/testcases/examples/setup_run_puppet.yaml new file mode 100644 index 00000000..e366c042 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/setup_run_puppet.yaml @@ -0,0 +1,55 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as test suite fails this long running test currently +# +enabled: False +cloud_config: | + #cloud-config + puppet: + # Every key present in the conf object will be added to puppet.conf: + # [name] + # subkey=value + # + # For example the configuration below will have the following section + # added to puppet.conf: + # [puppetd] + # server=puppetmaster.example.org + # certname=i-0123456.ip-X-Y-Z.cloud.internal + # + # The puppmaster ca certificate will be available in + # /var/lib/puppet/ssl/certs/ca.pem + conf: + agent: + server: "puppetmaster.example.org" + # certname supports substitutions at runtime: + # %i: instanceid + # Example: i-0123456 + # %f: fqdn of the machine + # Example: ip-X-Y-Z.cloud.internal + # + # NB: the certname will automatically be lowercased as required by puppet + certname: "%i.%f" + # ca_cert is a special case. It won't be added to puppet.conf. + # It holds the puppetmaster certificate in pem format. + # It should be a multi-line string (using the | yaml notation for + # multi-line strings). + # The puppetmaster certificate is located in + # /var/lib/puppet/ssl/ca/ca_crt.pem on the puppetmaster host. + # + ca_cert: | + -----BEGIN CERTIFICATE----- + MIICCTCCAXKgAwIBAgIBATANBgkqhkiG9w0BAQUFADANMQswCQYDVQQDDAJjYTAe + Fw0xMDAyMTUxNzI5MjFaFw0xNTAyMTQxNzI5MjFaMA0xCzAJBgNVBAMMAmNhMIGf + MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCu7Q40sm47/E1Pf+r8AYb/V/FWGPgc + b014OmNoX7dgCxTDvps/h8Vw555PdAFsW5+QhsGr31IJNI3kSYprFQcYf7A8tNWu + 1MASW2CfaEiOEi9F1R3R4Qlz4ix+iNoHiUDTjazw/tZwEdxaQXQVLwgTGRwVa+aA + qbutJKi93MILLwIDAQABo3kwdzA4BglghkgBhvhCAQ0EKxYpUHVwcGV0IFJ1Ynkv + T3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwDwYDVR0TAQH/BAUwAwEB/zAd + BgNVHQ4EFgQUu4+jHB+GYE5Vxo+ol1OAhevspjAwCwYDVR0PBAQDAgEGMA0GCSqG + SIb3DQEBBQUAA4GBAH/rxlUIjwNb3n7TXJcDJ6MMHUlwjr03BDJXKb34Ulndkpaf + +GAlzPXWa7bO908M9I8RnPfvtKnteLbvgTK+h+zX1XCty+S2EQWk29i2AdoqOTxb + hppiGMp0tT5Havu4aceCXiy2crVcudj3NFciy8X66SoECemW9UYDCb9T5D0d + -----END CERTIFICATE----- + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.yaml b/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.yaml new file mode 100644 index 00000000..6f78f994 --- /dev/null +++ b/tests/cloud_tests/testcases/examples/writing_out_arbitrary_files.yaml @@ -0,0 +1,45 @@ +# +# From cloud config examples on cloudinit.readthedocs.io +# +# 2016-11-17: Disabled as covered by module based tests +# +enabled: False +cloud_config: | + #cloud-config + write_files: + - encoding: b64 + content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4 + owner: root:root + path: /root/file_b64 + permissions: '0644' + - content: | + # My new /root/file_text + + SMBDOPTIONS="-D" + path: /root/file_text + - content: !!binary | + f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI + AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA + AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA + path: /root/file_binary + permissions: '0555' + - encoding: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /root/file_gzip + permissions: '0755' +collect_scripts: + file_b64: | + #!/bin/bash + file /root/file_b64 + file_text: | + #!/bin/bash + file /root/file_text + file_binary: | + #!/bin/bash + file /root/file_binary + file_gzip: | + #!/bin/bash + file /root/file_gzip + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/main/README.md b/tests/cloud_tests/testcases/main/README.md new file mode 100644 index 00000000..60346063 --- /dev/null +++ b/tests/cloud_tests/testcases/main/README.md @@ -0,0 +1,11 @@ +# Main Functionality Test Configs + +## purpose +Test main features and config options of cloud-init such as logging, output +redirection, early init and integration with init system + +## structure +Should have one or more test configs for all main cloud-init output and logging +options, and basic functionality test cases + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/main/command_output_simple.yaml b/tests/cloud_tests/testcases/main/command_output_simple.yaml new file mode 100644 index 00000000..08ca8940 --- /dev/null +++ b/tests/cloud_tests/testcases/main/command_output_simple.yaml @@ -0,0 +1,13 @@ +# +# Test functionality of simple output redirection +# +cloud_config: | + #cloud-config + output: { all: "| tee -a /var/log/cloud-init-test-output" } + final_message: "should be last line in cloud-init-test-output file" +collect_scripts: + cloud-init-test-output: | + #!/bin/bash + cat /var/log/cloud-init-test-output + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/README.md b/tests/cloud_tests/testcases/modules/README.md new file mode 100644 index 00000000..d66101f2 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/README.md @@ -0,0 +1,12 @@ +# Module Test Configs + +## Purpose +Test functionality of cloud config modules. See +[here](https://cloudinit.readthedocs.io/en/latest/topics/modules.html) for +a full list. + +## Structure +Should have one or more test configs for each module in cloudinit/config/. The +name of the test should indicate which module the config is verifying. + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/TODO.md b/tests/cloud_tests/testcases/modules/TODO.md new file mode 100644 index 00000000..0b933b3b --- /dev/null +++ b/tests/cloud_tests/testcases/modules/TODO.md @@ -0,0 +1,98 @@ +# TODO + +The following lists complete or partially misisng modules. If a module is +listed with nothing below it indicates that no work is completed on that +module. If there is a list below the module name that is the remainig +identified work. + +## apt_configure + + * apt_get_wrapper + * What does this do? How to use it? + * apt_get_command + * To specify a different 'apt-get' command, set 'apt_get_command'. + This must be a list, and the subcommand (update, upgrade) is appended to it. + * Modify default and verify the options got passed correctly. + * preserve sources + * TBD + +## chef +2016-11-17: Tests took > 60 seconds and test framework times out currently. + +## disable EC2 metadata + +## disk setup + +## emit upstart + +## fan + +## growpart + +## grub dpkg + +## landscape +2016-11-17: Module is not working + +## lxd +2016-11-17: Need a zfs backed test written + +## mcollective + +## migrator + +## mounts + +## phone home + +## power state change + +## puppet +2016-11-17: Tests took > 60 seconds and test framework times out currently. + +## resizefs + +## resolv conf +2016-11-17: Issues with changing resolv.conf and lxc backend. + +## redhat subscription +2016-11-17: Need RH support in test framework. + +## rightscale userdata +2016-11-17: Specific to RightScale cloud enviornment. + +## rsyslog + +## scripts per boot +Not applicable to write a test for this as it specifies when something should be run. + +## scripts per instance +Not applicable to write a test for this as it specifies when something should be run. + +## scripts per once +Not applicable to write a test for this as it specifies when something should be run. + +## scripts user +Not applicable to write a test for this as it specifies when something should be run. + +## scripts vendor +Not applicable to write a test for this as it specifies when something should be run. + +## snappy +2016-11-17: Need test to install snaps from store + +## snap-config +2016-11-17: Need to investigate + +## spacewalk + +## ssh authkey fingerprints +The authkey_hash key does not appear to work. In fact the default claims to be md5, however syslog only shows sha256 + +## update etc hosts +2016-11-17: Issues with changing /etc/hosts and lxc backend. + +## yum add repo +2016-11-17: Need RH support in test framework. + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_conf.yaml b/tests/cloud_tests/testcases/modules/apt_configure_conf.yaml new file mode 100644 index 00000000..de453000 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_conf.yaml @@ -0,0 +1,21 @@ +# +# Provide a configuration for APT +# +required_features: + - apt +cloud_config: | + #cloud-config + apt: + conf: | + APT { + Get { + Assume-Yes "true"; + Fix-Broken "true"; + } + } +collect_scripts: + 94cloud-init-config: | + #!/bin/bash + cat /etc/apt/apt.conf.d/94cloud-init-config + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.yaml b/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.yaml new file mode 100644 index 00000000..98800673 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_disable_suites.yaml @@ -0,0 +1,20 @@ +# +# Disables everything in sources.list +# +required_features: + - apt + - lsb_release +cloud_config: | + #cloud-config + apt: + disable_suites: + - $RELEASE + - $RELEASE-updates + - $RELEASE-backports + - $RELEASE-security +collect_scripts: + sources.list: | + #!/bin/bash + grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_primary.yaml b/tests/cloud_tests/testcases/modules/apt_configure_primary.yaml new file mode 100644 index 00000000..41bcf2fd --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_primary.yaml @@ -0,0 +1,26 @@ +# +# Setup a custome primary sources.list +# +required_features: + - apt + - apt_src_cont +cloud_config: | + #cloud-config + apt: + primary: + - arches: + - default + uri: "http://www.gtlib.gatech.edu/pub/ubuntu-releases/" +collect_scripts: + ubuntu.sources.list: | + #!/bin/bash + grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' | grep -c archive.ubuntu.com + gatech.sources.list: | + #!/bin/bash + grep -v '^#' /etc/apt/sources.list | sed '/^\s*$/d' | grep -c gtlib.gatech.edu + + sources.list: | + #!/bin/bash + cat /etc/apt/sources.list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_proxy.yaml b/tests/cloud_tests/testcases/modules/apt_configure_proxy.yaml new file mode 100644 index 00000000..be6c6f81 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_proxy.yaml @@ -0,0 +1,18 @@ +# +# Set apt proxy +# +required_features: + - apt +cloud_config: | + #cloud-config + apt: + proxy: "http://squid.internal:3128" + http_proxy: "http://squid.internal:3128" + ftp_proxy: "ftp://squid.internal:3128" + https_proxy: "https://squid.internal:3128" +collect_scripts: + 90cloud-init-aptproxy: | + #!/bin/bash + cat /etc/apt/apt.conf.d/90cloud-init-aptproxy + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_security.yaml b/tests/cloud_tests/testcases/modules/apt_configure_security.yaml new file mode 100644 index 00000000..83dd51df --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_security.yaml @@ -0,0 +1,18 @@ +# +# Add security to sources.list +# +required_features: + - apt + - ubuntu_repos +cloud_config: | + #cloud-config + apt: + security: + - arches: + - default +collect_scripts: + sources.list: | + #!/bin/bash + grep -c security.ubuntu.com /etc/apt/sources.list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_key.yaml b/tests/cloud_tests/testcases/modules/apt_configure_sources_key.yaml new file mode 100644 index 00000000..bde9398a --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_key.yaml @@ -0,0 +1,50 @@ +# +# Add a sources.list entry with a given key (Debian Jessie) +# +required_features: + - apt + - lsb_release +cloud_config: | + #cloud-config + apt: + sources: + source1: + source: "deb http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu $RELEASE main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.1.6 + Comment: Hostname: keyserver.ubuntu.com + + mQINBFbZRUIBEAC+A0PIKYBP9kLC4hQtRrffRS11uLo8/BdtmOdrlW0hpPHzCfKnjR3tvSEI + lqPHG1QrrjAXKZDnZMRz+h/px7lUztvytGzHPSJd5ARUzAyjyRezUhoJ3VSCxrPqx62avuWf + RfoJaIeHfDehL5/dTVkyiWxfVZ369ZX6JN2AgLsQTeybTQ75+2z0xPrrhnGmgh6g0qTYcAaq + M5ONOGiqeSBX/Smjh6ALy5XkhUiFGLsI7Yluf6XSICY/x7gd6RAfgSIQrUTNMoS1sqhT4aot + +xvOfQy8ySkfAK4NddXql6E/+ZqTmBY/Lr0YklFBy8jGT+UysfiIznPMIwbmgq5Li7BtDDtX + b8Uyi4edPpjtextezfXYn4NVIpPL5dPZS/FXh4HpzyH0pYCfrH4QDGA7i52AGmhpiOFjJMo6 + N33sdjZHOH/2Vyp+QZaQnsdUAi1N4M6c33tQbpIScn1SY+El8z5JDA4PBzkw8HpLCi1gGoa6 + V4kfbWqXXbGAJFkLkP/vc4+pY9axOlmCkJg7xCPwhI75y1cONgovhz+BEXOzolh5KZuGbGbj + xe0wva5DLBeIg7EQFf+99pOS7Syby3Xpm6ZbswEFV0cllK4jf/QMjtfInxobuMoI0GV0bE5l + WlRtPCK5FnbHwxi0wPNzB/5fwzJ77r6HgPrR0OkT0lWmbUyoOQARAQABtC1MYXVuY2hwYWQg + UFBBIGZvciBjbG91ZCBpbml0IGRldmVsb3BtZW50IHRlYW2JAjgEEwECACIFAlbZRUICGwMG + CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEAg9Bvvk0wTfHfcP/REK5N2s1JYc69qEa9ZN + o6oi+A7l6AYw+ZY88O5TJe7F9otv5VXCIKSUT0Vsepjgf0mtXAgf/sb2lsJn/jp7tzgov3YH + vSrkTkRydz8xcA87gwQKePuvTLxQpftF4flrBxgSueIn5O/tPrBOxLz7EVYBc78SKg9aj9L2 + yUp+YuNevlwfZCTYeBb9r3FHaab2HcgkwqYch66+nKYfwiLuQ9NzXXm0Wn0JcEQ6pWvJscbj + C9BdawWovfvMK5/YLfI6Btm7F4mIpQBdhSOUp/YXKmdvHpmwxMCN2QhqYK49SM7qE9aUDbJL + arppSEBtlCLWhRBZYLTUna+BkuQ1bHz4St++XTR49Qd7vDERALpApDjB2dxPfMiBzCMwQQyq + uy13exU8o2ETLg+dZSLfDTzrBNsBFmXlw8WW17nTISYdKeGKL+QdlUjpzdwUMMzHhAO8SmMH + zjeSlDSRMXBJFAFSbCl7EwmMKa3yVX0zInT91fNllZ3iatAmtVdqVH/BFQfTIMH2ET7A8WzJ + ZzVSuMRhqoKdr5AMcHuJGPUoVkVJHQA+NNvEiXSysF3faL7jmKapmUwrhpYYX2H8pf+VMu2e + cLflKTI28dl+ZQ4Pl/aVsxrti/pzhdYy05Sn5ddtySyIkvo8L1cU5MWpbvSlFPkTstBUDLBf + pb0uBy+g0oxJQg15 + =uy53 + -----END PGP PUBLIC KEY BLOCK----- +collect_scripts: + sources.list: | + #!/bin/bash + cat /etc/apt/sources.list.d/source1.list + apt_key_list: | + #!/bin/bash + apt-key finger + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.yaml b/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.yaml new file mode 100644 index 00000000..25088135 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_keyserver.yaml @@ -0,0 +1,23 @@ +# +# Add a sources.list entry with a key from a keyserver +# +required_features: + - apt + - lsb_release +cloud_config: | + #cloud-config + apt: + sources: + source1: + keyid: 1FF0D8535EF7E719E5C81B9C083D06FBE4D304DF + keyserver: keyserver.ubuntu.com + source: "deb http://ppa.launchpad.net/cloud-init-dev/test-archive/ubuntu $RELEASE main" +collect_scripts: + sources.list: | + #!/bin/bash + cat /etc/apt/sources.list.d/source1.list + apt_key_list: | + #!/bin/bash + apt-key finger + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml new file mode 100644 index 00000000..143cb080 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_list.yaml @@ -0,0 +1,22 @@ +# +# Generate a sources.list +# +required_features: + - apt + - lsb_release +cloud_config: | + #cloud-config + apt: + sources_list: | + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb-src $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + deb-src $SECURITY $RELEASE-security multiverse +collect_scripts: + sources.list: | + #/bin/bash + cat /etc/apt/sources.list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml new file mode 100644 index 00000000..9efdae52 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_configure_sources_ppa.yaml @@ -0,0 +1,29 @@ +# +# Add a PPA to source.list +# +# NOTE: on older ubuntu releases the sources file added is named +# 'curtin-dev-test-archive-trusty', without 'ubuntu' in the middle +required_features: + - apt + - ppa + - ppa_file_name +cloud_config: | + #cloud-config + apt: + sources: + source1: + keyid: 0165013E + keyserver: keyserver.ubuntu.com + source: "ppa:curtin-dev/test-archive" +collect_scripts: + sources.list: | + #!/bin/bash + cat /etc/apt/sources.list.d/curtin-dev-ubuntu-test-archive-*.list + apt-key: | + #!/bin/bash + apt-key finger + sources_full: | + #!/bin/bash + cat /etc/apt/sources.list + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml b/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml new file mode 100644 index 00000000..bd9b5d08 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_disable.yaml @@ -0,0 +1,15 @@ +# +# Disable apt pipelining value +# +required_features: + - apt +cloud_config: | + #cloud-config + apt: + apt_pipelining: false +collect_scripts: + 90cloud-init-pipelining: | + #!/bin/bash + cat /etc/apt/apt.conf.d/90cloud-init-pipelining + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml b/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml new file mode 100644 index 00000000..cbed3ba3 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/apt_pipelining_os.yaml @@ -0,0 +1,15 @@ +# +# Set apt pipelining value to OS +# +required_features: + - apt +cloud_config: | + #cloud-config + apt: + apt_pipelining: os +collect_scripts: + 90cloud-init-pipelining: | + #!/bin/bash + cat /etc/apt/apt.conf.d/90cloud-init-pipelining + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/bootcmd.yaml b/tests/cloud_tests/testcases/modules/bootcmd.yaml new file mode 100644 index 00000000..3a73994e --- /dev/null +++ b/tests/cloud_tests/testcases/modules/bootcmd.yaml @@ -0,0 +1,13 @@ +# +# Early boot command +# +cloud_config: | + #cloud-config + bootcmd: + - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts +collect_scripts: + hosts: | + #!/bin/bash + cat /etc/hosts + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/byobu.yaml b/tests/cloud_tests/testcases/modules/byobu.yaml new file mode 100644 index 00000000..a9aa1f3f --- /dev/null +++ b/tests/cloud_tests/testcases/modules/byobu.yaml @@ -0,0 +1,20 @@ +# +# Install and enable byobu system wide and default user +# +required_features: + - byobu +cloud_config: | + #cloud-config + byobu_by_default: enable +collect_scripts: + byobu_installed: | + #!/bin/bash + which byobu + byobu_profile_enabled: | + #!/bin/bash + ls /etc/profile.d/Z97-byobu.sh + byobu_launch_exists: | + #!/bin/bash + which /usr/bin/byobu-launch + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ca_certs.yaml b/tests/cloud_tests/testcases/modules/ca_certs.yaml new file mode 100644 index 00000000..d939f435 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ca_certs.yaml @@ -0,0 +1,52 @@ +# +# Remove existing ca_certs and install custom ca-cert +# +cloud_config: | + #cloud-config + ca-certs: + remove-defaults: true + trusted: + - | + -----BEGIN CERTIFICATE----- + MIIGJzCCBA+gAwIBAgIBATANBgkqhkiG9w0BAQUFADCBsjELMAkGA1UEBhMCRlIx + DzANBgNVBAgMBkFsc2FjZTETMBEGA1UEBwwKU3RyYXNib3VyZzEYMBYGA1UECgwP + d3d3LmZyZWVsYW4ub3JnMRAwDgYDVQQLDAdmcmVlbGFuMS0wKwYDVQQDDCRGcmVl + bGFuIFNhbXBsZSBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxIjAgBgkqhkiG9w0BCQEW + E2NvbnRhY3RAZnJlZWxhbi5vcmcwHhcNMTIwNDI3MTAzMTE4WhcNMjIwNDI1MTAz + MTE4WjB+MQswCQYDVQQGEwJGUjEPMA0GA1UECAwGQWxzYWNlMRgwFgYDVQQKDA93 + d3cuZnJlZWxhbi5vcmcxEDAOBgNVBAsMB2ZyZWVsYW4xDjAMBgNVBAMMBWFsaWNl + MSIwIAYJKoZIhvcNAQkBFhNjb250YWN0QGZyZWVsYW4ub3JnMIICIjANBgkqhkiG + 9w0BAQEFAAOCAg8AMIICCgKCAgEA3W29+ID6194bH6ejLrIC4hb2Ugo8v6ZC+Mrc + k2dNYMNPjcOKABvxxEtBamnSaeU/IY7FC/giN622LEtV/3oDcrua0+yWuVafyxmZ + yTKUb4/GUgafRQPf/eiX9urWurtIK7XgNGFNUjYPq4dSJQPPhwCHE/LKAykWnZBX + RrX0Dq4XyApNku0IpjIjEXH+8ixE12wH8wt7DEvdO7T3N3CfUbaITl1qBX+Nm2Z6 + q4Ag/u5rl8NJfXg71ZmXA3XOj7zFvpyapRIZcPmkvZYn7SMCp8dXyXHPdpSiIWL2 + uB3KiO4JrUYvt2GzLBUThp+lNSZaZ/Q3yOaAAUkOx+1h08285Pi+P8lO+H2Xic4S + vMq1xtLg2bNoPC5KnbRfuFPuUD2/3dSiiragJ6uYDLOyWJDivKGt/72OVTEPAL9o + 6T2pGZrwbQuiFGrGTMZOvWMSpQtNl+tCCXlT4mWqJDRwuMGrI4DnnGzt3IKqNwS4 + Qyo9KqjMIPwnXZAmWPm3FOKe4sFwc5fpawKO01JZewDsYTDxVj+cwXwFxbE2yBiF + z2FAHwfopwaH35p3C6lkcgP2k/zgAlnBluzACUI+MKJ/G0gv/uAhj1OHJQ3L6kn1 + SpvQ41/ueBjlunExqQSYD7GtZ1Kg8uOcq2r+WISE3Qc9MpQFFkUVllmgWGwYDuN3 + Zsez95kCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNT + TCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFFlfyRO6G8y5qEFKikl5 + ajb2fT7XMB8GA1UdIwQYMBaAFCNsLT0+KV14uGw+quK7Lh5sh/JTMA0GCSqGSIb3 + DQEBBQUAA4ICAQAT5wJFPqervbja5+90iKxi1d0QVtVGB+z6aoAMuWK+qgi0vgvr + mu9ot2lvTSCSnRhjeiP0SIdqFMORmBtOCFk/kYDp9M/91b+vS+S9eAlxrNCB5VOf + PqxEPp/wv1rBcE4GBO/c6HcFon3F+oBYCsUQbZDKSSZxhDm3mj7pb67FNbZbJIzJ + 70HDsRe2O04oiTx+h6g6pW3cOQMgIAvFgKN5Ex727K4230B0NIdGkzuj4KSML0NM + slSAcXZ41OoSKNjy44BVEZv0ZdxTDrRM4EwJtNyggFzmtTuV02nkUj1bYYYC5f0L + ADr6s0XMyaNk8twlWYlYDZ5uKDpVRVBfiGcq0uJIzIvemhuTrofh8pBQQNkPRDFT + Rq1iTo1Ihhl3/Fl1kXk1WR3jTjNb4jHX7lIoXwpwp767HAPKGhjQ9cFbnHMEtkro + RlJYdtRq5mccDtwT0GFyoJLLBZdHHMHJz0F9H7FNk2tTQQMhK5MVYwg+LIaee586 + CQVqfbscp7evlgjLW98H+5zylRHAgoH2G79aHljNKMp9BOuq6SnEglEsiWGVtu2l + hnx8SB3sVJZHeer8f/UQQwqbAO+Kdy70NmbSaqaVtp8jOxLiidWkwSyRTsuU6D8i + DiH5uEqBXExjrj0FslxcVKdVj5glVcSmkLwZKbEU1OKwleT/iXFhvooWhQ== + -----END CERTIFICATE----- +collect_scripts: + cert_count: | + #!/bin/bash + ls -l /etc/ssl/certs | wc -l + cert: | + #!/bin/bash + md5sum /etc/ssl/certs/ca-certificates.crt +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/debug_disable.yaml b/tests/cloud_tests/testcases/modules/debug_disable.yaml new file mode 100644 index 00000000..63218b18 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/debug_disable.yaml @@ -0,0 +1,9 @@ +# +# Do not run in debug mode +# +cloud_config: | + #cloud-config + debug: + verbose: False + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/debug_enable.yaml b/tests/cloud_tests/testcases/modules/debug_enable.yaml new file mode 100644 index 00000000..d44147db --- /dev/null +++ b/tests/cloud_tests/testcases/modules/debug_enable.yaml @@ -0,0 +1,9 @@ +# +# Run in debug mode +# +cloud_config: | + #cloud-config + debug: + verbose: True + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/final_message.yaml b/tests/cloud_tests/testcases/modules/final_message.yaml new file mode 100644 index 00000000..c9ed6118 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/final_message.yaml @@ -0,0 +1,13 @@ +# +# Print a final message with various predefined variables +# +cloud_config: | + #cloud-config + final_message: | + This is my final message! + $version + $timestamp + $datasource + $uptime + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/keys_to_console.yaml b/tests/cloud_tests/testcases/modules/keys_to_console.yaml new file mode 100644 index 00000000..5d86e739 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/keys_to_console.yaml @@ -0,0 +1,15 @@ +# +# Hide printing of ssh key and fingerprints for specific keys +# +required_features: + - syslog +cloud_config: | + #cloud-config + ssh_fp_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] + ssh_key_console_blacklist: [ssh-dss, ssh-dsa, ecdsa-sha2-nistp256] +collect_scripts: + syslog: | + #!/bin/bash + cat /var/log/syslog + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/landscape.yaml b/tests/cloud_tests/testcases/modules/landscape.yaml new file mode 100644 index 00000000..ed2c37c4 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/landscape.yaml @@ -0,0 +1,28 @@ +# +# Setup landscape client settings +# +# 2016-11-17: Disabled due to this not working +# +enabled: false +required_features: + - landscape +cloud_config: | + #cloud-conifg + landscape: + client: + log_level: "info" + url: "https://landscape.canonical.com/message-system" + ping_url: "http://landscape.canonical.com/ping" + data_path: "/var/lib/landscape/client" + http_proxy: "http://my.proxy.com/foobar" + https_proxy: "https://my.proxy.com/foobar" + tags: "server,cloud" + computer_title: "footitle" + registration_key: "fookey" + account_name: "fooaccount" +collect_scripts: + client.conf: | + #!/bin/bash + cat /etc/landscape/client.conf + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/locale.yaml b/tests/cloud_tests/testcases/modules/locale.yaml new file mode 100644 index 00000000..e01518a1 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/locale.yaml @@ -0,0 +1,22 @@ +# +# Set locale to non-default option and verify +# +required_features: + - engb_locale + - locale_gen +cloud_config: | + #cloud-config + locale: en_GB.UTF-8 + locale_configfile: /etc/default/locale +collect_scripts: + locale_default: | + #!/bin/bash + cat /etc/default/locale + locale_a: | + #!/bin/bash + locale -a + locale_gen: | + #!/bin/bash + cat /etc/locale.gen | grep -v '^#' | uniq + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.yaml b/tests/cloud_tests/testcases/modules/lxd_bridge.yaml new file mode 100644 index 00000000..e6b7e76a --- /dev/null +++ b/tests/cloud_tests/testcases/modules/lxd_bridge.yaml @@ -0,0 +1,32 @@ +# +# LXD configured with directory backend and IPv4 bridge +# +required_features: + - lxd +cloud_config: | + #cloud-config + lxd: + init: + storage_backend: dir + bridge: + mode: new + name: lxdbr0 + ipv4_address: 10.100.100.1 + ipv4_netmask: 24 + ipv4_dhcp_first: 10.100.100.100 + ipv4_dhcp_last: 10.100.100.200 + ipv4_nat: true + domain: lxd +collect_scripts: + lxc: | + #!/bin/bash + which lxc + lxd: | + #!/bin/bash + which lxd + lxc-bridge: | + #!/bin/bash + ip addr show lxdbr0 + cat /etc/default/lxd-bridge 2>/dev/null | grep -v ^# | sort -u + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.yaml b/tests/cloud_tests/testcases/modules/lxd_dir.yaml new file mode 100644 index 00000000..f93a3fa7 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/lxd_dir.yaml @@ -0,0 +1,19 @@ +# +# LXD configured with directory backend +# +required_features: + - lxd +cloud_config: | + #cloud-config + lxd: + init: + storage_backend: dir +collect_scripts: + lxc: | + #!/bin/bash + which lxc + lxd: | + #!/bin/bash + which lxd + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml new file mode 100644 index 00000000..fbef431b --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp.yaml @@ -0,0 +1,21 @@ +# +# Emtpy NTP config to setup using defaults +# +cloud_config: | + #cloud-config + ntp: + pools: {} + servers: {} +collect_scripts: + ntp_installed: | + #!/bin/bash + ntpd --version > /dev/null 2>&1 + echo $? + ntp_conf_dist_empty: | + #!/bin/bash + ls /etc/ntp.conf.dist | wc -l + ntp_conf_pool_list: | + #!/bin/bash + grep 'pool.ntp.org' /etc/ntp.conf | grep -v ^# + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.yaml b/tests/cloud_tests/testcases/modules/ntp_pools.yaml new file mode 100644 index 00000000..3a93faa2 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_pools.yaml @@ -0,0 +1,31 @@ +# +# NTP config using specific pools +# +# NOTE: lsb_release listed here because with recent cloud-init deb with +# (LP: 1628337) resolved, cloud-init will attempt to configure archives. +# this fails without lsb_release as UNAVAILABLE is used for $RELEASE +required_features: + - lsb_release +cloud_config: | + #cloud-config + ntp: + pools: + - 0.cloud-init.mypool + - 1.cloud-init.mypool + - 172.16.15.14 +collect_scripts: + ntp_installed_pools: | + #!/bin/bash + ntpd --version > /dev/null 2>&1 + echo $? + ntp_conf_dist_pools: | + #!/bin/bash + ls /etc/ntp.conf.dist | wc -l + ntp_conf_pools: | + #!/bin/bash + grep '^pool' /etc/ntp.conf + ntpq_servers: | + #!/bin/sh + ntpq -p -w + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_servers.yaml b/tests/cloud_tests/testcases/modules/ntp_servers.yaml new file mode 100644 index 00000000..d59d45a8 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_servers.yaml @@ -0,0 +1,27 @@ +# +# NTP config using specific servers +# +required_features: + - lsb_release +cloud_config: | + #cloud-config + ntp: + servers: + - 172.16.15.14 + - 172.16.17.18 +collect_scripts: + ntp_installed_servers: | + #!/bin/sh + ntpd --version > /dev/null 2>&1 + echo $? + ntp_conf_dist_servers: | + #!/bin/sh + cat /etc/ntp.conf.dist | wc -l + ntp_conf_servers: | + #!/bin/sh + grep '^server' /etc/ntp.conf + ntpq_servers: | + #!/bin/sh + ntpq -p -w + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml new file mode 100644 index 00000000..71d24b83 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml @@ -0,0 +1,33 @@ +# +# Update/upgrade via apt and then install a pair of packages +# +# NOTE: this should not require apt feature, use 'which' rather than 'dpkg -l' +# NOTE: the testcase for this looks for the command in history.log as +# /usr/bin/apt-get..., which is not how it always appears. it should +# instead look for just apt-get... +# NOTE: this testcase should not require 'apt_up_out', and should look for a +# call to 'apt-get upgrade' or 'apt-get dist-upgrade' in cloud-init.log +# rather than 'Calculating upgrade...' in output +required_features: + - apt + - apt_hist_fmt + - apt_up_out +cloud_config: | + #cloud-config + packages: + - htop + - tree + package_update: true + package_upgrade: true +collect_scripts: + apt_history_cmdline: | + #!/bin/bash + grep ^Commandline: /var/log/apt/history.log + dpkg_htop: | + #!/bin/bash + dpkg -l | grep htop | wc -l + dpkg_tree: | + #!/bin/bash + dpkg -l | grep tree | wc -l + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/runcmd.yaml b/tests/cloud_tests/testcases/modules/runcmd.yaml new file mode 100644 index 00000000..04e5a050 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/runcmd.yaml @@ -0,0 +1,13 @@ +# +# Run a simple command +# +cloud_config: | + #cloud-config + runcmd: + - echo cloud-init run cmd test > /tmp/run_cmd +collect_scripts: + run_cmd: | + #!/bin/bash + cat /tmp/run_cmd + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml new file mode 100644 index 00000000..f20d24f0 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml @@ -0,0 +1,34 @@ +# +# Create config for a salt minion +# +# 2016-11-17: Currently takes >60 seconds results in test failure +# +enabled: False +cloud_config: | + #cloud-config + salt_minion: + conf: + master: salt.mydomain.com + public_key: | + ------BEGIN PUBLIC KEY------- + + ------END PUBLIC KEY------- + private_key: | + ------BEGIN PRIVATE KEY------ + + ------END PRIVATE KEY------- +collect_scripts: + minion: | + #!/bin/bash + cat /etc/salt/minion + minion_id: | + #!/bin/bash + cat /etc/salt/minion_id + minion.pem: | + #!/bin/bash + cat /etc/salt/pki/minion/minion.pem + minion.pub: | + #!/bin/bash + cat /etc/salt/pki/minion/minion.pub + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/seed_random_command.yaml b/tests/cloud_tests/testcases/modules/seed_random_command.yaml new file mode 100644 index 00000000..6a9157eb --- /dev/null +++ b/tests/cloud_tests/testcases/modules/seed_random_command.yaml @@ -0,0 +1,18 @@ +# +# Use uuid to create a random string +# +# 2016-11-15 Disabled as this is not working currently +# +enabled: False +cloud_config: | + #cloud-config + random_seed: + command: ["cat", "/proc/sys/kernel/random/uuid"] + command_required: true + file: /root/seed +collect_scripts: + seed_data: | + #!/bin/bash + cat /root/seed + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/seed_random_data.yaml b/tests/cloud_tests/testcases/modules/seed_random_data.yaml new file mode 100644 index 00000000..a9b2c885 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/seed_random_data.yaml @@ -0,0 +1,15 @@ +# +# Push in random raw string to set as seed +# +cloud_config: | + #cloud-config + random_seed: + data: 'MYUb34023nD:LFDK10913jk;dfnk:Df' + encoding: raw + file: /root/seed +collect_scripts: + seed_data: | + #!/bin/bash + cat /root/seed + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_hostname.yaml b/tests/cloud_tests/testcases/modules/set_hostname.yaml new file mode 100644 index 00000000..c96344cf --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_hostname.yaml @@ -0,0 +1,20 @@ +# +# Set the hostname and update /etc/hosts +# +required_features: + - hostname +cloud_config: | + #cloud-config + hostname: myhostname +collect_scripts: + hosts: | + #!/bin/bash + grep ^127 /etc/hosts + hostname: | + #!/bin/bash + hostname + fqdn: | + #!/bin/bash + hostname --fqdn + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml new file mode 100644 index 00000000..daf75931 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_hostname_fqdn.yaml @@ -0,0 +1,22 @@ +# +# Set the hostname and update /etc/hosts +# +required_features: + - hostname +cloud_config: | + #cloud-config + manage_etc_hosts: true + hostname: myhostname + fqdn: host.myorg.com +collect_scripts: + hosts: | + #!/bin/bash + grep ^127 /etc/hosts + hostname: | + #!/bin/bash + hostname + fqdn: | + #!/bin/bash + hostname --fqdn + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password.yaml b/tests/cloud_tests/testcases/modules/set_password.yaml new file mode 100644 index 00000000..04d7c58a --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_password.yaml @@ -0,0 +1,19 @@ +# +# Set password of default user +# +required_features: + - ubuntu_user +cloud_config: | + #cloud-config + password: password + chpasswd: { expire: False } + ssh_pwauth: True +collect_scripts: + shadow: | + #!/bin/bash + cat /etc/shadow + sshd_config: | + #!/bin/bash + grep '^PasswordAuth' /etc/ssh/sshd_config + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password_expire.yaml b/tests/cloud_tests/testcases/modules/set_password_expire.yaml new file mode 100644 index 00000000..789604b0 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_password_expire.yaml @@ -0,0 +1,30 @@ +# +# Expire password for all users +# +required_features: + - sshd +cloud_config: | + #cloud-config + chpasswd: { expire: True } + users: + - name: tom + password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + lock_passwd: false + - name: dick + password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + lock_passwd: false + - name: harry + password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + lock_passwd: false + - name: jane + password: $1$xyz$sPMsLNmf66Ohl.ol6JvzE. + lock_passwd: false +collect_scripts: + shadow: | + #!/bin/bash + cat /etc/shadow + sshd_config: | + #!/bin/bash + grep '^PasswordAuth' /etc/ssh/sshd_config + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password_list.yaml b/tests/cloud_tests/testcases/modules/set_password_list.yaml new file mode 100644 index 00000000..a2a89c9d --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_password_list.yaml @@ -0,0 +1,40 @@ +# +# Set password of list of users +# +cloud_config: | + #cloud-config + ssh_pwauth: yes + users: + - name: tom + # md5 gotomgo + passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" + lock_passwd: false + - name: dick + # md5 gocubsgo + passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" + lock_passwd: false + - name: harry + # sha512 goharrygo + passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/" + lock_passwd: false + - name: jane + # sha256 gojanego + passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." + lock_passwd: false + - name: "mikey" + lock_passwd: false + chpasswd: + list: + - tom:mypassword123! + - dick:RANDOM + - harry:RANDOM + - mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +collect_scripts: + shadow: | + #!/bin/bash + cat /etc/shadow + sshd_config: | + #!/bin/bash + grep '^PasswordAuth' /etc/ssh/sshd_config + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/set_password_list_string.yaml b/tests/cloud_tests/testcases/modules/set_password_list_string.yaml new file mode 100644 index 00000000..c2a0f631 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/set_password_list_string.yaml @@ -0,0 +1,40 @@ +# +# Set password of list of users as a string +# +cloud_config: | + #cloud-config + ssh_pwauth: yes + users: + - name: tom + # md5 gotomgo + passwd: "$1$S7$tT1BEDIYrczeryDQJfdPe0" + lock_passwd: false + - name: dick + # md5 gocubsgo + passwd: "$1$ssisyfpf$YqvuJLfrrW6Cg/l53Pi1n1" + lock_passwd: false + - name: harry + # sha512 goharrygo + passwd: "$6$LF$9Z2p6rWK6TNC1DC6393ec0As.18KRAvKDbfsGJEdWN3sRQRwpdfoh37EQ3yUh69tP4GSrGW5XKHxMLiKowJgm/" + lock_passwd: false + - name: jane + # sha256 gojanego + passwd: "$5$iW$XsxmWCdpwIW8Yhv.Jn/R3uk6A4UaicfW5Xp7C9p9pg." + lock_passwd: false + - name: "mikey" + lock_passwd: false + chpasswd: + list: | + tom:mypassword123! + dick:RANDOM + harry:RANDOM + mikey:$5$xZ$B2YGGEx2AOf4PeW48KC6.QyT1W2B4rZ9Qbltudtha89 +collect_scripts: + shadow: | + #!/bin/bash + cat /etc/shadow + sshd_config: | + #!/bin/bash + grep '^PasswordAuth' /etc/ssh/sshd_config + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/snappy.yaml b/tests/cloud_tests/testcases/modules/snappy.yaml new file mode 100644 index 00000000..43f93295 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/snappy.yaml @@ -0,0 +1,15 @@ +# +# Install snappy +# +required_features: + - snap +cloud_config: | + #cloud-config + snappy: + system_snappy: auto +collect_scripts: + snapd: | + #!/bin/bash + dpkg -s snapd + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml new file mode 100644 index 00000000..746653ec --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_disable.yaml @@ -0,0 +1,15 @@ +# +# Disable fingerprint printing +# +required_features: + - syslog +cloud_config: | + #cloud-config + ssh_genkeytypes: [] + no_ssh_fingerprints: true +collect_scripts: + syslog: | + #!/bin/bash + cat /var/log/syslog + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.yaml b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.yaml new file mode 100644 index 00000000..9f5dc34a --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ssh_auth_key_fingerprints_enable.yaml @@ -0,0 +1,21 @@ +# +# Print auth keys with different hash than md5 +# +# NOTE: testcase checks for '256 SHA256:.*(ECDSA)' on output line on trusty +# this fails as line in output reads '256:.*(ECDSA)' +required_features: + - syslog + - ssh_key_fmt +cloud_config: | + #cloud-config + ssh_genkeytypes: + - ecdsa + - ed25519 + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXW9Gg5H7ehjdSc6qDzwNtgCy94XYHhEYlXZMO2+FJrH3wfHGiMfCwOHxcOMt2QiXItULthdeQWS9QjBSSjVRXf6731igFrqPFyS9qBlOQ5D29C4HBXFnQggGVpBNJ82IRJv7szbbe/vpgLBP4kttUza9Dr4e1YM1ln4PRnjfXea6T0m+m1ixNb5432pTXlqYOnNOxSIm1gHgMLxPuDrJvQERDKrSiKSjIdyC9Jd8t2e1tkNLY0stmckVRbhShmcJvlyofHWbc2Ca1mmtP7MlS1VQnfLkvU1IrFwkmaQmaggX6WR6coRJ6XFXdWcq/AI2K6GjSnl1dnnCxE8VCEXBlXgFzad+PMSG4yiL5j8Oo1ZVpkTdgBnw4okGqTYCXyZg6X00As9IBNQfZMFlQXlIo4FiWgj3CO5QHQOyOX6FuEumaU13GnERrSSdp9tCs1Qm3/DG2RSCQBWTfcgMcStIvKqvJ3IjFn0vGLvI3Ampnq9q1SHwmmzAPSdzcMA76HyMUA5VWaBvWHlUxzIM6unxZASnwvuCzpywSEB5J2OF+p6H+cStJwQ32XwmOG8pLp1srlVWpqZI58Du/lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs506oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w== +collect_scripts: + syslog: | + #!/bin/bash + cat /var/log/syslog + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_import_id.yaml b/tests/cloud_tests/testcases/modules/ssh_import_id.yaml new file mode 100644 index 00000000..b62d3f69 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ssh_import_id.yaml @@ -0,0 +1,17 @@ +# +# Import a user's ssh key via gh or lp +# +required_features: + - ubuntu_user + - sudo +cloud_config: | + #cloud-config + ssh_import_id: + - gh:powersj + - lp:smoser +collect_scripts: + auth_keys_ubuntu: | + #!/bin/bash + cat /home/ubuntu/.ssh/authorized_keys + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml b/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml new file mode 100644 index 00000000..659fd939 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ssh_keys_generate.yaml @@ -0,0 +1,44 @@ +# +# SSH keys generated using cloud-init +# +required_features: + - ubuntu_user +cloud_config: | + #cloud-config + ssh_genkeytypes: + - ecdsa + - ed25519 + authkey_hash: sha512 +collect_scripts: + auth_keys_root: | + #!/bin/bash + cat /root/.ssh/authorized_keys + auth_keys_ubuntu: | + #!/bin/bash + cat /home/ubuntu/ssh/authorized_keys + dsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_dsa_key.pub + dsa_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_dsa_key + rsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_rsa_key.pub + rsa_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_rsa_key + ecdsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_ecdsa_key.pub + ecdsa_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_ecdsa_key + ed25519_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_ed25519_key.pub + ed25519_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_ed25519_key + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml b/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml new file mode 100644 index 00000000..5ceb3623 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ssh_keys_provided.yaml @@ -0,0 +1,105 @@ +# +# SSH keys provided via cloud config +# +enabled: False +required_features: + - ubuntu_user + - sudo +cloud_config: | + #cloud-config + disable_root: false + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXW9Gg5H7ehjdSc6qDzwNtgCy94XYHhEYlXZMO2+FJrH3wfHGiMfCwOHxcOMt2QiXItULthdeQWS9QjBSSjVRXf6731igFrqPFyS9qBlOQ5D29C4HBXFnQggGVpBNJ82IRJv7szbbe/vpgLBP4kttUza9Dr4e1YM1ln4PRnjfXea6T0m+m1ixNb5432pTXlqYOnNOxSIm1gHgMLxPuDrJvQERDKrSiKSjIdyC9Jd8t2e1tkNLY0stmckVRbhShmcJvlyofHWbc2Ca1mmtP7MlS1VQnfLkvU1IrFwkmaQmaggX6WR6coRJ6XFXdWcq/AI2K6GjSnl1dnnCxE8VCEXBlXgFzad+PMSG4yiL5j8Oo1ZVpkTdgBnw4okGqTYCXyZg6X00As9IBNQfZMFlQXlIo4FiWgj3CO5QHQOyOX6FuEumaU13GnERrSSdp9tCs1Qm3/DG2RSCQBWTfcgMcStIvKqvJ3IjFn0vGLvI3Ampnq9q1SHwmmzAPSdzcMA76HyMUA5VWaBvWHlUxzIM6unxZASnwvuCzpywSEB5J2OF+p6H+cStJwQ32XwmOG8pLp1srlVWpqZI58Du/lzrkPqONphoZx0LDV86w7RUz1ksDzAdcm0tvmNRFMN1a0frDs506oA3aWK0oDk4Nmvk8sXGTYYw3iQSkOvDUUlIsqdaO+w== + ssh_keys: + rsa_private: | + -----BEGIN RSA PRIVATE KEY----- + MIIEowIBAAKCAQEAtPx6PqN3iSEsnTtibyIEy52Tra8T5fn0ryXyg46Di2NBwdnj + o8trNv9jenfV/UhmePl58lXjT43wV8OCMl6KsYXyBdegM35NNtono4I4mLLKFMR9 + 9TOtDn6iYcaNenVhF3ZCj9Z2nNOlTrdc0uchHqKMrxLjCRCUrL91Uf+xioTF901Y + RM+ZqC5lT92yAL76F4qPF+Lq1QtUfNfUIwwvOp5ccDZLPxij0YvyBzubYye9hJHu + yjbJv78R4JHV+L2WhzSoX3W/6WrxVzeXqFGqH894ccOaC/7tnqSP6V8lIQ6fE2+c + DurJcpM3CJRgkndGHjtU55Y71YkcdLksSMvezQIDAQABAoIBAQCrU4IJP8dNeaj5 + IpkY6NQvR/jfZqfogYi+MKb1IHin/4rlDfUvPcY9pt8ttLlObjYK+OcWn3Vx/sRw + 4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2unRQvLZpMRdywBm + lq95OrCghnG03aUsFJUZPpi5ydnwbA12ma+KHkG0EzaVlhA7X9N6z0K6U+zue2gl + goMLt/MH0rsYawkHrwiwXaIFQeyV4MJP0vmrZLbFk1bycu9X/xPtTYotWyWo4eKA + cb05uu04qwexkKHDM0KXtT0JecbTo2rOefFo8Uuab6uJY+fEHNocZ+v1vLA4aOxJ + ovp1JuXlAoGBAOWYNgKrlTfy5n0sKsNk+1RuL2jHJZJ3HMd0EIt7/fFQN3Fi08Hu + jtntqD30Wj+DJK8b8Lrt66FruxyEJm5VhVmwkukrLR5ige2f6ftZnoFCmdyy+0zP + dnPZSUe2H5ZPHa+qthJgHLn+al2P04tGh+1fGHC2PbP+e0Co+/ZRIOxrAoGBAMnN + IEen9/FRsqvnDd36I8XnJGskVRTZNjylxBmbKcuMWm+gNhOI7gsCAcqzD4BYZjjW + pLhrt/u9p+l4MOJy6OUUdM/okg12SnJEGryysOcVBcXyrvOfklWnANG4EAH5jt1N + ftTb1XTxzvWVuR/WJK0B5MZNYM71cumBdUDtPi+nAoGAYmoIXMSnxb+8xNL10aOr + h9ljQQp8NHgSQfyiSufvRk0YNuYh1vMnEIsqnsPrG2Zfhx/25GmvoxXGssaCorDN + 5FAn6QK06F1ZTD5L0Y3sv4OI6G1gAuC66ZWuL6sFhyyKkQ4f1WiVZ7SCa3CHQSAO + i9VDaKz1bf4bXvAQcNj9v9kCgYACSOZCqW4vN0OUmqsXhkt9ZB6Pb/veno70pNPR + jmYsvcwQU3oJQpWfXkhy6RAV3epaXmPDCsUsfns2M3wqNC7a2R5xdCqjKGGzZX4A + AO3rz9se4J6Gd5oKijeCKFlWDGNHsibrdgm2pz42nZlY+O21X74dWKbt8O16I1MW + hxkbJQKBgAXfuen/srVkJgPuqywUYag90VWCpHsuxdn+fZJa50SyZADr+RbiDfH2 + vek8Uo8ap8AEsv4Rfs9opUcUZevLp3g2741eOaidHVLm0l4iLIVl03otGOqvSzs+ + A3tFPEOxauXpzCt8f8eXsz0WQXAgIKW2h8zu5QHjomioU3i27mtE + -----END RSA PRIVATE KEY----- + rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97N root@xenial-lxd + dsa_private: | + -----BEGIN DSA PRIVATE KEY----- + MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXrhOVAfzZ6+jklP + 55mzvC7jO53PWWC31hq10xBoWdev0WtcNF9Tv+4bAa1263y51Rqo4GI7xx+xic1d + mLqqfYijBT9k48J/1tV0cs1Wjs6FP/IJTD/kYVC930JjYQMi722lBnUxsQIVAL7i + z3fTGKTvSzvW0wQlwnYpS2QFAoGANp+KdyS9V93HgxGQEN1rlj/TSv/a3EVdCKtE + nQf55aPHxDAVDVw5JtRh4pZbbRV4oGRPc9KOdjo5BU28vSM3Lmhkb+UaaDXwHkgI + nK193o74DKjADWZxuLyyiKHiMOhxozoxDfjWxs8nz6uqvSW0pr521EwIY6RajbED + nZ2a3GkCgYEAyoUomNRB6bmpsIfzt8zdtqLP5umIj2uhr9MVPL8/QdbxmJ72Z7pf + Q2z1B7QAdIBGOlqJXtlau7ABhWK29Efe+99ObyTSSdDc6RCDeAwUmBAiPRQhDH2E + wExw3doDSCUb28L1B50wBzQ8mC3KXp6C7IkBXWspb16DLHUHFSI8bkICFA5kVUcW + nCPOXEQsayANi8+Cb7BH + -----END DSA PRIVATE KEY----- + dsa_public: ssh-dss AAAAB3NzaC1kc3MAAACBAPkWy1zbchVIN7qTgM0/yyY8q4RZS8cNM4ZpeuE5UB/Nnr6OSU/nmbO8LuM7nc9ZYLfWGrXTEGhZ16/Ra1w0X1O/7hsBrXbrfLnVGqjgYjvHH7GJzV2Yuqp9iKMFP2Tjwn/W1XRyzVaOzoU/8glMP+RhUL3fQmNhAyLvbaUGdTGxAAAAFQC+4s930xik70s71tMEJcJ2KUtkBQAAAIA2n4p3JL1X3ceDEZAQ3WuWP9NK/9rcRV0Iq0SdB/nlo8fEMBUNXDkm1GHillttFXigZE9z0o52OjkFTby9IzcuaGRv5RpoNfAeSAicrX3ejvgMqMANZnG4vLKIoeIw6HGjOjEN+NbGzyfPq6q9JbSmvnbUTAhjpFqNsQOdnZrcaQAAAIEAyoUomNRB6bmpsIfzt8zdtqLP5umIj2uhr9MVPL8/QdbxmJ72Z7pfQ2z1B7QAdIBGOlqJXtlau7ABhWK29Efe+99ObyTSSdDc6RCDeAwUmBAiPRQhDH2EwExw3doDSCUb28L1B50wBzQ8mC3KXp6C7IkBXWspb16DLHUHFSI8bkI= root@xenial-lxd + ed25519_private: | + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNTOhteXao0Nl5DVThJ2+QAAAJgwt+lcMLfp + XAAAAAtzc2gtZWQyNTUxOQAAACDbnQGUruL42aVVsyHeaV5mYNTOhteXao0Nl5DVThJ2+Q + AAAEDQlFZpz9q8+/YJHS9+jPAqy2ZT6cGEv8HTB6RZtTjd/dudAZSu4vjZpVWzId5pXmZg + 1M6G15dqjQ2XkNVOEnb5AAAAD3Jvb3RAeGVuaWFsLWx4ZAECAwQFBg== + -----END OPENSSH PRIVATE KEY----- + ed25519_public: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINudAZSu4vjZpVWzId5pXmZg1M6G15dqjQ2XkNVOEnb5 root@xenial-lxd + ecdsa_private: | + -----BEGIN EC PRIVATE KEY----- + MHcCAQEEIDuK+QFc1wmyJY8uDqQVa1qHte30Rk/fdLxGIBkwJAyOoAoGCCqGSM49 + AwEHoUQDQgAEWxLlO+TL8gL91eET9p/HFQbqR1A691AkJgZk3jY5mpZqxgX4vcgb + 7f/CtXuM6s2svcDJqAeXr6Wk8OJJcMxylA== + -----END EC PRIVATE KEY----- + ecdsa_public: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFsS5Tvky/IC/dXhE/afxxUG6kdQOvdQJCYGZN42OZqWasYF+L3IG+3/wrV7jOrNrL3AyagHl6+lpPDiSXDMcpQ= root@xenial-lxd +collect_scripts: + auth_keys_root: | + #!/bin/bash + cat /root/.ssh/authorized_keys + auth_keys_ubuntu: | + #!/bin/bash + cat /home/ubuntu/ssh/authorized_keys + dsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_dsa_key.pub + dsa_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_dsa_key + rsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_rsa_key.pub + rsa_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_rsa_key + ecdsa_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_ecdsa_key.pub + ecdsa_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_ecdsa_key + ed25519_public: | + #!/bin/bash + cat /etc/ssh/ssh_host_ed25519_key.pub + ed25519_private: | + #!/bin/bash + cat /etc/ssh/ssh_host_ed25519_key + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/timezone.yaml b/tests/cloud_tests/testcases/modules/timezone.yaml new file mode 100644 index 00000000..5112aa9f --- /dev/null +++ b/tests/cloud_tests/testcases/modules/timezone.yaml @@ -0,0 +1,16 @@ +# +# Set system timezone +# +required_features: + - daylight_time +cloud_config: | + #cloud-config + timezone: US/Aleutian +collect_scripts: + timezone: | + #!/bin/bash + # date will convert this to system's configured time zone. + # use a static date to avoid dealing with daylight savings. + date "+%Z" --date="Thu, 03 Nov 2016 00:47:00 -0400" + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/user_groups.yaml b/tests/cloud_tests/testcases/modules/user_groups.yaml new file mode 100644 index 00000000..71cc9da3 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/user_groups.yaml @@ -0,0 +1,52 @@ +# +# Create groups and users with various options +# +required_features: + - ubuntu_user +cloud_config: | + #cloud-config + # Add groups to the system + groups: + - secret: [foobar,barfoo] + - cloud-users + + # Add users to the system. Users are added after groups are added. + users: + - default + - name: foobar + gecos: Foo B. Bar + primary-group: foobar + groups: users + expiredate: 2038-01-19 + lock_passwd: false + passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ + - name: barfoo + gecos: Bar B. Foo + sudo: ALL=(ALL) NOPASSWD:ALL + groups: cloud-users + lock_passwd: true + - name: cloudy + gecos: Magic Cloud App Daemon User + inactive: true + system: true +collect_scripts: + group_ubuntu: | + #!/bin/bash + getent group ubuntu + group_cloud_users: | + #!/bin/bash + getent group cloud-users + user_ubuntu: | + #!/bin/bash + getent passwd ubuntu + user_foobar: | + #!/bin/bash + getent passwd foobar + user_barfoo: | + #!/bin/bash + getent passwd barfoo + user_cloudy: | + #!/bin/bash + getent passwd cloudy + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/write_files.yaml b/tests/cloud_tests/testcases/modules/write_files.yaml new file mode 100644 index 00000000..ce936b7b --- /dev/null +++ b/tests/cloud_tests/testcases/modules/write_files.yaml @@ -0,0 +1,46 @@ +# +# Write various file types +# +# NOTE: on trusty 'file' has an output formatting error for binary files and +# has 2 spaces in 'LSB executable', which causes a failure here +required_features: + - no_file_fmt_e +cloud_config: | + #cloud-config + write_files: + - encoding: b64 + content: CiMgVGhpcyBmaWxlIGNvbnRyb2xzIHRoZSBzdGF0ZSBvZiBTRUxpbnV4 + owner: root:root + path: /root/file_b64 + permissions: '0644' + - content: | + # My new /root/file_text + + SMBDOPTIONS="-D" + path: /root/file_text + - content: !!binary | + f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI + AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA + AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA + path: /root/file_binary + permissions: '0555' + - encoding: gzip + content: !!binary | + H4sIAIDb/U8C/1NW1E/KzNMvzuBKTc7IV8hIzcnJVyjPL8pJ4QIA6N+MVxsAAAA= + path: /root/file_gzip + permissions: '0755' +collect_scripts: + file_b64: | + #!/bin/bash + file /root/file_b64 + file_text: | + #!/bin/bash + file /root/file_text + file_binary: | + #!/bin/bash + file /root/file_binary + file_gzip: | + #!/bin/bash + file /root/file_gzip + +# vi: ts=4 expandtab -- cgit v1.2.3 From cc1475d07b9d0727012634ee9c7a914d67b051f5 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Thu, 21 Sep 2017 11:58:28 -0400 Subject: suse: Support addition of zypper repos via cloud-config. This adds a config module so support for adding zypper repositories via cloud-config. LP: #1718675 --- cloudinit/config/cc_zypper_add_repo.py | 218 +++++++++++++++++++ config/cloud.cfg.tmpl | 3 + .../test_handler/test_handler_zypper_add_repo.py | 237 +++++++++++++++++++++ tests/unittests/test_handler/test_schema.py | 8 +- 4 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/cc_zypper_add_repo.py create mode 100644 tests/unittests/test_handler/test_handler_zypper_add_repo.py diff --git a/cloudinit/config/cc_zypper_add_repo.py b/cloudinit/config/cc_zypper_add_repo.py new file mode 100644 index 00000000..aba26952 --- /dev/null +++ b/cloudinit/config/cc_zypper_add_repo.py @@ -0,0 +1,218 @@ +# +# Copyright (C) 2017 SUSE LLC. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""zypper_add_repo: Add zyper repositories to the system""" + +import configobj +import os +from six import string_types +from textwrap import dedent + +from cloudinit.config.schema import get_schema_doc +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util + +distros = ['opensuse', 'sles'] + +schema = { + 'id': 'cc_zypper_add_repo', + 'name': 'ZypperAddRepo', + 'title': 'Configure zypper behavior and add zypper repositories', + 'description': dedent("""\ + Configure zypper behavior by modifying /etc/zypp/zypp.conf. The + configuration writer is "dumb" and will simply append the provided + configuration options to the configuration file. Option settings + that may be duplicate will be resolved by the way the zypp.conf file + is parsed. The file is in INI format. + Add repositories to the system. No validation is performed on the + repository file entries, it is assumed the user is familiar with + the zypper repository file format."""), + 'distros': distros, + 'examples': [dedent("""\ + zypper: + repos: + - id: opensuse-oss + name: os-oss + baseurl: http://dl.opensuse.org/dist/leap/v/repo/oss/ + enabled: 1 + autorefresh: 1 + - id: opensuse-oss-update + name: os-oss-up + baseurl: http://dl.opensuse.org/dist/leap/v/update + # any setting per + # https://en.opensuse.org/openSUSE:Standards_RepoInfo + # enable and autorefresh are on by default + config: + reposdir: /etc/zypp/repos.dir + servicesdir: /etc/zypp/services.d + download.use_deltarpm: true + # any setting in /etc/zypp/zypp.conf + """)], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'zypper': { + 'type': 'object', + 'properties': { + 'repos': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'description': dedent("""\ + The unique id of the repo, used when + writing + /etc/zypp/repos.d/.repo.""") + }, + 'baseurl': { + 'type': 'string', + 'format': 'uri', # built-in format type + 'description': 'The base repositoy URL' + } + }, + 'required': ['id', 'baseurl'], + 'additionalProperties': True + }, + 'minItems': 1 + }, + 'config': { + 'type': 'object', + 'description': dedent("""\ + Any supported zypo.conf key is written to + /etc/zypp/zypp.conf'""") + } + }, + 'required': [], + 'minProperties': 1, # Either config or repo must be provided + 'additionalProperties': False, # only repos and config allowed + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + +LOG = logging.getLogger(__name__) + + +def _canonicalize_id(repo_id): + repo_id = repo_id.replace(" ", "_") + return repo_id + + +def _format_repo_value(val): + if isinstance(val, bool): + # zypp prefers 1/0 + return 1 if val else 0 + if isinstance(val, (list, tuple)): + return "\n ".join([_format_repo_value(v) for v in val]) + if not isinstance(val, string_types): + return str(val) + return val + + +def _format_repository_config(repo_id, repo_config): + to_be = configobj.ConfigObj() + to_be[repo_id] = {} + # Do basic translation of the items -> values + for (k, v) in repo_config.items(): + # For now assume that people using this know the format + # of zypper repos and don't verify keys/values further + to_be[repo_id][k] = _format_repo_value(v) + lines = to_be.write() + return "\n".join(lines) + + +def _write_repos(repos, repo_base_path): + """Write the user-provided repo definition files + @param repos: A list of repo dictionary objects provided by the user's + cloud config. + @param repo_base_path: The directory path to which repo definitions are + written. + """ + + if not repos: + return + valid_repos = {} + for index, user_repo_config in enumerate(repos): + # Skip on absent required keys + missing_keys = set(['id', 'baseurl']).difference(set(user_repo_config)) + if missing_keys: + LOG.warning( + "Repo config at index %d is missing required config keys: %s", + index, ",".join(missing_keys)) + continue + repo_id = user_repo_config.get('id') + canon_repo_id = _canonicalize_id(repo_id) + repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) + if os.path.exists(repo_fn_pth): + LOG.info("Skipping repo %s, file %s already exists!", + repo_id, repo_fn_pth) + continue + elif repo_id in valid_repos: + LOG.info("Skipping repo %s, file %s already pending!", + repo_id, repo_fn_pth) + continue + + # Do some basic key formatting + repo_config = dict( + (k.lower().strip().replace("-", "_"), v) + for k, v in user_repo_config.items() + if k and k != 'id') + + # Set defaults if not present + for field in ['enabled', 'autorefresh']: + if field not in repo_config: + repo_config[field] = '1' + + valid_repos[repo_id] = (repo_fn_pth, repo_config) + + for (repo_id, repo_data) in valid_repos.items(): + repo_blob = _format_repository_config(repo_id, repo_data[-1]) + util.write_file(repo_data[0], repo_blob) + + +def _write_zypp_config(zypper_config): + """Write to the default zypp configuration file /etc/zypp/zypp.conf""" + if not zypper_config: + return + zypp_config = '/etc/zypp/zypp.conf' + zypp_conf_content = util.load_file(zypp_config) + new_settings = ['# Added via cloud.cfg'] + for setting, value in zypper_config.items(): + if setting == 'configdir': + msg = 'Changing the location of the zypper configuration is ' + msg += 'not supported, skipping "configdir" setting' + LOG.warning(msg) + continue + if value: + new_settings.append('%s=%s' % (setting, value)) + if len(new_settings) > 1: + new_config = zypp_conf_content + '\n'.join(new_settings) + else: + new_config = zypp_conf_content + util.write_file(zypp_config, new_config) + + +def handle(name, cfg, _cloud, log, _args): + zypper_section = cfg.get('zypper') + if not zypper_section: + LOG.debug(("Skipping module named %s," + " no 'zypper' relevant configuration found"), name) + return + repos = zypper_section.get('repos') + if not repos: + LOG.debug(("Skipping module named %s," + " no 'repos' configuration found"), name) + return + zypper_config = zypper_section.get('config', {}) + repo_base_path = zypper_config.get('reposdir', '/etc/zypp/repos.d/') + + _write_zypp_config(zypper_config) + _write_repos(repos, repo_base_path) + +# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 50e3bd86..32de9c9b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -84,6 +84,9 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} +{% if variant in ["suse"] %} + - zypper-add-repo +{% endif %} {% if variant not in ["freebsd"] %} - ntp {% endif %} diff --git a/tests/unittests/test_handler/test_handler_zypper_add_repo.py b/tests/unittests/test_handler/test_handler_zypper_add_repo.py new file mode 100644 index 00000000..315c2a5e --- /dev/null +++ b/tests/unittests/test_handler/test_handler_zypper_add_repo.py @@ -0,0 +1,237 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import glob +import os + +from cloudinit.config import cc_zypper_add_repo +from cloudinit import util + +from cloudinit.tests import helpers +from cloudinit.tests.helpers import mock + +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser +import logging +from six import StringIO + +LOG = logging.getLogger(__name__) + + +class TestConfig(helpers.FilesystemMockingTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.tmp = self.tmp_dir() + self.zypp_conf = 'etc/zypp/zypp.conf' + + def test_bad_repo_config(self): + """Config has no baseurl, no file should be written""" + cfg = { + 'repos': [ + { + 'id': 'foo', + 'name': 'suse-test', + 'enabled': '1' + }, + ] + } + self.patchUtils(self.tmp) + cc_zypper_add_repo._write_repos(cfg['repos'], '/etc/zypp/repos.d') + self.assertRaises(IOError, util.load_file, + "/etc/zypp/repos.d/foo.repo") + + def test_write_repos(self): + """Verify valid repos get written""" + cfg = self._get_base_config_repos() + root_d = self.tmp_dir() + cc_zypper_add_repo._write_repos(cfg['zypper']['repos'], root_d) + repos = glob.glob('%s/*.repo' % root_d) + expected_repos = ['testing-foo.repo', 'testing-bar.repo'] + if len(repos) != 2: + assert 'Number of repos written is "%d" expected 2' % len(repos) + for repo in repos: + repo_name = os.path.basename(repo) + if repo_name not in expected_repos: + assert 'Found repo with name "%s"; unexpected' % repo_name + # Validation that the content gets properly written is in another test + + def test_write_repo(self): + """Verify the content of a repo file""" + cfg = { + 'repos': [ + { + 'baseurl': 'http://foo', + 'name': 'test-foo', + 'id': 'testing-foo' + }, + ] + } + root_d = self.tmp_dir() + cc_zypper_add_repo._write_repos(cfg['repos'], root_d) + contents = util.load_file("%s/testing-foo.repo" % root_d) + parser = ConfigParser() + parser.readfp(StringIO(contents)) + expected = { + 'testing-foo': { + 'name': 'test-foo', + 'baseurl': 'http://foo', + 'enabled': '1', + 'autorefresh': '1' + } + } + for section in expected: + self.assertTrue(parser.has_section(section), + "Contains section {0}".format(section)) + for k, v in expected[section].items(): + self.assertEqual(parser.get(section, k), v) + + def test_config_write(self): + """Write valid configuration data""" + cfg = { + 'config': { + 'download.deltarpm': 'False', + 'reposdir': 'foo' + } + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg['config']) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + 'reposdir=foo' + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + + @mock.patch('cloudinit.log.logging') + def test_config_write_skip_configdir(self, mock_logging): + """Write configuration but skip writing 'configdir' setting""" + cfg = { + 'config': { + 'download.deltarpm': 'False', + 'reposdir': 'foo', + 'configdir': 'bar' + } + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg['config']) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + 'reposdir=foo' + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + # Not finding teh right path for mocking :( + # assert mock_logging.warning.called + + def test_empty_config_section_no_new_data(self): + """When the config section is empty no new data should be written to + zypp.conf""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = None + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_empty_config_value_no_new_data(self): + """When the config section is not empty but there are no values + no new data should be written to zypp.conf""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = { + 'download.deltarpm': None + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_handler_full_setup(self): + """Test that the handler ends up calling the renderers""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = { + 'download.deltarpm': 'False', + } + root_d = self.tmp_dir() + os.makedirs('%s/etc/zypp/repos.d' % root_d) + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo.handle('zypper_add_repo', cfg, None, LOG, []) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + repos = glob.glob('%s/etc/zypp/repos.d/*.repo' % root_d) + expected_repos = ['testing-foo.repo', 'testing-bar.repo'] + if len(repos) != 2: + assert 'Number of repos written is "%d" expected 2' % len(repos) + for repo in repos: + repo_name = os.path.basename(repo) + if repo_name not in expected_repos: + assert 'Found repo with name "%s"; unexpected' % repo_name + + def test_no_config_section_no_new_data(self): + """When there is no config section no new data should be written to + zypp.conf""" + cfg = self._get_base_config_repos() + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_no_repo_data(self): + """When there is no repo data nothing should happen""" + root_d = self.tmp_dir() + self.reRoot(root_d) + cc_zypper_add_repo._write_repos(None, root_d) + content = glob.glob('%s/*' % root_d) + self.assertEqual(len(content), 0) + + def _get_base_config_repos(self): + """Basic valid repo configuration""" + cfg = { + 'zypper': { + 'repos': [ + { + 'baseurl': 'http://foo', + 'name': 'test-foo', + 'id': 'testing-foo' + }, + { + 'baseurl': 'http://bar', + 'name': 'test-bar', + 'id': 'testing-bar' + } + ] + } + } + return cfg diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 745bb0ff..b8fc8930 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,7 +27,13 @@ class GetSchemaTest(CiTestCase): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() self.assertItemsEqual( - ['cc_bootcmd', 'cc_ntp', 'cc_resizefs', 'cc_runcmd'], + [ + 'cc_bootcmd', + 'cc_ntp', + 'cc_resizefs', + 'cc_runcmd', + 'cc_zypper_add_repo' + ], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) self.assertEqual( -- cgit v1.2.3 From 7fd04255ef238e06d1d9c9a5cc35060d36d83f2b Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Mon, 18 Sep 2017 12:32:10 -0400 Subject: systemd: remove limit on tasks created by cloud-init-final.service. Depending on distribution the default number of tasks (threads) maybe unexpectedly low or it may be the default systemd setting (512). Setting TasksMax to "infinity" in cloud-init-final.service removes the restriction on tasks created. LP: #1717969 --- systemd/cloud-final.service.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/systemd/cloud-final.service.tmpl b/systemd/cloud-final.service.tmpl index e2b91255..8207b18c 100644 --- a/systemd/cloud-final.service.tmpl +++ b/systemd/cloud-final.service.tmpl @@ -15,6 +15,7 @@ ExecStart=/usr/bin/cloud-init modules --mode=final RemainAfterExit=yes TimeoutSec=0 KillMode=process +TasksMax=infinity # Output needs to appear in instance console output StandardOutput=journal+console -- cgit v1.2.3 From aa024e331f8196855fa8d707a2dd7e26e1deab40 Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 3 Oct 2017 14:19:04 -0700 Subject: tests: re-enable tox with nocloud-kvm support With the addition of the nocloud-kvm support a few other python modules were pulled in as required and as a result this broke the tox run. The fix was to add paramiko and simplestreams to re-enable testing. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index aef1f84b..92232201 100644 --- a/tox.ini +++ b/tox.ini @@ -132,3 +132,5 @@ commands = {envpython} -m tests.cloud_tests {posargs} passenv = HOME deps = pylxd==2.2.4 + paramiko==2.3.1 + bzr+lp:simplestreams -- cgit v1.2.3 From 57e2e01c703cdd1818d4f4ab8a67f37037d78582 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 3 Oct 2017 18:56:52 -0500 Subject: network: bridge_stp value not always correct Update network_state to store the bridge_stp value as a boolean. The various renderers then can map the boolean value to the correct output as needed; eni uses 'on/off', sysconfig uses 'yes/no' and netplan will use the boolean directly. Update unittest values for sysconfig and netplan. Both contained the network_state string value which resulted in not correctly enable/disable STP in the target system. Update network_state comment (fd -> forward-delay, add stp as boolean) on bridge commands to match the expected format of a netplan bridge command. LP: #1721157 --- cloudinit/net/eni.py | 3 +++ cloudinit/net/netplan.py | 5 +++-- cloudinit/net/network_state.py | 17 +++++++++++++++-- tests/unittests/test_net.py | 5 +++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index bb80ec02..c6a71d16 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -95,6 +95,9 @@ def _iface_add_attrs(iface, index): ignore_map.append('mac_address') for key, value in iface.items(): + # convert bool to string for eni + if type(value) == bool: + value = 'on' if iface[key] else 'off' if not value or key in ignore_map: continue if key in multiline_keys: diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 3b06fbf0..d3788af8 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -244,9 +244,9 @@ class Renderer(renderer.Renderer): for config in network_state.iter_interfaces(): ifname = config.get('name') - # filter None entries up front so we can do simple if key in dict + # filter None (but not False) entries up front ifcfg = dict((key, value) for (key, value) in config.items() - if value) + if value is not None) if_type = ifcfg.get('type') if if_type == 'physical': @@ -318,6 +318,7 @@ class Renderer(renderer.Renderer): (port, cost) = costval.split() newvalue[port] = int(cost) br_config.update({newname: newvalue}) + if len(br_config) > 0: bridge.update({'parameters': br_config}) _extract_addresses(ifcfg, bridge) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 6faf01b7..890dbf8d 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -48,6 +48,7 @@ NET_CONFIG_TO_V2 = { 'bridge_maxwait': None, 'bridge_pathcost': 'path-cost', 'bridge_portprio': None, + 'bridge_stp': 'stp', 'bridge_waitport': None}} @@ -465,6 +466,18 @@ class NetworkStateInterpreter(object): for param, val in command.get('params', {}).items(): iface.update({param: val}) + # convert value to boolean + bridge_stp = iface.get('bridge_stp') + if bridge_stp and type(bridge_stp) != bool: + if bridge_stp in ['on', '1', 1]: + bridge_stp = True + elif bridge_stp in ['off', '0', 0]: + bridge_stp = False + else: + raise ValueError("Cannot convert bridge_stp value" + "(%s) to boolean", bridge_stp) + iface.update({'bridge_stp': bridge_stp}) + interfaces.update({iface['name']: iface}) @ensure_command_keys(['address']) @@ -525,8 +538,8 @@ class NetworkStateInterpreter(object): v2_command = { br0: { 'interfaces': ['interface0', 'interface1'], - 'fd': 0, - 'stp': 'off', + 'forward-delay': 0, + 'stp': False, 'maxwait': 0, } } diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f2496151..17c9342b 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -756,6 +756,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true eth3: 50 eth4: 75 priority: 22 + stp: false routes: - to: ::/0 via: 2001:4800:78ff:1b::1 @@ -820,7 +821,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true NM_CONTROLLED=no ONBOOT=yes PRIO=22 - STP=off + STP=no TYPE=Bridge USERCTL=no"""), 'ifcfg-eth0': textwrap.dedent("""\ @@ -1296,7 +1297,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true NM_CONTROLLED=no ONBOOT=yes PRIO=22 - STP=off + STP=no TYPE=Bridge USERCTL=no """), -- cgit v1.2.3 From 6eb4dc24fe314ce5c98b05b21988402cda95afce Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Oct 2017 14:21:00 -0400 Subject: tools: Give specific --abbrev=8 to "git describe" The tools that use "git describe" were just assuming a consisent number of characters in the hash. It seems ubuntu 16.04 would use 7 and later versions use 8. To avoid that discrepency in developer environments, set it to 8. --- tools/make-tarball | 2 +- tools/read-version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/make-tarball b/tools/make-tarball index 91c45624..3197689f 100755 --- a/tools/make-tarball +++ b/tools/make-tarball @@ -35,7 +35,7 @@ while [ $# -ne 0 ]; do done rev=${1:-HEAD} -version=$(git describe "--match=[0-9]*" ${long_opt} $rev) +version=$(git describe --abbrev=8 "--match=[0-9]*" ${long_opt} $rev) archive_base="cloud-init-$version" if [ -z "$output" ]; then diff --git a/tools/read-version b/tools/read-version index ddb28383..d9ed30da 100755 --- a/tools/read-version +++ b/tools/read-version @@ -56,7 +56,7 @@ if os.path.isdir(os.path.join(_tdir, ".git")) and which("git"): flags = [] if use_tags: flags = ['--tags'] - cmd = ['git', 'describe', '--match=[0-9]*'] + flags + cmd = ['git', 'describe', '--abbrev=8', '--match=[0-9]*'] + flags version = tiny_p(cmd).strip() -- cgit v1.2.3 From 45d361cb0b7f5e4e7d79522bd285871898358623 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 5 Oct 2017 14:26:34 -0600 Subject: net: Handle bridge stp values of 0 and convert to boolean type Update unit tests to pass a 0 instead of 'off' to validate that network state is properly written. --- cloudinit/net/network_state.py | 2 +- tests/unittests/test_net.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 890dbf8d..0e830ee8 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -468,7 +468,7 @@ class NetworkStateInterpreter(object): # convert value to boolean bridge_stp = iface.get('bridge_stp') - if bridge_stp and type(bridge_stp) != bool: + if bridge_stp is not None and type(bridge_stp) != bool: if bridge_stp in ['on', '1', 1]: bridge_stp = True elif bridge_stp in ['off', '0', 0]: diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 17c9342b..bbb63cb3 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1283,7 +1283,7 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true - eth0 - eth1 params: - bridge_stp: 'off' + bridge_stp: 0 bridge_bridgeprio: 22 subnets: - type: static -- cgit v1.2.3