From 5bba5db2655d88b8aba8fa06b30f8e91e2ca6836 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 1 Aug 2017 18:00:00 -0500 Subject: cc_ntp: fallback on timesyncd configuration if ntp is not installable Some systems like Ubuntu-Core do not provide an ntp package for installation but do include systemd-timesyncd (an ntp client). On such systems cloud-init will generate a timesyncd configuration using the 'servers' and 'pools' values as ntp hosts for timesyncd to use. LP: #1686485 --- cloudinit/config/cc_ntp.py | 58 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 13 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 31ed64e3..a02b4bf1 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -50,6 +50,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' +TIMESYNCD_CONF = '/etc/systemd/timesyncd.conf.d/cloud-init.conf' NR_POOL_SERVERS = 4 distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu'] @@ -132,20 +133,50 @@ def handle(name, cfg, cloud, log, _args): " is a %s %instead"), type_utils.obj_name(ntp_cfg)) validate_cloudconfig_schema(cfg, schema) + if ntp_installable(): + service_name = 'ntp' + confpath = NTP_CONF + template_name = None + packages = ['ntp'] + check_exe = 'ntpd' + else: + service_name = 'systemd-timesyncd' + confpath = TIMESYNCD_CONF + template_name = 'timesyncd.conf' + packages = [] + check_exe = '/lib/systemd/systemd-timesyncd' + rename_ntp_conf() # ensure when ntp is installed it has a configuration file # to use instead of starting up with packaged defaults - write_ntp_config_template(ntp_cfg, cloud) - install_ntp(cloud.distro.install_packages, packages=['ntp'], - check_exe="ntpd") - # if ntp was already installed, it may not have started + write_ntp_config_template(ntp_cfg, cloud, confpath, template=template_name) + install_ntp(cloud.distro.install_packages, packages=packages, + check_exe=check_exe) + try: - reload_ntp(systemd=cloud.distro.uses_systemd()) + reload_ntp(service_name, systemd=cloud.distro.uses_systemd()) except util.ProcessExecutionError as e: LOG.exception("Failed to reload/start ntp service: %s", e) raise +def ntp_installable(): + """Check if we can install ntp package + + Ubuntu-Core systems do not have an ntp package available, so + we always return False. Other systems require package managers to install + the ntp package If we fail to find one of the package managers, then we + cannot install ntp. + """ + if util.system_is_snappy(): + return False + + if any(map(util.which, ['apt-get', 'dnf', 'yum', 'zypper'])): + return True + + return False + + def install_ntp(install_func, packages=None, check_exe="ntpd"): if util.which(check_exe): return @@ -156,7 +187,7 @@ def install_ntp(install_func, packages=None, check_exe="ntpd"): def rename_ntp_conf(config=None): - """Rename any existing ntp.conf file and render from template""" + """Rename any existing ntp.conf file""" if config is None: # For testing config = NTP_CONF if os.path.exists(config): @@ -171,7 +202,7 @@ def generate_server_names(distro): return names -def write_ntp_config_template(cfg, cloud): +def write_ntp_config_template(cfg, cloud, path, template=None): servers = cfg.get('servers', []) pools = cfg.get('pools', []) @@ -185,19 +216,20 @@ def write_ntp_config_template(cfg, cloud): 'pools': pools, } - template_fn = cloud.get_template_filename('ntp.conf.%s' % - (cloud.distro.name)) + if template is None: + template = 'ntp.conf.%s' % cloud.distro.name + + template_fn = cloud.get_template_filename(template) if not template_fn: template_fn = cloud.get_template_filename('ntp.conf') if not template_fn: raise RuntimeError(("No template found, " - "not rendering %s"), NTP_CONF) + "not rendering %s"), path) - templater.render_to_file(template_fn, NTP_CONF, params) + templater.render_to_file(template_fn, path, params) -def reload_ntp(systemd=False): - service = 'ntp' +def reload_ntp(service, systemd=False): if systemd: cmd = ['systemctl', 'reload-or-restart', service] else: -- cgit v1.2.3 From cc9762a2d737ead386ffb9f067adc5e543224560 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 22 Aug 2017 20:06:20 -0600 Subject: schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema This branch does a few things: - Add 'schema' subcommand to cloud-init CLI for validating cloud-config files against strict module jsonschema definitions - Add --annotate parameter to 'cloud-init schema' to annotate existing cloud-config file content with validation errors - Add jsonschema definition to cc_runcmd - Add unit test coverage for cc_runcmd - Update CLI capabilities documentation This branch only imports development (and analyze) subparsers when the specific subcommand is provided on the CLI to avoid adding costly unused file imports during cloud-init system boot. The schema command allows a person to quickly validate a cloud-config text file against cloud-init's known module schemas to avoid costly roundtrips deploying instances in their cloud of choice. As of this branch, only cc_ntp and cc_runcmd cloud-config modules define schemas. Schema validation will ignore all undefined config keys until all modules define a strict schema. To perform validation of runcmd and ntp sections of a cloud-config file: $ cat > cloud.cfg < Date: Wed, 23 Aug 2017 13:24:38 -0600 Subject: cc_landscape & cc_puppet: Fix six.StringIO use in writing configs Both landscape and puppet modules had issues with the way they wrote /etc/landscape/client.conf or /etc/puppet/puppet.conf in either python3 or python2. This branch adds initial unit tests for both modules which will get better exercise under both python2 and python3. The unit tests shed light on a few issues: - In the cc_landscape module py3 can't provide six.StringIO content to ConfigParser.write, so we need to use six.BytesIO instead - In the cc_puppet module, python <= 2.7 doesn't support using six.StringIO as a context manager, so we drop the context manager fanciness and directly set outputstream = StringIO(). - The docstring in cc_puppet is fixed to document the 'conf' sub-key requiring valid puppet section names for each key-value list. LP: #1699282 LP: #1710932 --- cloudinit/config/cc_landscape.py | 4 +- cloudinit/config/cc_puppet.py | 33 ++--- cloudinit/helpers.py | 14 +- .../test_handler/test_handler_landscape.py | 129 +++++++++++++++++++ .../unittests/test_handler/test_handler_puppet.py | 142 +++++++++++++++++++++ 5 files changed, 298 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_landscape.py create mode 100644 tests/unittests/test_handler/test_handler_puppet.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 86b71383..8f9f1abd 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -57,7 +57,7 @@ The following default client config is provided, but can be overridden:: import os -from six import StringIO +from six import BytesIO from configobj import ConfigObj @@ -109,7 +109,7 @@ def handle(_name, cfg, cloud, log, _args): ls_cloudcfg, ] merged = merge_together(merge_data) - contents = StringIO() + contents = BytesIO() merged.write(contents) util.ensure_dir(os.path.dirname(LSC_CLIENT_CFG_FILE)) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index dc11561b..28b1d568 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -15,21 +15,23 @@ This module handles puppet installation and configuration. If the ``puppet`` key does not exist in global configuration, no action will be taken. If a config entry for ``puppet`` is present, then by default the latest version of puppet will be installed. If ``install`` is set to ``false``, puppet will not -be installed. However, this may result in an error if puppet is not already +be installed. However, this will result in an error if puppet is not already present on the system. The version of puppet to be installed can be specified under ``version``, and defaults to ``none``, which selects the latest version in the repos. If the ``puppet`` config key exists in the config archive, this module will attempt to start puppet even if no installation was performed. -Puppet configuration can be specified under the ``conf`` key. The configuration -is specified as a dictionary which is converted into ``=`` format -and appended to ``puppet.conf`` under the ``[puppetd]`` section. The +Puppet configuration can be specified under the ``conf`` key. The +configuration is specified as a dictionary containing high-level ``
`` +keys and lists of ``=`` pairs within each section. Each section +name and ``=`` pair is written directly to ``puppet.conf``. As +such, section names should be one of: ``main``, ``master``, ``agent`` or +``user`` and keys should be valid puppet configuration options. The ``certname`` key supports string substitutions for ``%i`` and ``%f``, corresponding to the instance id and fqdn of the machine respectively. -If ``ca_cert`` is present under ``conf``, it will not be written to -``puppet.conf``, but instead will be used as the puppermaster certificate. -It should be specified in pem format as a multi-line string (using the ``|`` -yaml notation). +If ``ca_cert`` is present, it will not be written to ``puppet.conf``, but +instead will be used as the puppermaster certificate. It should be specified +in pem format as a multi-line string (using the ``|`` yaml notation). **Internal name:** ``cc_puppet`` @@ -43,12 +45,13 @@ yaml notation). install: version: conf: - server: "puppetmaster.example.org" - certname: "%i.%f" - ca_cert: | - -------BEGIN CERTIFICATE------- - - -------END CERTIFICATE------- + agent: + server: "puppetmaster.example.org" + certname: "%i.%f" + ca_cert: | + -------BEGIN CERTIFICATE------- + + -------END CERTIFICATE------- """ from six import StringIO @@ -127,7 +130,7 @@ def handle(name, cfg, cloud, log, _args): util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: - # Iterate throug the config items, we'll use ConfigParser.set + # Iterate through the config items, we'll use ConfigParser.set # to overwrite or create new items as needed for (o, v) in cfg.items(): if o == 'certname': diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index f01021aa..1979cd96 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -13,7 +13,7 @@ from time import time import contextlib import os -import six +from six import StringIO from six.moves.configparser import ( NoSectionError, NoOptionError, RawConfigParser) @@ -441,12 +441,12 @@ class DefaultingConfigParser(RawConfigParser): def stringify(self, header=None): contents = '' - with six.StringIO() as outputstream: - self.write(outputstream) - outputstream.flush() - contents = outputstream.getvalue() - if header: - contents = "\n".join([header, contents]) + outputstream = StringIO() + self.write(outputstream) + outputstream.flush() + contents = outputstream.getvalue() + if header: + contents = '\n'.join([header, contents, '']) return contents # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py new file mode 100644 index 00000000..7c247fa9 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -0,0 +1,129 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_landscape +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call + +from configobj import ConfigObj +import logging + + +LOG = logging.getLogger(__name__) + + +class TestLandscape(FilesystemMockingTestCase): + + with_logs = True + + def setUp(self): + super(TestLandscape, self).setUp() + self.new_root = self.tmp_dir() + self.conf = self.tmp_path('client.conf', self.new_root) + self.default_file = self.tmp_path('default_landscape', self.new_root) + + def _get_cloud(self, distro): + self.patchUtils(self.new_root) + paths = helpers.Paths({'templates_dir': self.new_root}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skips_empty_landscape_cloudconfig(self): + """Empty landscape cloud-config section does no work.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'landscape': {}} + cc_landscape.handle('notimportant', cfg, mycloud, LOG, None) + self.assertFalse(mycloud.distro.install_packages.called) + + def test_handler_error_on_invalid_landscape_type(self): + """Raise an error when landscape configuraiton option is invalid.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': 'wrongtype'} + with self.assertRaises(RuntimeError) as context_manager: + cc_landscape.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "'landscape' key existed in config, but not a dict", + str(context_manager.exception)) + + @mock.patch('cloudinit.config.cc_landscape.util') + def test_handler_restarts_landscape_client(self, m_util): + """handler restarts lansdscape-client after install.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(['service', 'landscape-client', 'restart'])], + m_util.subp.call_args_list) + + def test_handler_installs_client_and_creates_config_file(self): + """Write landscape client.conf and install landscape-client.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client'}} + mycloud.distro = mock.MagicMock() + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}, + 'LS_DEFAULT_FILE': {'new': self.default_file}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call('landscape-client')], + mycloud.distro.install_packages.call_args) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) + default_content = util.load_file(self.default_file) + self.assertEqual('RUN=1\n', default_content) + + def test_handler_writes_merged_client_config_file_with_defaults(self): + """Merge and write options from LSC_CLIENT_CFG_FILE with defaults.""" + # Write existing sparse client.conf file + util.write_file(self.conf, '[client]\ncomputer_title = My PC\n') + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client', + 'computer_title': 'My PC'}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) + + def test_handler_writes_merged_provided_cloudconfig_with_defaults(self): + """Merge and write options from cloud-config options with defaults.""" + # Write empty sparse client.conf file + util.write_file(self.conf, '') + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {'computer_title': 'My PC'}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client', + 'computer_title': 'My PC'}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py new file mode 100644 index 00000000..805c76ba --- /dev/null +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -0,0 +1,142 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_puppet +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from ..helpers import CiTestCase, mock + +import logging + + +LOG = logging.getLogger(__name__) + + +@mock.patch('cloudinit.config.cc_puppet.util') +@mock.patch('cloudinit.config.cc_puppet.os') +class TestAutostartPuppet(CiTestCase): + + with_logs = True + + def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): + """Update /etc/default/puppet to autostart if it exists.""" + + def _fake_exists(path): + return path == '/etc/default/puppet' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + self.assertEqual( + [mock.call(['sed', '-i', '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False)], + m_util.subp.call_args_list) + + def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_util): + """If systemctl is present, enable puppet via systemctl.""" + + def _fake_exists(path): + return path == '/bin/systemctl' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + expected_calls = [mock.call( + ['/bin/systemctl', 'enable', 'puppet.service'], capture=False)] + self.assertEqual(expected_calls, m_util.subp.call_args_list) + + def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_util): + """If chkconfig is present, enable puppet via checkcfg.""" + + def _fake_exists(path): + return path == '/sbin/chkconfig' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + expected_calls = [mock.call( + ['/sbin/chkconfig', 'puppet', 'on'], capture=False)] + self.assertEqual(expected_calls, m_util.subp.call_args_list) + + +@mock.patch('cloudinit.config.cc_puppet._autostart_puppet') +class TestPuppetHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestPuppetHandle, self).setUp() + self.new_root = self.tmp_dir() + self.conf = self.tmp_path('puppet.conf') + + def _get_cloud(self, distro): + paths = helpers.Paths({'templates_dir': self.new_root}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skips_missing_puppet_key_in_cloudconfig(self, m_auto): + """Cloud-config containing no 'puppet' key is skipped.""" + mycloud = self._get_cloud('ubuntu') + cfg = {} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "no 'puppet' configuration found", self.logs.getvalue()) + self.assertEqual(0, m_auto.call_count) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_starts_puppet_service(self, m_subp, m_auto): + """Cloud-config 'puppet' configuration starts puppet.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'puppet': {'install': False}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual(1, m_auto.call_count) + self.assertEqual( + [mock.call(['service', 'puppet', 'start'], capture=False)], + m_subp.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_empty_puppet_config_installs_puppet(self, m_subp, m_auto): + """Cloud-config empty 'puppet' configuration installs latest puppet.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', None))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_installs_puppet_on_true(self, m_subp, _): + """Cloud-config with 'puppet' key installs when 'install' is True.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {'install': True}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', None))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_installs_puppet_version(self, m_subp, _): + """Cloud-config 'puppet' configuration can specify a version.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {'version': '3.8'}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', '3.8'))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_updates_puppet_conf(self, m_subp, m_auto): + """When 'conf' is provided update values in PUPPET_CONF_PATH.""" + mycloud = self._get_cloud('ubuntu') + cfg = { + 'puppet': { + 'conf': {'agent': {'server': 'puppetmaster.example.org'}}}} + util.write_file(self.conf, '[agent]\nserver = origpuppet\nother = 3') + puppet_conf_path = 'cloudinit.config.cc_puppet.PUPPET_CONF_PATH' + mycloud.distro = mock.MagicMock() + with mock.patch(puppet_conf_path, self.conf): + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + content = util.load_file(self.conf) + expected = '[agent]\nserver = puppetmaster.example.org\nother = 3\n\n' + self.assertEqual(expected, content) -- cgit v1.2.3 From cbda576a7bbf846710ad55940bf8ca1f2d2194b9 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Fri, 25 Aug 2017 11:13:58 -0400 Subject: suse: Add support for openSUSE and return SLES to a working state. This gets initial opensuse and SLES support back to a working state. Still missing is more complete network file writing and unit tests. --- cloudinit/config/cc_resolv_conf.py | 2 +- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/opensuse.py | 212 +++++++++++++++++++++ cloudinit/distros/sles.py | 160 +--------------- templates/hosts.opensuse.tmpl | 26 +++ templates/hosts.suse.tmpl | 3 - tests/unittests/test_distros/test_opensuse.py | 12 ++ tests/unittests/test_distros/test_sles.py | 12 ++ .../unittests/test_handler/test_handler_locale.py | 12 +- .../test_handler/test_handler_set_hostname.py | 5 +- tox.ini | 16 ++ 11 files changed, 297 insertions(+), 165 deletions(-) create mode 100644 cloudinit/distros/opensuse.py create mode 100644 templates/hosts.opensuse.tmpl create mode 100644 tests/unittests/test_distros/test_opensuse.py create mode 100644 tests/unittests/test_distros/test_sles.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 2548d1f1..9812562a 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -55,7 +55,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -distros = ['fedora', 'rhel', 'sles'] +distros = ['fedora', 'opensuse', 'rhel', 'sles'] def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 1fd48a7b..807b3ea2 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -35,7 +35,7 @@ OSFAMILIES = { 'redhat': ['centos', 'fedora', 'rhel'], 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], - 'suse': ['sles'], + 'suse': ['opensuse', 'sles'], 'arch': ['arch'], } diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py new file mode 100644 index 00000000..a219e9fb --- /dev/null +++ b/cloudinit/distros/opensuse.py @@ -0,0 +1,212 @@ +# Copyright (C) 2017 SUSE LLC +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Author: Robert Schweikert +# Author: Juerg Haefliger +# +# Leaning very heavily on the RHEL and Debian implementation +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import distros + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.distros import net_util +from cloudinit.distros import rhel_util as rhutil +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + clock_conf_fn = '/etc/sysconfig/clock' + hostname_conf_fn = '/etc/HOSTNAME' + init_cmd = ['service'] + locale_conf_fn = '/etc/sysconfig/language' + network_conf_fn = '/etc/sysconfig/network' + network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' + resolve_conf_fn = '/etc/resolv.conf' + route_conf_tpl = '/etc/sysconfig/network/ifroute-%s' + systemd_hostname_conf_fn = '/etc/hostname' + systemd_locale_conf_fn = '/etc/locale.conf' + tz_local_fn = '/etc/localtime' + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + self._runner = helpers.Runners(paths) + self.osfamily = 'suse' + cfg['ssh_svcname'] = 'sshd' + if self.uses_systemd(): + self.init_cmd = ['systemctl'] + cfg['ssh_svcname'] = 'sshd.service' + + def apply_locale(self, locale, out_fn=None): + if self.uses_systemd(): + if not out_fn: + out_fn = self.systemd_locale_conf_fn + locale_cfg = {'LANG': locale} + else: + if not out_fn: + out_fn = self.locale_conf_fn + locale_cfg = {'RC_LANG': locale} + rhutil.update_sysconfig_file(out_fn, locale_cfg) + + def install_packages(self, pkglist): + self.package_command( + 'install', + args='--auto-agree-with-licenses', + pkgs=pkglist + ) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['zypper'] + # No user interaction possible, enable non-interactive mode + cmd.append('--non-interactive') + + # Comand is the operation, such as install + if command == 'upgrade': + command = 'update' + cmd.append(command) + + # args are the arguments to the command, not global options + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, capture=False) + + def set_timezone(self, tz): + tz_file = self._find_tz_file(tz) + if self.uses_systemd(): + # Currently, timedatectl complains if invoked during startup + # so for compatibility, create the link manually. + util.del_file(self.tz_local_fn) + util.sym_link(tz_file, self.tz_local_fn) + else: + # Adjust the sysconfig clock zone setting + clock_cfg = { + 'TIMEZONE': str(tz), + } + rhutil.update_sysconfig_file(self.clock_conf_fn, clock_cfg) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ['refresh'], freq=PER_INSTANCE) + + def _bring_up_interfaces(self, device_names): + if device_names and 'all' in device_names: + raise RuntimeError(('Distro %s can not translate ' + 'the device name "all"') % (self.name)) + return distros.Distro._bring_up_interfaces(self, device_names) + + def _read_hostname(self, filename, default=None): + if self.uses_systemd() and filename.endswith('/previous-hostname'): + return util.load_file(filename).strip() + elif self.uses_systemd(): + (out, _err) = util.subp(['hostname']) + if len(out): + return out + else: + return default + else: + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_system_hostname(self): + if self.uses_systemd(): + host_fn = self.systemd_hostname_conf_fn + else: + host_fn = self.hostname_conf_fn + return (host_fn, self._read_hostname(host_fn)) + + def _write_hostname(self, hostname, out_fn): + if self.uses_systemd() and out_fn.endswith('/previous-hostname'): + util.write_file(out_fn, hostname) + elif self.uses_systemd(): + util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + else: + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(hostname) + util.write_file(out_fn, str(conf), 0o644) + + def _write_network(self, settings): + # Convert debian settings to ifcfg format + entries = net_util.translate_network(settings) + LOG.debug("Translated ubuntu style network settings %s into %s", + settings, entries) + # Make the intermediate format as the suse format... + nameservers = [] + searchservers = [] + dev_names = entries.keys() + for (dev, info) in entries.items(): + net_fn = self.network_script_tpl % (dev) + route_fn = self.route_conf_tpl % (dev) + mode = None + if info.get('auto', None): + mode = 'auto' + else: + mode = 'manual' + bootproto = info.get('bootproto', None) + gateway = info.get('gateway', None) + net_cfg = { + 'BOOTPROTO': bootproto, + 'BROADCAST': info.get('broadcast'), + 'GATEWAY': gateway, + 'IPADDR': info.get('address'), + 'LLADDR': info.get('hwaddress'), + 'NETMASK': info.get('netmask'), + 'STARTMODE': mode, + 'USERCONTROL': 'no' + } + if dev != 'lo': + net_cfg['ETHTOOL_OPTIONS'] = '' + else: + net_cfg['FIREWALL'] = 'no' + rhutil.update_sysconfig_file(net_fn, net_cfg, True) + if gateway and bootproto == 'static': + default_route = 'default %s' % gateway + util.write_file(route_fn, default_route, 0o644) + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + if 'dns-search' in info: + searchservers.extend(info['dns-search']) + if nameservers or searchservers: + rhutil.update_resolve_conf_file(self.resolve_conf_fn, + nameservers, searchservers) + return dev_names + +# vi: ts=4 expandtab diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index dbec2edf..6e336cbf 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -1,167 +1,17 @@ -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2017 SUSE LLC # -# Author: Juerg Haefliger +# Author: Robert Schweikert # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import distros +from cloudinit.distros import opensuse -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros import rhel_util -from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -class Distro(distros.Distro): - clock_conf_fn = '/etc/sysconfig/clock' - locale_conf_fn = '/etc/sysconfig/language' - network_conf_fn = '/etc/sysconfig/network' - hostname_conf_fn = '/etc/HOSTNAME' - network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' - resolve_conf_fn = '/etc/resolv.conf' - tz_local_fn = '/etc/localtime' - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'suse' - - def install_packages(self, pkglist): - self.package_command('install', args='-l', pkgs=pkglist) - - def _write_network(self, settings): - # Convert debian settings to ifcfg format - entries = net_util.translate_network(settings) - LOG.debug("Translated ubuntu style network settings %s into %s", - settings, entries) - # Make the intermediate format as the suse format... - nameservers = [] - searchservers = [] - dev_names = entries.keys() - for (dev, info) in entries.items(): - net_fn = self.network_script_tpl % (dev) - mode = info.get('auto') - if mode and mode.lower() == 'true': - mode = 'auto' - else: - mode = 'manual' - net_cfg = { - 'BOOTPROTO': info.get('bootproto'), - 'BROADCAST': info.get('broadcast'), - 'GATEWAY': info.get('gateway'), - 'IPADDR': info.get('address'), - 'LLADDR': info.get('hwaddress'), - 'NETMASK': info.get('netmask'), - 'STARTMODE': mode, - 'USERCONTROL': 'no' - } - if dev != 'lo': - net_cfg['ETHERDEVICE'] = dev - net_cfg['ETHTOOL_OPTIONS'] = '' - else: - net_cfg['FIREWALL'] = 'no' - rhel_util.update_sysconfig_file(net_fn, net_cfg, True) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - if 'dns-search' in info: - searchservers.extend(info['dns-search']) - if nameservers or searchservers: - rhel_util.update_resolve_conf_file(self.resolve_conf_fn, - nameservers, searchservers) - return dev_names - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - locale_cfg = { - 'RC_LANG': locale, - } - rhel_util.update_sysconfig_file(out_fn, locale_cfg) - - def _write_hostname(self, hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0o644) - - def _read_system_hostname(self): - host_fn = self.hostname_conf_fn - return (host_fn, self._read_hostname(host_fn)) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def _bring_up_interfaces(self, device_names): - if device_names and 'all' in device_names: - raise RuntimeError(('Distro %s can not translate ' - 'the device name "all"') % (self.name)) - return distros.Distro._bring_up_interfaces(self, device_names) - - def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Adjust the sysconfig clock zone setting - clock_cfg = { - 'TIMEZONE': str(tz), - } - rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['zypper'] - # No user interaction possible, enable non-interactive mode - cmd.append('--non-interactive') - - # Comand is the operation, such as install - cmd.append(command) - - # args are the arguments to the command, not global options - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ['refresh'], freq=PER_INSTANCE) +class Distro(opensuse.Distro): + pass # vi: ts=4 expandtab diff --git a/templates/hosts.opensuse.tmpl b/templates/hosts.opensuse.tmpl new file mode 100644 index 00000000..655da3f7 --- /dev/null +++ b/templates/hosts.opensuse.tmpl @@ -0,0 +1,26 @@ +* + This file /etc/cloud/templates/hosts.opensuse.tmpl is only utilized + if enabled in cloud-config. Specifically, in order to enable it + you need to add the following to config: + manage_etc_hosts: True +*# +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in +# /etc/cloud/templates/hosts.opensuse.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.0.1 localhost + +# The following lines are desirable for IPv6 capable hosts +::1 localhost ipv6-localhost ipv6-loopback +fe00::0 ipv6-localnet + +ff00::0 ipv6-mcastprefix +ff02::1 ipv6-allnodes +ff02::2 ipv6-allrouters +ff02::3 ipv6-allhosts + diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl index 399ec9b4..b6082692 100644 --- a/templates/hosts.suse.tmpl +++ b/templates/hosts.suse.tmpl @@ -14,12 +14,9 @@ you need to add the following to config: # # The following lines are desirable for IPv4 capable hosts 127.0.0.1 localhost -127.0.0.1 {{fqdn}} {{hostname}} - # The following lines are desirable for IPv6 capable hosts ::1 localhost ipv6-localhost ipv6-loopback -::1 {{fqdn}} {{hostname}} fe00::0 ipv6-localnet ff00::0 ipv6-mcastprefix diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py new file mode 100644 index 00000000..bdb1d633 --- /dev/null +++ b/tests/unittests/test_distros/test_opensuse.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import CiTestCase + +from . import _get_distro + + +class TestopenSUSE(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("opensuse") + self.assertEqual(distro.osfamily, 'suse') diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py new file mode 100644 index 00000000..c656aacc --- /dev/null +++ b/tests/unittests/test_distros/test_sles.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import CiTestCase + +from . import _get_distro + + +class TestSLES(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("sles") + self.assertEqual(distro.osfamily, 'suse') diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index e9a810c5..aaf6c762 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -49,9 +49,15 @@ class TestLocale(t_help.FilesystemMockingTestCase): } cc = self._get_cloud('sles') cc_locale.handle('cc_locale', cfg, cc, LOG, []) - - contents = util.load_file('/etc/sysconfig/language', decode=False) + if cc.distro.uses_systemd: + locale_conf = cc.distro.systemd_locale_conf_fn + else: + locale_conf = cc.distro.locale_conf_fn + contents = util.load_file(locale_conf, decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) + if cc.distro.uses_systemd(): + self.assertEqual({'LANG': cfg['locale']}, dict(n_cfg)) + else: + self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 4b18de75..8165bf9a 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -70,7 +70,8 @@ class TestHostname(t_help.FilesystemMockingTestCase): cc = cloud.Cloud(ds, paths, {}, distro, None) self.patchUtils(self.tmp) cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) - contents = util.load_file("/etc/HOSTNAME") - self.assertEqual('blah', contents.strip()) + if not distro.uses_systemd(): + contents = util.load_file(distro.hostname_conf_fn) + self.assertEqual('blah', contents.strip()) # vi: ts=4 expandtab diff --git a/tox.ini b/tox.ini index 1e7ca2d3..a17156ce 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,22 @@ deps = six==1.9.0 -r{toxinidir}/test-requirements.txt +[testenv:opensusel42] +basepython = python2.7 +commands = nosetests {posargs:tests/unittests} +deps = + # requirements + argparse==1.3.0 + jinja2==2.8 + PyYAML==3.11 + PrettyTable==0.7.2 + oauthlib==0.7.2 + configobj==5.0.6 + requests==2.11.1 + jsonpatch==1.11 + six==1.9.0 + -r{toxinidir}/test-requirements.txt + [testenv:tip-pycodestyle] commands = {envpython} -m pycodestyle {posargs:cloudinit/ tests/ tools/} deps = pycodestyle -- cgit v1.2.3 From fa266bf8818a08e37cd32a603d076ba2db300124 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 31 Aug 2017 20:01:57 -0600 Subject: upstart: do not package upstart jobs, drop ubuntu-init-switch module. The ubuntu-init-switch module allowed the use to launch an instance that was booted with upstart and have it switch its init system to systemd and then reboot itself. It was only useful for the time period when Ubuntu was transitioning to systemd but only produced images using upstart. Also, do not run setup with --init-system=upstart. This means that by default, debian packages built with packages/bddeb will not have upstart unit files included. No other removal is done here. --- cloudinit/config/cc_ubuntu_init_switch.py | 160 ------------------------------ config/cloud.cfg.tmpl | 3 - doc/rtd/topics/modules.rst | 1 - packages/bddeb | 3 +- packages/debian/dirs | 1 - packages/debian/rules.in | 2 +- setup.py | 2 + tests/cloud_tests/configs/modules/TODO.md | 2 - 8 files changed, 4 insertions(+), 170 deletions(-) delete mode 100644 cloudinit/config/cc_ubuntu_init_switch.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py deleted file mode 100644 index 5dd26901..00000000 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (C) 2014 Canonical Ltd. -# -# Author: Scott Moser -# -# This file is part of cloud-init. See LICENSE file for license information. - -""" -Ubuntu Init Switch ------------------- -**Summary:** reboot system into another init. - -This module provides a way for the user to boot with systemd even if the image -is set to boot with upstart. It should be run as one of the first -``cloud_init_modules``, and will switch the init system and then issue a -reboot. The next boot will come up in the target init system and no action -will be taken. This should be inert on non-ubuntu systems, and also -exit quickly. - -.. note:: - best effort is made, but it's possible this system will break, and probably - won't interact well with any other mechanism you've used to switch the init - system. - -**Internal name:** ``cc_ubuntu_init_switch`` - -**Module frequency:** once per instance - -**Supported distros:** ubuntu - -**Config keys**:: - - init_switch: - target: systemd (can be 'systemd' or 'upstart') - reboot: true (reboot if a change was made, or false to not reboot) -""" - -from cloudinit.distros import ubuntu -from cloudinit import log as logging -from cloudinit.settings import PER_INSTANCE -from cloudinit import util - -import os -import time - -frequency = PER_INSTANCE -REBOOT_CMD = ["/sbin/reboot", "--force"] - -DEFAULT_CONFIG = { - 'init_switch': {'target': None, 'reboot': True} -} - -SWITCH_INIT = """ -#!/bin/sh -# switch_init: [upstart | systemd] - -is_systemd() { - [ "$(dpkg-divert --listpackage /sbin/init)" = "systemd-sysv" ] -} -debug() { echo "$@" 1>&2; } -fail() { echo "$@" 1>&2; exit 1; } - -if [ "$1" = "systemd" ]; then - if is_systemd; then - debug "already systemd, nothing to do" - else - [ -f /lib/systemd/systemd ] || fail "no systemd available"; - dpkg-divert --package systemd-sysv --divert /sbin/init.diverted \\ - --rename /sbin/init - fi - [ -f /sbin/init ] || ln /lib/systemd/systemd /sbin/init -elif [ "$1" = "upstart" ]; then - if is_systemd; then - rm -f /sbin/init - dpkg-divert --package systemd-sysv --rename --remove /sbin/init - else - debug "already upstart, nothing to do." - fi -else - fail "Error. expect 'upstart' or 'systemd'" -fi -""" - -distros = ['ubuntu'] - - -def handle(name, cfg, cloud, log, args): - """Handler method activated by cloud-init.""" - - if not isinstance(cloud.distro, ubuntu.Distro): - log.debug("%s: distro is '%s', not ubuntu. returning", - name, cloud.distro.__class__) - return - - cfg = util.mergemanydict([cfg, DEFAULT_CONFIG]) - target = cfg['init_switch']['target'] - reboot = cfg['init_switch']['reboot'] - - if len(args) != 0: - target = args[0] - if len(args) > 1: - reboot = util.is_true(args[1]) - - if not target: - log.debug("%s: target=%s. nothing to do", name, target) - return - - if not util.which('dpkg'): - log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name) - return - - supported = ('upstart', 'systemd') - if target not in supported: - log.warn("%s: target set to %s, expected one of: %s", - name, target, str(supported)) - - if os.path.exists("/run/systemd/system"): - current = "systemd" - else: - current = "upstart" - - if current == target: - log.debug("%s: current = target = %s. nothing to do", name, target) - return - - try: - util.subp(['sh', '-s', target], data=SWITCH_INIT) - except util.ProcessExecutionError as e: - log.warn("%s: Failed to switch to init '%s'. %s", name, target, e) - return - - if util.is_false(reboot): - log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.", - name, current, target) - return - - try: - log.warn("%s: switched '%s' to '%s'. rebooting.", - name, current, target) - logging.flushLoggers(log) - _fire_reboot(log, wait_attempts=4, initial_sleep=4) - except Exception as e: - util.logexc(log, "Requested reboot did not happen!") - raise - - -def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): - util.subp(REBOOT_CMD) - start = time.time() - wait_time = initial_sleep - for _i in range(0, wait_attempts): - time.sleep(wait_time) - wait_time *= backoff - elapsed = time.time() - start - log.debug("Rebooted, but still running after %s seconds", int(elapsed)) - # If we got here, not good - elapsed = time.time() - start - raise RuntimeError(("Reboot did not happen" - " after %s seconds!") % (int(elapsed))) - -# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index f4b9069b..a537d65a 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -45,9 +45,6 @@ datasource_list: ['ConfigDrive', 'Azure', 'OpenStack', 'Ec2'] # The modules that run in the 'init' stage cloud_init_modules: - migrator -{% if variant in ["ubuntu", "unknown", "debian"] %} - - ubuntu-init-switch -{% endif %} - seed_random - bootcmd - write-files diff --git a/doc/rtd/topics/modules.rst b/doc/rtd/topics/modules.rst index c963c09a..cdb0f419 100644 --- a/doc/rtd/topics/modules.rst +++ b/doc/rtd/topics/modules.rst @@ -50,7 +50,6 @@ Modules .. automodule:: cloudinit.config.cc_ssh_authkey_fingerprints .. automodule:: cloudinit.config.cc_ssh_import_id .. automodule:: cloudinit.config.cc_timezone -.. automodule:: cloudinit.config.cc_ubuntu_init_switch .. automodule:: cloudinit.config.cc_update_etc_hosts .. automodule:: cloudinit.config.cc_update_hostname .. automodule:: cloudinit.config.cc_users_groups diff --git a/packages/bddeb b/packages/bddeb index 609a94fb..7c123548 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -112,8 +112,7 @@ def get_parser(): parser.add_argument("--init-system", dest="init_system", help=("build deb with INIT_SYSTEM=xxx" " (default: %(default)s"), - default=os.environ.get("INIT_SYSTEM", - "upstart,systemd")) + default=os.environ.get("INIT_SYSTEM", "systemd")) parser.add_argument("--release", dest="release", help=("build with changelog referencing RELEASE"), diff --git a/packages/debian/dirs b/packages/debian/dirs index 9a633c60..1315cf8a 100644 --- a/packages/debian/dirs +++ b/packages/debian/dirs @@ -1,6 +1,5 @@ var/lib/cloud usr/bin -etc/init usr/share/doc/cloud etc/cloud lib/udev/rules.d diff --git a/packages/debian/rules.in b/packages/debian/rules.in index 053b7649..b87a5e84 100755 --- a/packages/debian/rules.in +++ b/packages/debian/rules.in @@ -1,6 +1,6 @@ ## template:basic #!/usr/bin/make -f -INIT_SYSTEM ?= upstart,systemd +INIT_SYSTEM ?= systemd export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) PYVER ?= python${pyver} diff --git a/setup.py b/setup.py index 5c65c7fe..7662bd8b 100755 --- a/setup.py +++ b/setup.py @@ -191,6 +191,8 @@ class InitsysInstallData(install): datakeys = [k for k in INITSYS_ROOTS if k.partition(".")[0] == system] for k in datakeys: + if not INITSYS_FILES[k]: + continue self.distribution.data_files.append( (INITSYS_ROOTS[k], INITSYS_FILES[k])) # Force that command to reinitalize (with new file list) diff --git a/tests/cloud_tests/configs/modules/TODO.md b/tests/cloud_tests/configs/modules/TODO.md index d496da95..0b933b3b 100644 --- a/tests/cloud_tests/configs/modules/TODO.md +++ b/tests/cloud_tests/configs/modules/TODO.md @@ -89,8 +89,6 @@ Not applicable to write a test for this as it specifies when something should be ## ssh authkey fingerprints The authkey_hash key does not appear to work. In fact the default claims to be md5, however syslog only shows sha256 -## ubuntu init switch - ## update etc hosts 2016-11-17: Issues with changing /etc/hosts and lxc backend. -- cgit v1.2.3 From 409918f9ba83e45e9bc5cc0b6c589e2fc8ae9b60 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 29 Aug 2017 09:59:20 -0400 Subject: Use /run/cloud-init for tempfile operations. During boot, the usage of /tmp is not safe. In systemd systems, systemd-tmpfiles-clean may run at any point and clear out a temp file while cloud-init is using it. The solution here is to use /run/cloud-init/tmp. LP: #1707222 --- cloudinit/config/cc_bootcmd.py | 3 +- cloudinit/config/cc_chef.py | 3 +- cloudinit/config/cc_snappy.py | 4 +- cloudinit/net/dhcp.py | 3 +- cloudinit/sources/helpers/azure.py | 4 +- cloudinit/temp_utils.py | 93 ++++++++++++++++++++++ cloudinit/util.py | 36 +-------- packages/bddeb | 5 +- .../unittests/test_datasource/test_azure_helper.py | 4 +- tests/unittests/test_net.py | 3 +- 10 files changed, 112 insertions(+), 46 deletions(-) create mode 100644 cloudinit/temp_utils.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 604f93b0..9c0476af 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -37,6 +37,7 @@ specified either as lists or strings. For invocation details, see ``runcmd``. import os from cloudinit.settings import PER_ALWAYS +from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS @@ -49,7 +50,7 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return - with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: + with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 02c70b10..c192dd32 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -71,6 +71,7 @@ import itertools import json import os +from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -303,7 +304,7 @@ def install_chef(cloud, chef_cfg, log): "omnibus_url_retries", default=OMNIBUS_URL_RETRIES)) content = url_helper.readurl(url=url, retries=retries).contents - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd util.write_file(tmpf, content, mode=0o700) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a9682f19..eecb8178 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,11 +63,11 @@ is ``auto``. Options are: from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE +from cloudinit import temp_utils from cloudinit import util import glob import os -import tempfile LOG = logging.getLogger(__name__) @@ -183,7 +183,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # config # Note, however, we do not touch config files on disk. nested_cfg = {'config': {shortname: config}} - (fd, cfg_tmpf) = tempfile.mkstemp() + (fd, cfg_tmpf) = temp_utils.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index c7febc57..c842c839 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -9,6 +9,7 @@ import os import re from cloudinit.net import find_fallback_nic, get_devicelist +from cloudinit import temp_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -47,7 +48,7 @@ def maybe_perform_dhcp_discovery(nic=None): if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') return {} - with util.tempdir(prefix='cloud-init-dhcp-') as tmpdir: + with temp_utils.tempdir(prefix='cloud-init-dhcp-') as tmpdir: return dhcp_discovery(dhclient_path, nic, tmpdir) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index e22409d1..28ed0ae2 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -6,10 +6,10 @@ import os import re import socket import struct -import tempfile import time from cloudinit import stages +from cloudinit import temp_utils from contextlib import contextmanager from xml.etree import ElementTree @@ -111,7 +111,7 @@ class OpenSSLManager(object): } def __init__(self): - self.tmpdir = tempfile.mkdtemp() + self.tmpdir = temp_utils.mkdtemp() self.certificate = None self.generate_certificate() diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py new file mode 100644 index 00000000..0355f19d --- /dev/null +++ b/cloudinit/temp_utils.py @@ -0,0 +1,93 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import contextlib +import errno +import os +import shutil +import tempfile + +_TMPDIR = None +_ROOT_TMPDIR = "/run/cloud-init/tmp" + + +def _tempfile_dir_arg(odir=None): + """Return the proper 'dir' argument for tempfile functions. + + When root, cloud-init will use /run/cloud-init/tmp to avoid + any cleaning that a distro boot might do on /tmp (such as + systemd-tmpfiles-clean). + + If the caller of this function (mkdtemp or mkstemp) was provided + with a 'dir' argument, then that is respected. + + @param odir: original 'dir' arg to 'mkdtemp' or other.""" + + if odir is not None: + return odir + + global _TMPDIR + if _TMPDIR: + return _TMPDIR + + if os.getuid() == 0: + tdir = _ROOT_TMPDIR + else: + tdir = os.environ.get('TMPDIR', '/tmp') + if not os.path.isdir(tdir): + os.makedirs(tdir) + os.chmod(tdir, 0o1777) + + _TMPDIR = tdir + return tdir + + +def ExtendedTemporaryFile(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + fh = tempfile.NamedTemporaryFile(**kwargs) + # Replace its unlink with a quiet version + # that does not raise errors when the + # file to unlink has been unlinked elsewhere.. + + def _unlink_if_exists(path): + try: + os.unlink(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise e + + fh.unlink = _unlink_if_exists + + # Add a new method that will unlink + # right 'now' but still lets the exit + # method attempt to remove it (which will + # not throw due to our del file being quiet + # about files that are not there) + def unlink_now(): + fh.unlink(fh.name) + + setattr(fh, 'unlink_now', unlink_now) + return fh + + +@contextlib.contextmanager +def tempdir(**kwargs): + # This seems like it was only added in python 3.2 + # Make it since its useful... + # See: http://bugs.python.org/file12970/tempdir.patch + tdir = mkdtemp(**kwargs) + try: + yield tdir + finally: + shutil.rmtree(tdir) + + +def mkdtemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkdtemp(**kwargs) + + +def mkstemp(**kwargs): + kwargs['dir'] = _tempfile_dir_arg(kwargs.pop('dir', None)) + return tempfile.mkstemp(**kwargs) + +# vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 609e94c8..ae5cda8d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -30,7 +30,6 @@ import stat import string import subprocess import sys -import tempfile import time from errno import ENOENT, ENOEXEC @@ -45,6 +44,7 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import mergers from cloudinit import safeyaml +from cloudinit import temp_utils from cloudinit import type_utils from cloudinit import url_helper from cloudinit import version @@ -349,26 +349,6 @@ class DecompressionError(Exception): pass -def ExtendedTemporaryFile(**kwargs): - fh = tempfile.NamedTemporaryFile(**kwargs) - # Replace its unlink with a quiet version - # that does not raise errors when the - # file to unlink has been unlinked elsewhere.. - LOG.debug("Created temporary file %s", fh.name) - fh.unlink = del_file - - # Add a new method that will unlink - # right 'now' but still lets the exit - # method attempt to remove it (which will - # not throw due to our del file being quiet - # about files that are not there) - def unlink_now(): - fh.unlink(fh.name) - - setattr(fh, 'unlink_now', unlink_now) - return fh - - def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: @@ -790,18 +770,6 @@ def umask(n_msk): os.umask(old) -@contextlib.contextmanager -def tempdir(**kwargs): - # This seems like it was only added in python 3.2 - # Make it since its useful... - # See: http://bugs.python.org/file12970/tempdir.patch - tdir = tempfile.mkdtemp(**kwargs) - try: - yield tdir - finally: - del_dir(tdir) - - def center(text, fill, max_len): return '{0:{fill}{align}{size}}'.format(text, fill=fill, align="^", size=max_len) @@ -1587,7 +1555,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): mtypes = [''] mounted = mounts() - with tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: umount = False if os.path.realpath(device) in mounted: mountpoint = mounted[os.path.realpath(device)]['mountpoint'] diff --git a/packages/bddeb b/packages/bddeb index 7c123548..4f2e2ddf 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -21,8 +21,9 @@ def find_root(): if "avoid-pep8-E402-import-not-top-of-file": # Use the util functions from cloudinit sys.path.insert(0, find_root()) - from cloudinit import templater from cloudinit import util + from cloudinit import temp_utils + from cloudinit import templater DEBUILD_ARGS = ["-S", "-d"] @@ -148,7 +149,7 @@ def main(): capture = False templ_data = {'debian_release': args.release} - with util.tempdir() as tdir: + with temp_utils.tempdir() as tdir: # output like 0.7.6-1022-g36e92d3 ver_data = read_version() diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 80ce003d..44b99eca 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -275,7 +275,7 @@ class TestOpenSSLManager(TestCase): mock.patch('builtins.open')) @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp') + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp') def test_openssl_manager_creates_a_tmpdir(self, mkdtemp): manager = azure_helper.OpenSSLManager() self.assertEqual(mkdtemp.return_value, manager.tmpdir) @@ -292,7 +292,7 @@ class TestOpenSSLManager(TestCase): manager.clean_up() @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) - @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) + @mock.patch.object(azure_helper.temp_utils, 'mkdtemp', mock.MagicMock()) @mock.patch.object(azure_helper.util, 'del_dir') def test_clean_up(self, del_dir): manager = azure_helper.OpenSSLManager() diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index c10ef905..f2496151 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -9,6 +9,7 @@ from cloudinit.net import network_state from cloudinit.net import renderers from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack +from cloudinit import temp_utils from cloudinit import util from cloudinit.tests.helpers import CiTestCase @@ -2150,7 +2151,7 @@ class TestCmdlineConfigParsing(CiTestCase): static['mac_address'] = macs['eth1'] expected = {'version': 1, 'config': [dhcp, static]} - with util.tempdir() as tmpd: + with temp_utils.tempdir() as tmpd: for fname, content in pairs: fp = os.path.join(tmpd, fname) files.append(fp) -- cgit v1.2.3 From ed8f1b159174715403cb1ffa200ff6d080770152 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Sat, 2 Sep 2017 01:51:29 -0600 Subject: schema and docs: Add jsonschema to resizefs and bootcmd modules Add schema definitions to both cc_resizefs and cc_bootcmd modules. Extend schema.py to parse and document enumerated json types. Schema definitions are used to generate module documention and log warnings for schema infractions. This branch also does the following: - drops vestigial 'resize_rootfs_tmp' option from cc_resizefs. That option only created the specified directory and didn't make use of that directory for any resize operations. - Drop yaml.dumps calls from schema documentation generation to avoid yaml import costs on module load - Add __doc__ = get_schema_doc(schema) definitions it each module to supplement python help() calls for cc_runcmd, cc_bootcmd, cc_ntp and cc_resizefs - Add a SCHEMA_EXAMPLES_SPACER_TEMPLATE string to docs for modules which contain more than one example --- cloudinit/config/cc_bootcmd.py | 87 ++++--- cloudinit/config/cc_ntp.py | 48 +--- cloudinit/config/cc_resizefs.py | 149 +++++++----- cloudinit/config/cc_runcmd.py | 5 +- cloudinit/config/schema.py | 19 +- .../unittests/test_handler/test_handler_bootcmd.py | 145 ++++++++++++ .../test_handler/test_handler_resizefs.py | 257 ++++++++++++++++++++- tests/unittests/test_handler/test_schema.py | 44 ++-- 8 files changed, 586 insertions(+), 168 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_bootcmd.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 9c0476af..233da1ef 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -3,45 +3,73 @@ # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Chad Smith # # This file is part of cloud-init. See LICENSE file for license information. -""" -Bootcmd -------- -**Summary:** run commands early in boot process - -This module runs arbitrary commands very early in the boot process, -only slightly after a boothook would run. This is very similar to a -boothook, but more user friendly. The environment variable ``INSTANCE_ID`` -will be set to the current instance id for all run commands. Commands can be -specified either as lists or strings. For invocation details, see ``runcmd``. - -.. note:: - bootcmd should only be used for things that could not be done later in the - boot process. - -**Internal name:** ``cc_bootcmd`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - bootcmd: - - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts - - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] -""" +"""Bootcmd: run arbitrary commands early in the boot process.""" import os +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +distros = ['all'] + +schema = { + 'id': 'cc_bootcmd', + 'name': 'Bootcmd', + 'title': 'Run arbitrary commands early in the boot process', + 'description': dedent("""\ + This module runs arbitrary commands very early in the boot process, + only slightly after a boothook would run. This is very similar to a + boothook, but more user friendly. The environment variable + ``INSTANCE_ID`` will be set to the current instance id for all run + commands. Commands can be specified either as lists or strings. For + invocation details, see ``runcmd``. + + .. note:: + bootcmd should only be used for things that could not be done later + in the boot process."""), + 'distros': distros, + 'examples': [dedent("""\ + bootcmd: + - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts + - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] + """)], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'bootcmd': { + 'type': 'array', + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject items of non-string non-list + 'additionalProperties': False, + 'minItems': 1, + 'required': [], + 'uniqueItems': True + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): @@ -50,13 +78,14 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return + validate_cloudconfig_schema(cfg, schema) with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except Exception: - util.logexc(log, "Failed to shellify bootcmd") + except Exception as e: + util.logexc(log, "Failed to shellify bootcmd: %s", str(e)) raise try: diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index a02b4bf1..15ae1ecd 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -4,39 +4,10 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -NTP ---- -**Summary:** enable and configure ntp - -Handle ntp configuration. If ntp is not installed on the system and ntp -configuration is specified, ntp will be installed. If there is a default ntp -config file in the image or one is present in the distro's ntp package, it will -be copied to ``/etc/ntp.conf.dist`` before any changes are made. A list of ntp -pools and ntp servers can be provided under the ``ntp`` config key. If no ntp -servers or pools are provided, 4 pools will be used in the format -``{0-3}.{distro}.pool.ntp.org``. - -**Internal name:** ``cc_ntp`` - -**Module frequency:** per instance - -**Supported distros:** centos, debian, fedora, opensuse, ubuntu - -**Config keys**:: - - ntp: - pools: - - 0.company.pool.ntp.org - - 1.company.pool.ntp.org - - ntp.myorg.org - servers: - - my.ntp.server.local - - ntp.ubuntu.com - - 192.168.23.2 -""" +"""NTP: enable and configure ntp""" -from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater @@ -76,10 +47,13 @@ schema = { ``{0-3}.{distro}.pool.ntp.org``."""), 'distros': distros, 'examples': [ - {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org', - 'ntp.myorg.org'], - 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com', - '192.168.23.2']}}], + dedent("""\ + ntp: + pools: [0.int.pool.ntp.org, 1.int.pool.ntp.org, ntp.myorg.org] + servers: + - ntp.server.local + - ntp.ubuntu.com + - 192.168.23.2""")], 'frequency': PER_INSTANCE, 'type': 'object', 'properties': { @@ -117,6 +91,8 @@ schema = { } } +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): """Enable and configure ntp.""" diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index ceee952b..f14d3836 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -6,31 +6,8 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Resizefs --------- -**Summary:** resize filesystem +"""Resizefs: cloud-config module which resizes the filesystem""" -Resize a filesystem to use all avaliable space on partition. This module is -useful along with ``cc_growpart`` and will ensure that if the root partition -has been resized the root filesystem will be resized along with it. By default, -``cc_resizefs`` will resize the root partition and will block the boot process -while the resize command is running. Optionally, the resize operation can be -performed in the background while cloud-init continues running modules. This -can be enabled by setting ``resize_rootfs`` to ``true``. This module can be -disabled altogether by setting ``resize_rootfs`` to ``false``. - -**Internal name:** ``cc_resizefs`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - resize_rootfs: - resize_rootfs_tmp: -""" import errno import getopt @@ -38,11 +15,47 @@ import os import re import shlex import stat +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import util +NOBLOCK = "noblock" + frequency = PER_ALWAYS +distros = ['all'] + +schema = { + 'id': 'cc_resizefs', + 'name': 'Resizefs', + 'title': 'Resize filesystem', + 'description': dedent("""\ + Resize a filesystem to use all avaliable space on partition. This + module is useful along with ``cc_growpart`` and will ensure that if the + root partition has been resized the root filesystem will be resized + along with it. By default, ``cc_resizefs`` will resize the root + partition and will block the boot process while the resize command is + running. Optionally, the resize operation can be performed in the + background while cloud-init continues running modules. This can be + enabled by setting ``resize_rootfs`` to ``true``. This module can be + disabled altogether by setting ``resize_rootfs`` to ``false``."""), + 'distros': distros, + 'examples': [ + 'resize_rootfs: false # disable root filesystem resize operation'], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'resize_rootfs': { + 'enum': [True, False, NOBLOCK], + 'description': dedent("""\ + Whether to resize the root partition. Default: 'true'""") + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() def _resize_btrfs(mount_point, devpth): @@ -131,8 +144,6 @@ RESIZE_FS_PRECHECK_CMDS = { 'ufs': _can_skip_resize_ufs } -NOBLOCK = "noblock" - def rootdev_from_cmdline(cmdline): found = None @@ -161,71 +172,81 @@ def can_skip_resize(fs_type, resize_what, devpth): return False -def handle(name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = args[0] - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if not util.translate_bool(resize_root, addons=[NOBLOCK]): - log.debug("Skipping module named %s, resizing disabled", name) - return - - # TODO(harlowja) is the directory ok to be used?? - resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") - util.ensure_dir(resize_root_d) +def is_device_path_writable_block(devpath, info, log): + """Return True if devpath is a writable block device. - # TODO(harlowja): allow what is to be resized to be configurable?? - resize_what = "/" - result = util.get_mount_info(resize_what, log) - if not result: - log.warn("Could not determine filesystem type of %s", resize_what) - return - - (devpth, fs_type, mount_point) = result - - info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) - log.debug("resize_info: %s" % info) + @param devpath: Path to the root device we want to resize. + @param info: String representing information about the requested device. + @param log: Logger to which logs will be added upon error. + @returns Boolean True if block device is writable + """ container = util.is_container() # Ensure the path is a block device. - if (devpth == "/dev/root" and not os.path.exists(devpth) and + if (devpath == "/dev/root" and not os.path.exists(devpath) and not container): - devpth = util.rootdev_from_cmdline(util.get_cmdline()) - if devpth is None: + devpath = util.rootdev_from_cmdline(util.get_cmdline()) + if devpath is None: log.warn("Unable to find device '/dev/root'") - return - log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth) + return False + log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) try: - statret = os.stat(devpth) + statret = os.stat(devpath) except OSError as exc: if container and exc.errno == errno.ENOENT: log.debug("Device '%s' did not exist in container. " - "cannot resize: %s", devpth, info) + "cannot resize: %s", devpath, info) elif exc.errno == errno.ENOENT: log.warn("Device '%s' did not exist. cannot resize: %s", - devpth, info) + devpath, info) else: raise exc - return + return False - if not os.access(devpth, os.W_OK): + if not os.access(devpath, os.W_OK): if container: log.debug("'%s' not writable in container. cannot resize: %s", - devpth, info) + devpath, info) else: - log.warn("'%s' not writable. cannot resize: %s", devpth, info) + log.warn("'%s' not writable. cannot resize: %s", devpath, info) return if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." - " cannot resize: %s" % (devpth, info)) + " cannot resize: %s" % (devpath, info)) else: log.warn("device '%s' not a block device. cannot resize: %s" % - (devpth, info)) + (devpath, info)) + return False + return True + + +def handle(name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = args[0] + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + validate_cloudconfig_schema(cfg, schema) + if not util.translate_bool(resize_root, addons=[NOBLOCK]): + log.debug("Skipping module named %s, resizing disabled", name) + return + + # TODO(harlowja): allow what is to be resized to be configurable?? + resize_what = "/" + result = util.get_mount_info(resize_what, log) + if not result: + log.warn("Could not determine filesystem type of %s", resize_what) + return + + (devpth, fs_type, mount_point) = result + + info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) + log.debug("resize_info: %s" % info) + + if not is_device_path_writable_block(devpth, info, log): return resizer = None diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 7c3ccd41..7f995693 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -8,7 +8,8 @@ """Runcmd: run arbitrary commands at rc.local with output to the console""" -from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -67,6 +68,8 @@ schema = { } } +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 73dd5c2e..c17d973e 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -14,6 +14,7 @@ import re import sys import yaml +_YAML_MAP = {True: 'true', False: 'false', None: 'null'} SCHEMA_UNDEFINED = b'UNDEFINED' CLOUD_CONFIG_HEADER = b'#cloud-config' SCHEMA_DOC_TMPL = """ @@ -34,6 +35,8 @@ SCHEMA_DOC_TMPL = """ {examples} """ SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' +SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n' +SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n # --- Example{0} ---' class SchemaValidationError(ValueError): @@ -212,6 +215,9 @@ def _schemapath_for_cloudconfig(config, original_content): def _get_property_type(property_dict): """Return a string representing a property type from a given jsonschema.""" property_type = property_dict.get('type', SCHEMA_UNDEFINED) + if property_type == SCHEMA_UNDEFINED and property_dict.get('enum'): + property_type = [ + str(_YAML_MAP.get(k, k)) for k in property_dict['enum']] if isinstance(property_type, list): property_type = '/'.join(property_type) items = property_dict.get('items', {}) @@ -249,15 +255,14 @@ def _get_schema_examples(schema, prefix=''): examples = schema.get('examples') if not examples: return '' - rst_content = '\n**Examples**::\n\n' - for example in examples: - if isinstance(example, str): - example_content = example - else: - example_content = yaml.dump(example, default_flow_style=False) + rst_content = SCHEMA_EXAMPLES_HEADER + for count, example in enumerate(examples): # Python2.6 is missing textwrapper.indent - lines = example_content.split('\n') + lines = example.split('\n') indented_lines = [' {0}'.format(line) for line in lines] + if rst_content != SCHEMA_EXAMPLES_HEADER: + indented_lines.insert( + 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1)) rst_content += '\n'.join(indented_lines) return rst_content diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py new file mode 100644 index 00000000..580017ed --- /dev/null +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -0,0 +1,145 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_bootcmd +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from cloudinit.tests.helpers import CiTestCase, mock, skipIf + +import logging +import tempfile + +try: + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False +except ImportError: + _missing_jsonschema_dep = True + +LOG = logging.getLogger(__name__) + + +class FakeExtendedTempFile(object): + def __init__(self, suffix): + self.suffix = suffix + self.handle = tempfile.NamedTemporaryFile( + prefix="ci-%s." % self.__class__.__name__, delete=False) + + def __enter__(self): + return self.handle + + def __exit__(self, exc_type, exc_value, traceback): + self.handle.close() + + +class TestBootcmd(CiTestCase): + + with_logs = True + + _etmpfile_path = ('cloudinit.config.cc_bootcmd.temp_utils.' + 'ExtendedTemporaryFile') + + def setUp(self): + super(TestBootcmd, self).setUp() + self.subp = util.subp + self.new_root = self.tmp_dir() + + def _get_cloud(self, distro): + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + paths.datasource = myds + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skip_if_no_bootcmd(self): + """When the provided config doesn't contain bootcmd, skip it.""" + cfg = {} + mycloud = self._get_cloud('ubuntu') + cc_bootcmd.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "Skipping module named notimportant, no 'bootcmd' key", + self.logs.getvalue()) + + def test_handler_invalid_command_set(self): + """Commands which can't be converted to shell will raise errors.""" + invalid_config = {'bootcmd': 1} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError) as context_manager: + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + self.assertIn('Failed to shellify bootcmd', self.logs.getvalue()) + self.assertEqual( + "'int' object is not iterable", + str(context_manager.exception)) + + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + def test_handler_schema_validation_warns_non_array_type(self): + """Schema validation warns of non-array type for bootcmd key. + + Schema validation is not strict, so bootcmd attempts to shellify the + invalid content. + """ + invalid_config = {'bootcmd': 1} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError): + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + self.assertIn( + 'Invalid config:\nbootcmd: 1 is not of type \'array\'', + self.logs.getvalue()) + self.assertIn('Failed to shellify', self.logs.getvalue()) + + @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency') + def test_handler_schema_validation_warns_non_array_item_type(self): + """Schema validation warns of non-array or string bootcmd items. + + Schema validation is not strict, so bootcmd attempts to shellify the + invalid content. + """ + invalid_config = { + 'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} + cc = self._get_cloud('ubuntu') + with self.assertRaises(RuntimeError) as context_manager: + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + expected_warnings = [ + 'bootcmd.1: 20 is not valid under any of the given schemas', + 'bootcmd.3: {\'a\': \'n\'} is not valid under any of the given' + ' schema' + ] + logs = self.logs.getvalue() + for warning in expected_warnings: + self.assertIn(warning, logs) + self.assertIn('Failed to shellify', logs) + self.assertEqual( + 'Unable to shellify type int which is not a list or string', + str(context_manager.exception)) + + def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self): + """Valid schema runs a bootcmd script with INSTANCE_ID in the env.""" + cc = self._get_cloud('ubuntu') + out_file = self.tmp_path('bootcmd.out', self.new_root) + my_id = "b6ea0f59-e27d-49c6-9f87-79f19765a425" + valid_config = {'bootcmd': [ + 'echo {0} $INSTANCE_ID > {1}'.format(my_id, out_file)]} + + with mock.patch(self._etmpfile_path, FakeExtendedTempFile): + cc_bootcmd.handle('cc_bootcmd', valid_config, cc, LOG, []) + self.assertEqual(my_id + ' iid-datasource-none\n', + util.load_file(out_file)) + + def test_handler_runs_bootcmd_script_with_error(self): + """When a valid script generates an error, that error is raised.""" + cc = self._get_cloud('ubuntu') + valid_config = {'bootcmd': ['exit 1']} # Script with error + + with mock.patch(self._etmpfile_path, FakeExtendedTempFile): + with self.assertRaises(util.ProcessExecutionError) as ctxt_manager: + cc_bootcmd.handle('does-not-matter', valid_config, cc, LOG, []) + self.assertIn( + 'Unexpected error while running command.\n' + "Command: ['/bin/sh',", + str(ctxt_manager.exception)) + self.assertIn( + 'Failed to run bootcmd module does-not-matter', + self.logs.getvalue()) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 52591b8b..76dddbf8 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -1,17 +1,30 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config import cc_resizefs +from cloudinit.config.cc_resizefs import ( + can_skip_resize, handle, is_device_path_writable_block, + rootdev_from_cmdline) +import logging import textwrap -import unittest + +from cloudinit.tests.helpers import (CiTestCase, mock, skipIf, util, + wrap_and_call) + + +LOG = logging.getLogger(__name__) + try: - from unittest import mock + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False except ImportError: - import mock + _missing_jsonschema_dep = True + +class TestResizefs(CiTestCase): + with_logs = True -class TestResizefs(unittest.TestCase): def setUp(self): super(TestResizefs, self).setUp() self.name = "resizefs" @@ -34,7 +47,7 @@ class TestResizefs(unittest.TestCase): 58720296 3145728 3 freebsd-swap (1.5G) 61866024 1048496 - free - (512M) """) - res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth) + res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') @@ -52,8 +65,238 @@ class TestResizefs(unittest.TestCase): => 34 297086 da0 GPT (145M) 34 297086 1 freebsd-ufs (145M) """) - res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth) + res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) + def test_handle_noops_on_disabled(self): + """The handle function logs when the configuration disables resize.""" + cfg = {'resize_rootfs': False} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + self.assertIn( + 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', + self.logs.getvalue()) + + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + def test_handle_schema_validation_logs_invalid_resize_rootfs_value(self): + """The handle reports json schema violations as a warning. + + Invalid values for resize_rootfs result in disabling the module. + """ + cfg = {'resize_rootfs': 'junk'} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + logs = self.logs.getvalue() + self.assertIn( + "WARNING: Invalid config:\nresize_rootfs: 'junk' is not one of" + " [True, False, 'noblock']", + logs) + self.assertIn( + 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', + logs) + + @mock.patch('cloudinit.config.cc_resizefs.util.get_mount_info') + def test_handle_warns_on_unknown_mount_info(self, m_get_mount_info): + """handle warns when get_mount_info sees unknown filesystem for /.""" + m_get_mount_info.return_value = None + cfg = {'resize_rootfs': True} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + logs = self.logs.getvalue() + self.assertNotIn("WARNING: Invalid config:\nresize_rootfs:", logs) + self.assertIn( + 'WARNING: Could not determine filesystem type of /\n', + logs) + self.assertEqual( + [mock.call('/', LOG)], + m_get_mount_info.call_args_list) + + def test_handle_warns_on_undiscoverable_root_path_in_commandline(self): + """handle noops when the root path is not found on the commandline.""" + cfg = {'resize_rootfs': True} + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + + def fake_mount_info(path, log): + self.assertEqual('/', path) + self.assertEqual(LOG, log) + return ('/dev/root', 'ext4', '/') + + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'get_mount_info': {'side_effect': fake_mount_info}, + 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, + handle, 'cc_resizefs', cfg, _cloud=None, log=LOG, + args=[]) + logs = self.logs.getvalue() + self.assertIn("WARNING: Unable to find device '/dev/root'", logs) + + +class TestRootDevFromCmdline(CiTestCase): + + def test_rootdev_from_cmdline_with_no_root(self): + """Return None from rootdev_from_cmdline when root is not present.""" + invalid_cases = [ + 'BOOT_IMAGE=/adsf asdfa werasef root adf', 'BOOT_IMAGE=/adsf', ''] + for case in invalid_cases: + self.assertIsNone(rootdev_from_cmdline(case)) + + def test_rootdev_from_cmdline_with_root_startswith_dev(self): + """Return the cmdline root when the path starts with /dev.""" + self.assertEqual( + '/dev/this', rootdev_from_cmdline('asdf root=/dev/this')) + + def test_rootdev_from_cmdline_with_root_without_dev_prefix(self): + """Add /dev prefix to cmdline root when the path lacks the prefix.""" + self.assertEqual('/dev/this', rootdev_from_cmdline('asdf root=this')) + + def test_rootdev_from_cmdline_with_root_with_label(self): + """When cmdline root contains a LABEL, our root is disk/by-label.""" + self.assertEqual( + '/dev/disk/by-label/unique', + rootdev_from_cmdline('asdf root=LABEL=unique')) + + def test_rootdev_from_cmdline_with_root_with_uuid(self): + """When cmdline root contains a UUID, our root is disk/by-uuid.""" + self.assertEqual( + '/dev/disk/by-uuid/adsfdsaf-adsf', + rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf')) + + +class TestIsDevicePathWritableBlock(CiTestCase): + + with_logs = True + + def test_is_device_path_writable_block_warns_missing_cmdline_root(self): + """When root does not exist isn't in the cmdline, log warning.""" + info = 'does not matter' + + def fake_mount_info(path, log): + self.assertEqual('/', path) + self.assertEqual(LOG, log) + return ('/dev/root', 'ext4', '/') + + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'get_mount_info': {'side_effect': fake_mount_info}, + 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, + is_device_path_writable_block, '/dev/root', info, LOG) + self.assertFalse(is_valid) + logs = self.logs.getvalue() + self.assertIn("WARNING: Unable to find device '/dev/root'", logs) + + def test_is_device_path_writable_block_does_not_exist(self): + """When devpath does not exist, a warning is logged.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "WARNING: Device '/I/dont/exist' did not exist." + ' cannot resize: %s' % info, + self.logs.getvalue()) + + def test_is_device_path_writable_block_does_not_exist_in_container(self): + """When devpath does not exist in a container, log a debug message.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: Device '/I/dont/exist' did not exist in container." + ' cannot resize: %s' % info, + self.logs.getvalue()) + + def test_is_device_path_writable_block_raises_oserror(self): + """When unexpected OSError is raises by os.stat it is reraised.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + with self.assertRaises(OSError) as context_manager: + wrap_and_call( + 'cloudinit.config.cc_resizefs', + {'util.is_container': {'return_value': True}, + 'os.stat': {'side_effect': OSError('Something unexpected')}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertEqual( + 'Something unexpected', str(context_manager.exception)) + + def test_is_device_path_writable_block_readonly(self): + """When root device is readonly, emit a warning and return False.""" + fake_devpath = self.tmp_path('dev/readonly') + util.write_file(fake_devpath, '', mode=0o400) # read-only + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'rootdev_from_cmdline': {'return_value': fake_devpath}}, + is_device_path_writable_block, '/dev/root', info, LOG) + self.assertFalse(is_valid) + logs = self.logs.getvalue() + self.assertIn( + "Converted /dev/root to '{0}' per kernel cmdline".format( + fake_devpath), + logs) + self.assertIn( + "WARNING: '{0}' not writable. cannot resize".format(fake_devpath), + logs) + + def test_is_device_path_writable_block_readonly_in_container(self): + """When root device is readonly, emit debug log and return False.""" + fake_devpath = self.tmp_path('dev/readonly') + util.write_file(fake_devpath, '', mode=0o400) # read-only + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: '{0}' not writable in container. cannot resize".format( + fake_devpath), + self.logs.getvalue()) + + def test_is_device_path_writable_block_non_block(self): + """When device is not a block device, emit warning return False.""" + fake_devpath = self.tmp_path('dev/readwrite') + util.write_file(fake_devpath, '', mode=0o600) # read-write + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "WARNING: device '{0}' not a block device. cannot resize".format( + fake_devpath), + self.logs.getvalue()) + + def test_is_device_path_writable_block_non_block_on_container(self): + """When device is non-block device in container, emit debug log.""" + fake_devpath = self.tmp_path('dev/readwrite') + util.write_file(fake_devpath, '', mode=0o600) # read-write + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: device '{0}' not a block device in container." + ' cannot resize'.format(fake_devpath), + self.logs.getvalue()) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 6137e3cf..745bb0ff 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,7 +27,7 @@ class GetSchemaTest(CiTestCase): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() self.assertItemsEqual( - ['cc_ntp', 'cc_runcmd'], + ['cc_bootcmd', 'cc_ntp', 'cc_resizefs', 'cc_runcmd'], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) self.assertEqual( @@ -205,6 +205,17 @@ class GetSchemaDocTest(CiTestCase): '**prop1:** (string/integer) prop-description', get_schema_doc(full_schema)) + def test_get_schema_doc_handles_enum_types(self): + """get_schema_doc converts enum types to yaml and delimits with '/'.""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'prop1': {'enum': [True, False, 'stuff'], + 'description': 'prop-description'}}}) + self.assertIn( + '**prop1:** (true/false/stuff) prop-description', + get_schema_doc(full_schema)) + def test_get_schema_doc_handles_nested_oneof_property_types(self): """get_schema_doc describes array items oneOf declarations in type.""" full_schema = copy(self.required_schema) @@ -219,29 +230,11 @@ class GetSchemaDocTest(CiTestCase): '**prop1:** (array of (string)/(integer)) prop-description', get_schema_doc(full_schema)) - def test_get_schema_doc_returns_restructured_text_with_examples(self): - """get_schema_doc returns indented examples when present in schema.""" - full_schema = copy(self.required_schema) - full_schema.update( - {'examples': [{'ex1': [1, 2, 3]}], - 'properties': { - 'prop1': {'type': 'array', 'description': 'prop-description', - 'items': {'type': 'integer'}}}}) - self.assertIn( - dedent(""" - **Config schema**: - **prop1:** (array of integer) prop-description - - **Examples**:: - - ex1"""), - get_schema_doc(full_schema)) - - def test_get_schema_doc_handles_unstructured_examples(self): - """get_schema_doc properly indented examples which as just strings.""" + def test_get_schema_doc_handles_string_examples(self): + """get_schema_doc properly indented examples as a list of strings.""" full_schema = copy(self.required_schema) full_schema.update( - {'examples': ['My example:\n [don\'t, expand, "this"]'], + {'examples': ['ex1:\n [don\'t, expand, "this"]', 'ex2: true'], 'properties': { 'prop1': {'type': 'array', 'description': 'prop-description', 'items': {'type': 'integer'}}}}) @@ -252,8 +245,11 @@ class GetSchemaDocTest(CiTestCase): **Examples**:: - My example: - [don't, expand, "this"]"""), + ex1: + [don't, expand, "this"] + # --- Example2 --- + ex2: true + """), get_schema_doc(full_schema)) def test_get_schema_doc_raises_key_errors(self): -- cgit v1.2.3 From cf10a2ff2e2f666d9370f38297a5a105e809ea3c Mon Sep 17 00:00:00 2001 From: Ethan Apodaca Date: Wed, 13 Sep 2017 22:18:26 -0600 Subject: chef: Add option to pin chef omnibus install version Most users of chef will want to pin the version that is installed. Typically new versions of chef have to be evaluated for breakage etc. This change proposes a new optional `omnibus_version` field to the chef configuration. The changeset also adds documentation referencing the new field. LP: #1462693 --- cloudinit/config/cc_chef.py | 45 ++++++++---- cloudinit/util.py | 25 +++++++ doc/examples/cloud-config-chef.txt | 4 ++ tests/unittests/test_handler/test_handler_chef.py | 88 +++++++++++++++++++---- 4 files changed, 138 insertions(+), 24 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index c192dd32..46abedd1 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,6 +58,9 @@ file). log_level: log_location: node_name: + omnibus_url: + omnibus_url_retries: + omnibus_version: pid_file: server_url: show_time: @@ -71,7 +74,6 @@ import itertools import json import os -from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -280,6 +282,31 @@ def run_chef(chef_cfg, log): util.subp(cmd, capture=False) +def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): + """Install an omnibus unified package from url. + + @param url: URL where blob of chef content may be downloaded. Defaults to + OMNIBUS_URL. + @param retries: Number of retries to perform when attempting to read url. + Defaults to OMNIBUS_URL_RETRIES + @param omnibus_version: Optional version string to require for omnibus + install. + """ + if url is None: + url = OMNIBUS_URL + if retries is None: + retries = OMNIBUS_URL_RETRIES + + if omnibus_version is None: + args = [] + else: + args = ['-v', omnibus_version] + content = url_helper.readurl(url=url, retries=retries).contents + return util.subp_blob_in_tempfile( + blob=content, args=args, + basename='chef-omnibus-install', capture=False) + + def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', @@ -298,17 +325,11 @@ def install_chef(cloud, chef_cfg, log): # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) elif install_type == 'omnibus': - # This will install as a omnibus unified package - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - retries = max(0, util.get_cfg_option_int(chef_cfg, - "omnibus_url_retries", - default=OMNIBUS_URL_RETRIES)) - content = url_helper.readurl(url=url, retries=retries).contents - with temp_utils.tempdir() as tmpd: - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, content, mode=0o700) - util.subp([tmpf], capture=False) + omnibus_version = util.get_cfg_option_str(chef_cfg, "omnibus_version") + install_chef_from_omnibus( + url=util.get_cfg_option_str(chef_cfg, "omnibus_url"), + retries=util.get_cfg_option_int(chef_cfg, "omnibus_url_retries"), + omnibus_version=omnibus_version) else: log.warn("Unknown chef install type '%s'", install_type) run = False diff --git a/cloudinit/util.py b/cloudinit/util.py index ae5cda8d..7e9d94fc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1742,6 +1742,31 @@ def delete_dir_contents(dirname): del_file(node_fullpath) +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir() as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + write_file(tmpf, blob, mode=0o700) + return subp(*args, **kwargs) + + def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, logstring=False, decode="replace", target=None, update_env=None): diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 9d235817..58d5fdc7 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -94,6 +94,10 @@ chef: # if install_type is 'omnibus', change the url to download omnibus_url: "https://www.chef.io/chef/install.sh" + # if install_type is 'omnibus', pass pinned version string + # to the install script + omnibus_version: "12.3.0" + # Capture all subprocess output into a logfile # Useful for troubleshooting cloud-init issues diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index e5785cfd..0136a93d 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,11 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +import httpretty import json import logging import os -import shutil import six -import tempfile from cloudinit import cloud from cloudinit.config import cc_chef @@ -14,18 +13,83 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import ( + CiTestCase, FilesystemMockingTestCase, mock, skipIf) LOG = logging.getLogger(__name__) CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) -class TestChef(t_help.FilesystemMockingTestCase): +class TestInstallChefOmnibus(CiTestCase): + + def setUp(self): + self.new_root = self.tmp_dir() + + @httpretty.activate + def test_install_chef_from_omnibus_runs_chef_url_content(self): + """install_chef_from_omnibus runs downloaded OMNIBUS_URL as script.""" + chef_outfile = self.tmp_path('chef.out', self.new_root) + response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + httpretty.register_uri( + httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200) + cc_chef.install_chef_from_omnibus() + self.assertEqual('Hi Mom\n', util.load_file(chef_outfile)) + + @mock.patch('cloudinit.config.cc_chef.url_helper.readurl') + @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl): + """install_chef_from_omnibus retries OMNIBUS_URL upon failure.""" + + class FakeURLResponse(object): + contents = '#!/bin/bash\necho "Hi Mom" > {0}/chef.out'.format( + self.new_root) + + m_rdurl.return_value = FakeURLResponse() + + cc_chef.install_chef_from_omnibus() + expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, + 'url': cc_chef.OMNIBUS_URL} + self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + cc_chef.install_chef_from_omnibus(retries=10) + expected_kwargs = {'retries': 10, + 'url': cc_chef.OMNIBUS_URL} + self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + expected_subp_kwargs = { + 'args': ['-v', '2.0'], + 'basename': 'chef-omnibus-install', + 'blob': m_rdurl.return_value.contents, + 'capture': False + } + self.assertItemsEqual( + expected_subp_kwargs, + m_subp_blob.call_args_list[0][1]) + + @httpretty.activate + @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): + """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" + chef_outfile = self.tmp_path('chef.out', self.new_root) + response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + httpretty.register_uri( + httpretty.GET, cc_chef.OMNIBUS_URL, body=response) + cc_chef.install_chef_from_omnibus(omnibus_version='2.0') + + called_kwargs = m_subp_blob.call_args_list[0][1] + expected_kwargs = { + 'args': ['-v', '2.0'], + 'basename': 'chef-omnibus-install', + 'blob': response, + 'capture': False + } + self.assertItemsEqual(expected_kwargs, called_kwargs) + + +class TestChef(FilesystemMockingTestCase): + def setUp(self): super(TestChef, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) + self.tmp = self.tmp_dir() def fetch_cloud(self, distro_kind): cls = distros.fetch(distro_kind) @@ -43,8 +107,8 @@ class TestChef(t_help.FilesystemMockingTestCase): for d in cc_chef.CHEF_DIRS: self.assertFalse(os.path.isdir(d)) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_basic_config(self): """ test basic config looks sane @@ -122,8 +186,8 @@ class TestChef(t_help.FilesystemMockingTestCase): 'c': 'd', }, json.loads(c)) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_template_deletes(self): tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) @@ -143,8 +207,8 @@ class TestChef(t_help.FilesystemMockingTestCase): self.assertNotIn('json_attribs', c) self.assertNotIn('Formatter.show_time', c) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_validation_cert_and_validation_key(self): # test validation_cert content is written to validation_key path tpl_file = util.load_file('templates/chef_client.rb.tmpl') -- cgit v1.2.3 From f761f2b5f58c8cf13cfee63619f32046216cf66a Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 13 Sep 2017 22:26:03 -0600 Subject: cloud-config modules: honor distros definitions in each module Modules can optionally define a list of supported distros on which they can run by declaring a distros attribute in the cc_*py module. This branch fixes handling of cloudinit.stages.Modules.run_section. The behavior of run_section is now the following: - always run a module if the module doesn't declare a distros attribute - always run a module if the module declares distros = [ALL_DISTROS] - skip a module if the distribution on which we run isn't in module.distros - force a run of a skipped module if unverified_modules configuration contains the module name LP: #1715738 LP: #1715690 --- cloudinit/config/cc_runcmd.py | 3 +- cloudinit/distros/__init__.py | 4 + cloudinit/stages.py | 33 ++++--- tests/unittests/test_runs/test_simple_run.py | 125 ++++++++++++++++++++++----- 4 files changed, 131 insertions(+), 34 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 7f995693..449872f0 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -10,6 +10,7 @@ from cloudinit.config.schema import ( get_schema_doc, validate_cloudconfig_schema) +from cloudinit.distros import ALL_DISTROS from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -23,7 +24,7 @@ from textwrap import dedent # configuration options before actually attempting to deploy with said # configuration. -distros = ['all'] +distros = [ALL_DISTROS] schema = { 'id': 'cc_runcmd', diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index b714b9ab..d5becd12 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -30,6 +30,10 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +# Used when a cloud-config module can be run on all cloud-init distibutions. +# The value 'all' is surfaced in module documentation for distro support. +ALL_DISTROS = 'all' + OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['centos', 'fedora', 'rhel'], diff --git a/cloudinit/stages.py b/cloudinit/stages.py index a1c4a517..d0452688 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -821,28 +821,35 @@ class Modules(object): skipped = [] forced = [] overridden = self.cfg.get('unverified_modules', []) + active_mods = [] + all_distros = set([distros.ALL_DISTROS]) for (mod, name, _freq, _args) in mostly_mods: - worked_distros = set(mod.distros) + worked_distros = set(mod.distros) # Minimally [] per fixup_modules worked_distros.update( distros.Distro.expand_osfamily(mod.osfamilies)) - # module does not declare 'distros' or lists this distro - if not worked_distros or d_name in worked_distros: - continue - - if name in overridden: - forced.append(name) - else: - skipped.append(name) + # Skip only when the following conditions are all met: + # - distros are defined in the module != ALL_DISTROS + # - the current d_name isn't in distros + # - and the module is unverified and not in the unverified_modules + # override list + if worked_distros and worked_distros != all_distros: + if d_name not in worked_distros: + if name not in overridden: + skipped.append(name) + continue + forced.append(name) + active_mods.append([mod, name, _freq, _args]) if skipped: - LOG.info("Skipping modules %s because they are not verified " + LOG.info("Skipping modules '%s' because they are not verified " "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) + "'unverified_modules' in config.", + ','.join(skipped), d_name) if forced: - LOG.info("running unverified_modules: %s", forced) + LOG.info("running unverified_modules: '%s'", ', '.join(forced)) - return self._run_modules(mostly_mods) + return self._run_modules(active_mods) def read_runtime_config(): diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 5cf666fe..b8fb4794 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -1,8 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. import os -import shutil -import tempfile from cloudinit.tests import helpers @@ -12,16 +10,19 @@ from cloudinit import util class TestSimpleRun(helpers.FilesystemMockingTestCase): - def _patchIn(self, root): - self.patchOS(root) - self.patchUtils(root) - - def test_none_ds(self): - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.replicateTestRoot('simple_ubuntu', new_root) - cfg = { + + with_logs = True + + def setUp(self): + super(TestSimpleRun, self).setUp() + self.new_root = self.tmp_dir() + self.replicateTestRoot('simple_ubuntu', self.new_root) + + # Seed cloud.cfg file for our tests + self.cfg = { 'datasource_list': ['None'], + 'runcmd': ['ls /etc'], # test ALL_DISTROS + 'spacewalk': {}, # test non-ubuntu distros module definition 'write_files': [ { 'path': '/etc/blah.ini', @@ -29,14 +30,17 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): 'permissions': 0o755, }, ], - 'cloud_init_modules': ['write-files'], + 'cloud_init_modules': ['write-files', 'spacewalk', 'runcmd'], } - cloud_cfg = util.yaml_dumps(cfg) - util.ensure_dir(os.path.join(new_root, 'etc', 'cloud')) - util.write_file(os.path.join(new_root, 'etc', + cloud_cfg = util.yaml_dumps(self.cfg) + util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) + util.write_file(os.path.join(self.new_root, 'etc', 'cloud', 'cloud.cfg'), cloud_cfg) - self._patchIn(new_root) + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + def test_none_ds_populates_var_lib_cloud(self): + """Init and run_section default behavior creates appropriate dirs.""" # Now start verifying whats created initer = stages.Init() initer.read_cfg() @@ -51,10 +55,16 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) - initer.cloudify().run('consume_data', - initer.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) + def test_none_ds_runs_modules_which_do_not_define_distros(self): + """Any modules which do not define a distros attribute are run.""" + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') @@ -63,5 +73,80 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') self.assertEqual(contents, 'blah') + self.assertNotIn( + "Skipping modules ['write-files'] because they are not verified on" + " distro 'ubuntu'", + self.logs.getvalue()) + + def test_none_ds_skips_modules_which_define_unmatched_distros(self): + """Skip modules which define distros which don't match the current.""" + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertIn( + "Skipping modules 'spacewalk' because they are not verified on" + " distro 'ubuntu'", + self.logs.getvalue()) + self.assertNotIn('spacewalk', which_ran) + + def test_none_ds_runs_modules_which_distros_all(self): + """Skip modules which define distros attribute as supporting 'all'. + + This is done in the module with the declaration: + distros = [ALL_DISTROS]. runcmd is an example. + """ + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertIn('runcmd', which_ran) + self.assertNotIn( + "Skipping modules 'runcmd' because they are not verified on" + " distro 'ubuntu'", + self.logs.getvalue()) + + def test_none_ds_forces_run_via_unverified_modules(self): + """run_section forced skipped modules by using unverified_modules.""" + + # re-write cloud.cfg with unverified_modules override + self.cfg['unverified_modules'] = ['spacewalk'] # Would have skipped + cloud_cfg = util.yaml_dumps(self.cfg) + util.ensure_dir(os.path.join(self.new_root, 'etc', 'cloud')) + util.write_file(os.path.join(self.new_root, 'etc', + 'cloud', 'cloud.cfg'), cloud_cfg) + + initer = stages.Init() + initer.read_cfg() + initer.initialize() + initer.fetch() + initer.instancify() + initer.update() + initer.cloudify().run('consume_data', initer.consume_data, + args=[PER_INSTANCE], freq=PER_INSTANCE) + + mods = stages.Modules(initer) + (which_ran, failures) = mods.run_section('cloud_init_modules') + self.assertTrue(len(failures) == 0) + self.assertIn('spacewalk', which_ran) + self.assertIn( + "running unverified_modules: 'spacewalk'", + self.logs.getvalue()) # vi: ts=4 expandtab -- cgit v1.2.3 From 29a9296cd68516e76d0bd8da320754a222c4ee45 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Wed, 13 Sep 2017 11:08:56 -0400 Subject: resizefs: pass mount point to xfs_growfs Supposedly it was never a feature to be able to pass a path to a block device to xfs_growfs and have it grow the filesystem. The behavior changed upstream recently. It is only supported to pass the mount point of a mounted XFS filesystem. This causes breakages in cloud-init. Upstream xfs change was commit b97815a0321072a7154ecab63e297af84066fc78. https://git.kernel.org/pub/scm/fs/xfs/xfsprogs-dev.git/commit/?id=b97815a0321 rhbz: rhbz: https://bugzilla.redhat.com/show_bug.cgi?id=1490505 Signed-off-by: Dusty Mabe --- cloudinit/config/cc_resizefs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index f14d3836..f42b6a63 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -67,7 +67,7 @@ def _resize_ext(mount_point, devpth): def _resize_xfs(mount_point, devpth): - return ('xfs_growfs', devpth) + return ('xfs_growfs', mount_point) def _resize_ufs(mount_point, devpth): -- cgit v1.2.3 From 024ecc1f758d36cc0aa5ebce65704eed6bd66d45 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Sep 2017 14:05:13 -0600 Subject: resizefs: Drop check for read-only device file, do not warn on overlayroot. As root user, os.access(, os.W_OK) will always return True so that path will never get executed. Also avoid a warning if the root is overlayroot, which is the common case on a MAAS booted 'ephemeral' system. --- cloudinit/config/cc_resizefs.py | 12 ++-- .../test_handler/test_handler_resizefs.py | 72 +++++++--------------- 2 files changed, 26 insertions(+), 58 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index f42b6a63..f774baa3 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -192,6 +192,10 @@ def is_device_path_writable_block(devpath, info, log): return False log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) + if devpath == 'overlayroot': + log.debug("Not attempting to resize devpath '%s': %s", devpath, info) + return False + try: statret = os.stat(devpath) except OSError as exc: @@ -205,14 +209,6 @@ def is_device_path_writable_block(devpath, info, log): raise exc return False - if not os.access(devpath, os.W_OK): - if container: - log.debug("'%s' not writable in container. cannot resize: %s", - devpath, info) - else: - log.warn("'%s' not writable. cannot resize: %s", devpath, info) - return - if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 76dddbf8..3e5d436c 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -166,6 +166,18 @@ class TestIsDevicePathWritableBlock(CiTestCase): with_logs = True + def test_is_device_path_writable_block_false_on_overlayroot(self): + """When devpath is overlayroot (on MAAS), is_dev_writable is False.""" + info = 'does not matter' + is_writable = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, 'overlayroot', info, LOG) + self.assertFalse(is_writable) + self.assertIn( + "Not attempting to resize devpath 'overlayroot'", + self.logs.getvalue()) + def test_is_device_path_writable_block_warns_missing_cmdline_root(self): """When root does not exist isn't in the cmdline, log warning.""" info = 'does not matter' @@ -178,24 +190,24 @@ class TestIsDevicePathWritableBlock(CiTestCase): exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' with mock.patch(exists_mock_path) as m_exists: m_exists.return_value = False - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}, 'get_mount_info': {'side_effect': fake_mount_info}, 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, is_device_path_writable_block, '/dev/root', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) logs = self.logs.getvalue() self.assertIn("WARNING: Unable to find device '/dev/root'", logs) def test_is_device_path_writable_block_does_not_exist(self): """When devpath does not exist, a warning is logged.""" info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}}, is_device_path_writable_block, '/I/dont/exist', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "WARNING: Device '/I/dont/exist' did not exist." ' cannot resize: %s' % info, @@ -204,11 +216,11 @@ class TestIsDevicePathWritableBlock(CiTestCase): def test_is_device_path_writable_block_does_not_exist_in_container(self): """When devpath does not exist in a container, log a debug message.""" info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': True}}, is_device_path_writable_block, '/I/dont/exist', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "DEBUG: Device '/I/dont/exist' did not exist in container." ' cannot resize: %s' % info, @@ -226,57 +238,17 @@ class TestIsDevicePathWritableBlock(CiTestCase): self.assertEqual( 'Something unexpected', str(context_manager.exception)) - def test_is_device_path_writable_block_readonly(self): - """When root device is readonly, emit a warning and return False.""" - fake_devpath = self.tmp_path('dev/readonly') - util.write_file(fake_devpath, '', mode=0o400) # read-only - info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - - exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' - with mock.patch(exists_mock_path) as m_exists: - m_exists.return_value = False - is_valid = wrap_and_call( - 'cloudinit.config.cc_resizefs.util', - {'is_container': {'return_value': False}, - 'rootdev_from_cmdline': {'return_value': fake_devpath}}, - is_device_path_writable_block, '/dev/root', info, LOG) - self.assertFalse(is_valid) - logs = self.logs.getvalue() - self.assertIn( - "Converted /dev/root to '{0}' per kernel cmdline".format( - fake_devpath), - logs) - self.assertIn( - "WARNING: '{0}' not writable. cannot resize".format(fake_devpath), - logs) - - def test_is_device_path_writable_block_readonly_in_container(self): - """When root device is readonly, emit debug log and return False.""" - fake_devpath = self.tmp_path('dev/readonly') - util.write_file(fake_devpath, '', mode=0o400) # read-only - info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - - is_valid = wrap_and_call( - 'cloudinit.config.cc_resizefs.util', - {'is_container': {'return_value': True}}, - is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) - self.assertIn( - "DEBUG: '{0}' not writable in container. cannot resize".format( - fake_devpath), - self.logs.getvalue()) - def test_is_device_path_writable_block_non_block(self): """When device is not a block device, emit warning return False.""" fake_devpath = self.tmp_path('dev/readwrite') util.write_file(fake_devpath, '', mode=0o600) # read-write info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}}, is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "WARNING: device '{0}' not a block device. cannot resize".format( fake_devpath), @@ -288,11 +260,11 @@ class TestIsDevicePathWritableBlock(CiTestCase): util.write_file(fake_devpath, '', mode=0o600) # read-write info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': True}}, is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "DEBUG: device '{0}' not a block device in container." ' cannot resize'.format(fake_devpath), -- cgit v1.2.3 From 27613443139578dd1b968d1f5f953bd2bc6245a4 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 20 Sep 2017 23:41:22 -0600 Subject: docs: fix sphinx module schema documentation Create a copy of each modules schema attribute when generating sphinx docs to avoid altering the actual module dict in memory. This avoids illegible rendering of module examples and distros where each character of a list was represented on a separate line by itself. Fixes ntp, resizefs, runcmd and bootcmd docs. --- cloudinit/config/schema.py | 12 +++++++----- tests/unittests/test_cli.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index c17d973e..bb291ff8 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -8,6 +8,7 @@ from cloudinit.util import find_modules, read_file_or_url import argparse from collections import defaultdict +from copy import deepcopy import logging import os import re @@ -273,12 +274,13 @@ def get_schema_doc(schema): @param schema: Dict of jsonschema to render. @raise KeyError: If schema lacks an expected key. """ - schema['property_doc'] = _get_property_doc(schema) - schema['examples'] = _get_schema_examples(schema) - schema['distros'] = ', '.join(schema['distros']) + schema_copy = deepcopy(schema) + schema_copy['property_doc'] = _get_property_doc(schema) + schema_copy['examples'] = _get_schema_examples(schema) + schema_copy['distros'] = ', '.join(schema['distros']) # Need an underbar of the same length as the name - schema['title_underbar'] = re.sub(r'.', '-', schema['name']) - return SCHEMA_DOC_TMPL.format(**schema) + schema_copy['title_underbar'] = re.sub(r'.', '-', schema['name']) + return SCHEMA_DOC_TMPL.format(**schema_copy) FULL_SCHEMA = None diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 258a9f08..fccbbd23 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -125,6 +125,21 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): 'Expected either --config-file argument or --doc\n', self.stderr.getvalue()) + def test_wb_devel_schema_subcommand_doc_content(self): + """Validate that doc content is sane from known examples.""" + stdout = six.StringIO() + self.patchStdoutAndStderr(stdout=stdout) + self._call_main(['cloud-init', 'devel', 'schema', '--doc']) + expected_doc_sections = [ + '**Supported distros:** all', + '**Supported distros:** centos, debian, fedora', + '**Config schema**:\n **resize_rootfs:** (true/false/noblock)', + '**Examples**::\n\n runcmd:\n - [ ls, -l, / ]\n' + ] + stdout = stdout.getvalue() + for expected in expected_doc_sections: + self.assertIn(expected, stdout) + @mock.patch('cloudinit.cmd.main.main_single') def test_single_subcommand(self, m_main_single): """The subcommand 'single' calls main_single with valid args.""" -- cgit v1.2.3 From f010594beb75e146091db47b7d72d1fc1d763e98 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Mon, 2 Oct 2017 12:53:56 -0600 Subject: Remove prettytable dependency, introduce simpletable The first revision of this rendered tables with less decoration but there was a desire upstream to avoid possibly breaking some parsing someone might be doing, so it has been revised to render the same as prettytable for the cases cloud-init actually uses. --- cloudinit/config/cc_ssh_authkey_fingerprints.py | 4 ++-- cloudinit/netinfo.py | 8 ++++---- packages/pkg-deps.json | 3 --- requirements.txt | 3 --- tools/build-on-freebsd | 1 - tox.ini | 3 --- 6 files changed, 6 insertions(+), 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 0066e97f..35d8c57f 100755 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -28,7 +28,7 @@ the keys can be specified, but defaults to ``md5``. import base64 import hashlib -from prettytable import PrettyTable +from cloudinit.simpletable import SimpleTable from cloudinit.distros import ug_util from cloudinit import ssh_util @@ -74,7 +74,7 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', return tbl_fields = ['Keytype', 'Fingerprint (%s)' % (hash_meth), 'Options', 'Comment'] - tbl = PrettyTable(tbl_fields) + tbl = SimpleTable(tbl_fields) for entry in key_entries: if _is_printable_key(entry): row = [] diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 39c79dee..8f99d99c 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -13,7 +13,7 @@ import re from cloudinit import log as logging from cloudinit import util -from prettytable import PrettyTable +from cloudinit.simpletable import SimpleTable LOG = logging.getLogger() @@ -170,7 +170,7 @@ def netdev_pformat(): lines.append(util.center("Net device info failed", '!', 80)) else: fields = ['Device', 'Up', 'Address', 'Mask', 'Scope', 'Hw-Address'] - tbl = PrettyTable(fields) + tbl = SimpleTable(fields) for (dev, d) in netdev.items(): tbl.add_row([dev, d["up"], d["addr"], d["mask"], ".", d["hwaddr"]]) if d.get('addr6'): @@ -194,7 +194,7 @@ def route_pformat(): if routes.get('ipv4'): fields_v4 = ['Route', 'Destination', 'Gateway', 'Genmask', 'Interface', 'Flags'] - tbl_v4 = PrettyTable(fields_v4) + tbl_v4 = SimpleTable(fields_v4) for (n, r) in enumerate(routes.get('ipv4')): route_id = str(n) tbl_v4.add_row([route_id, r['destination'], @@ -207,7 +207,7 @@ def route_pformat(): if routes.get('ipv6'): fields_v6 = ['Route', 'Proto', 'Recv-Q', 'Send-Q', 'Local Address', 'Foreign Address', 'State'] - tbl_v6 = PrettyTable(fields_v6) + tbl_v6 = SimpleTable(fields_v6) for (n, r) in enumerate(routes.get('ipv6')): route_id = str(n) tbl_v6.add_row([route_id, r['proto'], diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json index 822d29d9..72409dd8 100644 --- a/packages/pkg-deps.json +++ b/packages/pkg-deps.json @@ -34,9 +34,6 @@ "jsonschema" : { "3" : "python34-jsonschema" }, - "prettytable" : { - "3" : "python34-prettytable" - }, "pyflakes" : { "2" : "pyflakes", "3" : "python34-pyflakes" diff --git a/requirements.txt b/requirements.txt index 61d1e90b..dd10d85d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,6 @@ # Used for untemplating any files or strings with parameters. jinja2 -# This is used for any pretty printing of tabular data. -PrettyTable - # This one is currently only used by the MAAS datasource. If that # datasource is removed, this is no longer needed oauthlib diff --git a/tools/build-on-freebsd b/tools/build-on-freebsd index ff9153ad..d23fde2b 100755 --- a/tools/build-on-freebsd +++ b/tools/build-on-freebsd @@ -18,7 +18,6 @@ pkgs=" py27-jsonpatch py27-jsonpointer py27-oauthlib - py27-prettytable py27-requests py27-serial py27-six diff --git a/tox.ini b/tox.ini index 776f4253..aef1f84b 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,6 @@ deps = # requirements jinja2==2.8 pyyaml==3.11 - PrettyTable==0.7.2 oauthlib==1.0.3 pyserial==3.0.1 configobj==5.0.6 @@ -89,7 +88,6 @@ deps = argparse==1.2.1 jinja2==2.2.1 pyyaml==3.10 - PrettyTable==0.7.2 oauthlib==0.6.0 configobj==4.6.0 requests==2.6.0 @@ -105,7 +103,6 @@ deps = argparse==1.3.0 jinja2==2.8 PyYAML==3.11 - PrettyTable==0.7.2 oauthlib==0.7.2 configobj==5.0.6 requests==2.11.1 -- cgit v1.2.3 From cc1475d07b9d0727012634ee9c7a914d67b051f5 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Thu, 21 Sep 2017 11:58:28 -0400 Subject: suse: Support addition of zypper repos via cloud-config. This adds a config module so support for adding zypper repositories via cloud-config. LP: #1718675 --- cloudinit/config/cc_zypper_add_repo.py | 218 +++++++++++++++++++ config/cloud.cfg.tmpl | 3 + .../test_handler/test_handler_zypper_add_repo.py | 237 +++++++++++++++++++++ tests/unittests/test_handler/test_schema.py | 8 +- 4 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/cc_zypper_add_repo.py create mode 100644 tests/unittests/test_handler/test_handler_zypper_add_repo.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_zypper_add_repo.py b/cloudinit/config/cc_zypper_add_repo.py new file mode 100644 index 00000000..aba26952 --- /dev/null +++ b/cloudinit/config/cc_zypper_add_repo.py @@ -0,0 +1,218 @@ +# +# Copyright (C) 2017 SUSE LLC. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""zypper_add_repo: Add zyper repositories to the system""" + +import configobj +import os +from six import string_types +from textwrap import dedent + +from cloudinit.config.schema import get_schema_doc +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util + +distros = ['opensuse', 'sles'] + +schema = { + 'id': 'cc_zypper_add_repo', + 'name': 'ZypperAddRepo', + 'title': 'Configure zypper behavior and add zypper repositories', + 'description': dedent("""\ + Configure zypper behavior by modifying /etc/zypp/zypp.conf. The + configuration writer is "dumb" and will simply append the provided + configuration options to the configuration file. Option settings + that may be duplicate will be resolved by the way the zypp.conf file + is parsed. The file is in INI format. + Add repositories to the system. No validation is performed on the + repository file entries, it is assumed the user is familiar with + the zypper repository file format."""), + 'distros': distros, + 'examples': [dedent("""\ + zypper: + repos: + - id: opensuse-oss + name: os-oss + baseurl: http://dl.opensuse.org/dist/leap/v/repo/oss/ + enabled: 1 + autorefresh: 1 + - id: opensuse-oss-update + name: os-oss-up + baseurl: http://dl.opensuse.org/dist/leap/v/update + # any setting per + # https://en.opensuse.org/openSUSE:Standards_RepoInfo + # enable and autorefresh are on by default + config: + reposdir: /etc/zypp/repos.dir + servicesdir: /etc/zypp/services.d + download.use_deltarpm: true + # any setting in /etc/zypp/zypp.conf + """)], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'zypper': { + 'type': 'object', + 'properties': { + 'repos': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'description': dedent("""\ + The unique id of the repo, used when + writing + /etc/zypp/repos.d/.repo.""") + }, + 'baseurl': { + 'type': 'string', + 'format': 'uri', # built-in format type + 'description': 'The base repositoy URL' + } + }, + 'required': ['id', 'baseurl'], + 'additionalProperties': True + }, + 'minItems': 1 + }, + 'config': { + 'type': 'object', + 'description': dedent("""\ + Any supported zypo.conf key is written to + /etc/zypp/zypp.conf'""") + } + }, + 'required': [], + 'minProperties': 1, # Either config or repo must be provided + 'additionalProperties': False, # only repos and config allowed + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + +LOG = logging.getLogger(__name__) + + +def _canonicalize_id(repo_id): + repo_id = repo_id.replace(" ", "_") + return repo_id + + +def _format_repo_value(val): + if isinstance(val, bool): + # zypp prefers 1/0 + return 1 if val else 0 + if isinstance(val, (list, tuple)): + return "\n ".join([_format_repo_value(v) for v in val]) + if not isinstance(val, string_types): + return str(val) + return val + + +def _format_repository_config(repo_id, repo_config): + to_be = configobj.ConfigObj() + to_be[repo_id] = {} + # Do basic translation of the items -> values + for (k, v) in repo_config.items(): + # For now assume that people using this know the format + # of zypper repos and don't verify keys/values further + to_be[repo_id][k] = _format_repo_value(v) + lines = to_be.write() + return "\n".join(lines) + + +def _write_repos(repos, repo_base_path): + """Write the user-provided repo definition files + @param repos: A list of repo dictionary objects provided by the user's + cloud config. + @param repo_base_path: The directory path to which repo definitions are + written. + """ + + if not repos: + return + valid_repos = {} + for index, user_repo_config in enumerate(repos): + # Skip on absent required keys + missing_keys = set(['id', 'baseurl']).difference(set(user_repo_config)) + if missing_keys: + LOG.warning( + "Repo config at index %d is missing required config keys: %s", + index, ",".join(missing_keys)) + continue + repo_id = user_repo_config.get('id') + canon_repo_id = _canonicalize_id(repo_id) + repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) + if os.path.exists(repo_fn_pth): + LOG.info("Skipping repo %s, file %s already exists!", + repo_id, repo_fn_pth) + continue + elif repo_id in valid_repos: + LOG.info("Skipping repo %s, file %s already pending!", + repo_id, repo_fn_pth) + continue + + # Do some basic key formatting + repo_config = dict( + (k.lower().strip().replace("-", "_"), v) + for k, v in user_repo_config.items() + if k and k != 'id') + + # Set defaults if not present + for field in ['enabled', 'autorefresh']: + if field not in repo_config: + repo_config[field] = '1' + + valid_repos[repo_id] = (repo_fn_pth, repo_config) + + for (repo_id, repo_data) in valid_repos.items(): + repo_blob = _format_repository_config(repo_id, repo_data[-1]) + util.write_file(repo_data[0], repo_blob) + + +def _write_zypp_config(zypper_config): + """Write to the default zypp configuration file /etc/zypp/zypp.conf""" + if not zypper_config: + return + zypp_config = '/etc/zypp/zypp.conf' + zypp_conf_content = util.load_file(zypp_config) + new_settings = ['# Added via cloud.cfg'] + for setting, value in zypper_config.items(): + if setting == 'configdir': + msg = 'Changing the location of the zypper configuration is ' + msg += 'not supported, skipping "configdir" setting' + LOG.warning(msg) + continue + if value: + new_settings.append('%s=%s' % (setting, value)) + if len(new_settings) > 1: + new_config = zypp_conf_content + '\n'.join(new_settings) + else: + new_config = zypp_conf_content + util.write_file(zypp_config, new_config) + + +def handle(name, cfg, _cloud, log, _args): + zypper_section = cfg.get('zypper') + if not zypper_section: + LOG.debug(("Skipping module named %s," + " no 'zypper' relevant configuration found"), name) + return + repos = zypper_section.get('repos') + if not repos: + LOG.debug(("Skipping module named %s," + " no 'repos' configuration found"), name) + return + zypper_config = zypper_section.get('config', {}) + repo_base_path = zypper_config.get('reposdir', '/etc/zypp/repos.d/') + + _write_zypp_config(zypper_config) + _write_repos(repos, repo_base_path) + +# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 50e3bd86..32de9c9b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -84,6 +84,9 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} +{% if variant in ["suse"] %} + - zypper-add-repo +{% endif %} {% if variant not in ["freebsd"] %} - ntp {% endif %} diff --git a/tests/unittests/test_handler/test_handler_zypper_add_repo.py b/tests/unittests/test_handler/test_handler_zypper_add_repo.py new file mode 100644 index 00000000..315c2a5e --- /dev/null +++ b/tests/unittests/test_handler/test_handler_zypper_add_repo.py @@ -0,0 +1,237 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import glob +import os + +from cloudinit.config import cc_zypper_add_repo +from cloudinit import util + +from cloudinit.tests import helpers +from cloudinit.tests.helpers import mock + +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser +import logging +from six import StringIO + +LOG = logging.getLogger(__name__) + + +class TestConfig(helpers.FilesystemMockingTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.tmp = self.tmp_dir() + self.zypp_conf = 'etc/zypp/zypp.conf' + + def test_bad_repo_config(self): + """Config has no baseurl, no file should be written""" + cfg = { + 'repos': [ + { + 'id': 'foo', + 'name': 'suse-test', + 'enabled': '1' + }, + ] + } + self.patchUtils(self.tmp) + cc_zypper_add_repo._write_repos(cfg['repos'], '/etc/zypp/repos.d') + self.assertRaises(IOError, util.load_file, + "/etc/zypp/repos.d/foo.repo") + + def test_write_repos(self): + """Verify valid repos get written""" + cfg = self._get_base_config_repos() + root_d = self.tmp_dir() + cc_zypper_add_repo._write_repos(cfg['zypper']['repos'], root_d) + repos = glob.glob('%s/*.repo' % root_d) + expected_repos = ['testing-foo.repo', 'testing-bar.repo'] + if len(repos) != 2: + assert 'Number of repos written is "%d" expected 2' % len(repos) + for repo in repos: + repo_name = os.path.basename(repo) + if repo_name not in expected_repos: + assert 'Found repo with name "%s"; unexpected' % repo_name + # Validation that the content gets properly written is in another test + + def test_write_repo(self): + """Verify the content of a repo file""" + cfg = { + 'repos': [ + { + 'baseurl': 'http://foo', + 'name': 'test-foo', + 'id': 'testing-foo' + }, + ] + } + root_d = self.tmp_dir() + cc_zypper_add_repo._write_repos(cfg['repos'], root_d) + contents = util.load_file("%s/testing-foo.repo" % root_d) + parser = ConfigParser() + parser.readfp(StringIO(contents)) + expected = { + 'testing-foo': { + 'name': 'test-foo', + 'baseurl': 'http://foo', + 'enabled': '1', + 'autorefresh': '1' + } + } + for section in expected: + self.assertTrue(parser.has_section(section), + "Contains section {0}".format(section)) + for k, v in expected[section].items(): + self.assertEqual(parser.get(section, k), v) + + def test_config_write(self): + """Write valid configuration data""" + cfg = { + 'config': { + 'download.deltarpm': 'False', + 'reposdir': 'foo' + } + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg['config']) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + 'reposdir=foo' + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + + @mock.patch('cloudinit.log.logging') + def test_config_write_skip_configdir(self, mock_logging): + """Write configuration but skip writing 'configdir' setting""" + cfg = { + 'config': { + 'download.deltarpm': 'False', + 'reposdir': 'foo', + 'configdir': 'bar' + } + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg['config']) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + 'reposdir=foo' + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + # Not finding teh right path for mocking :( + # assert mock_logging.warning.called + + def test_empty_config_section_no_new_data(self): + """When the config section is empty no new data should be written to + zypp.conf""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = None + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_empty_config_value_no_new_data(self): + """When the config section is not empty but there are no values + no new data should be written to zypp.conf""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = { + 'download.deltarpm': None + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_handler_full_setup(self): + """Test that the handler ends up calling the renderers""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = { + 'download.deltarpm': 'False', + } + root_d = self.tmp_dir() + os.makedirs('%s/etc/zypp/repos.d' % root_d) + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo.handle('zypper_add_repo', cfg, None, LOG, []) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + repos = glob.glob('%s/etc/zypp/repos.d/*.repo' % root_d) + expected_repos = ['testing-foo.repo', 'testing-bar.repo'] + if len(repos) != 2: + assert 'Number of repos written is "%d" expected 2' % len(repos) + for repo in repos: + repo_name = os.path.basename(repo) + if repo_name not in expected_repos: + assert 'Found repo with name "%s"; unexpected' % repo_name + + def test_no_config_section_no_new_data(self): + """When there is no config section no new data should be written to + zypp.conf""" + cfg = self._get_base_config_repos() + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_no_repo_data(self): + """When there is no repo data nothing should happen""" + root_d = self.tmp_dir() + self.reRoot(root_d) + cc_zypper_add_repo._write_repos(None, root_d) + content = glob.glob('%s/*' % root_d) + self.assertEqual(len(content), 0) + + def _get_base_config_repos(self): + """Basic valid repo configuration""" + cfg = { + 'zypper': { + 'repos': [ + { + 'baseurl': 'http://foo', + 'name': 'test-foo', + 'id': 'testing-foo' + }, + { + 'baseurl': 'http://bar', + 'name': 'test-bar', + 'id': 'testing-bar' + } + ] + } + } + return cfg diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 745bb0ff..b8fc8930 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,7 +27,13 @@ class GetSchemaTest(CiTestCase): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() self.assertItemsEqual( - ['cc_bootcmd', 'cc_ntp', 'cc_resizefs', 'cc_runcmd'], + [ + 'cc_bootcmd', + 'cc_ntp', + 'cc_resizefs', + 'cc_runcmd', + 'cc_zypper_add_repo' + ], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) self.assertEqual( -- cgit v1.2.3