From 5bba5db2655d88b8aba8fa06b30f8e91e2ca6836 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 1 Aug 2017 18:00:00 -0500 Subject: cc_ntp: fallback on timesyncd configuration if ntp is not installable Some systems like Ubuntu-Core do not provide an ntp package for installation but do include systemd-timesyncd (an ntp client). On such systems cloud-init will generate a timesyncd configuration using the 'servers' and 'pools' values as ntp hosts for timesyncd to use. LP: #1686485 --- tests/unittests/test_handler/test_handler_ntp.py | 105 ++++++++++++++++++++++- 1 file changed, 101 insertions(+), 4 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 7f278646..83d5faa2 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -16,6 +16,14 @@ servers {{servers}} pools {{pools}} """ +TIMESYNCD_TEMPLATE = b"""\ +## template:jinja +[Time] +{% if servers or pools -%} +NTP={% for host in servers|list + pools|list %}{{ host }} {% endfor -%} +{% endif -%} +""" + try: import jsonschema assert jsonschema # avoid pyflakes error F401: import unused @@ -59,6 +67,14 @@ class TestNtp(FilesystemMockingTestCase): cc_ntp.install_ntp(install_func, packages=['ntp'], check_exe='ntpd') install_func.assert_not_called() + @mock.patch("cloudinit.config.cc_ntp.util") + def test_ntp_install_no_op_with_empty_pkg_list(self, mock_util): + """ntp_install calls install_func with empty list""" + mock_util.which.return_value = None # check_exe not found + install_func = mock.MagicMock() + cc_ntp.install_ntp(install_func, packages=[], check_exe='timesyncd') + install_func.assert_called_once_with([]) + def test_ntp_rename_ntp_conf(self): """When NTP_CONF exists, rename_ntp moves it.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) @@ -68,6 +84,30 @@ class TestNtp(FilesystemMockingTestCase): self.assertFalse(os.path.exists(ntpconf)) self.assertTrue(os.path.exists("{0}.dist".format(ntpconf))) + @mock.patch("cloudinit.config.cc_ntp.util") + def test_reload_ntp_defaults(self, mock_util): + """Test service is restarted/reloaded (defaults)""" + service = 'ntp' + cmd = ['service', service, 'restart'] + cc_ntp.reload_ntp(service) + mock_util.subp.assert_called_with(cmd, capture=True) + + @mock.patch("cloudinit.config.cc_ntp.util") + def test_reload_ntp_systemd(self, mock_util): + """Test service is restarted/reloaded (systemd)""" + service = 'ntp' + cmd = ['systemctl', 'reload-or-restart', service] + cc_ntp.reload_ntp(service, systemd=True) + mock_util.subp.assert_called_with(cmd, capture=True) + + @mock.patch("cloudinit.config.cc_ntp.util") + def test_reload_ntp_systemd_timesycnd(self, mock_util): + """Test service is restarted/reloaded (systemd/timesyncd)""" + service = 'systemd-timesycnd' + cmd = ['systemctl', 'reload-or-restart', service] + cc_ntp.reload_ntp(service, systemd=True) + mock_util.subp.assert_called_with(cmd, capture=True) + def test_ntp_rename_ntp_conf_skip_missing(self): """When NTP_CONF doesn't exist rename_ntp doesn't create a file.""" ntpconf = self.tmp_path("ntp.conf", self.new_root) @@ -94,7 +134,7 @@ class TestNtp(FilesystemMockingTestCase): with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: stream.write(NTP_TEMPLATE) with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud) + cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf) content = util.read_file_or_url('file://' + ntp_conf).contents self.assertEqual( "servers ['192.168.2.1', '192.168.2.2']\npools []\n", @@ -120,7 +160,7 @@ class TestNtp(FilesystemMockingTestCase): with open('{0}.{1}.tmpl'.format(ntp_conf, distro), 'wb') as stream: stream.write(NTP_TEMPLATE) with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template(cfg, mycloud) + cc_ntp.write_ntp_config_template(cfg, mycloud, ntp_conf) content = util.read_file_or_url('file://' + ntp_conf).contents self.assertEqual( "servers []\npools ['10.0.0.1', '10.0.0.2']\n", @@ -139,7 +179,7 @@ class TestNtp(FilesystemMockingTestCase): with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: stream.write(NTP_TEMPLATE) with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): - cc_ntp.write_ntp_config_template({}, mycloud) + cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf) content = util.read_file_or_url('file://' + ntp_conf).contents default_pools = [ "{0}.{1}.pool.ntp.org".format(x, distro) @@ -152,7 +192,8 @@ class TestNtp(FilesystemMockingTestCase): ",".join(default_pools)), self.logs.getvalue()) - def test_ntp_handler_mocked_template(self): + @mock.patch("cloudinit.config.cc_ntp.ntp_installable") + def test_ntp_handler_mocked_template(self, m_ntp_install): """Test ntp handler renders ubuntu ntp.conf template.""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] servers = ['192.168.23.3', '192.168.23.4'] @@ -164,6 +205,8 @@ class TestNtp(FilesystemMockingTestCase): } mycloud = self._get_cloud('ubuntu') ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist + m_ntp_install.return_value = True + # Create ntp.conf.tmpl with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: stream.write(NTP_TEMPLATE) @@ -176,6 +219,34 @@ class TestNtp(FilesystemMockingTestCase): 'servers {0}\npools {1}\n'.format(servers, pools), content.decode()) + @mock.patch("cloudinit.config.cc_ntp.util") + def test_ntp_handler_mocked_template_snappy(self, m_util): + """Test ntp handler renders timesycnd.conf template on snappy.""" + pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] + servers = ['192.168.23.3', '192.168.23.4'] + cfg = { + 'ntp': { + 'pools': pools, + 'servers': servers + } + } + mycloud = self._get_cloud('ubuntu') + m_util.system_is_snappy.return_value = True + + # Create timesyncd.conf.tmpl + tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root) + template = '{0}.tmpl'.format(tsyncd_conf) + with open(template, 'wb') as stream: + stream.write(TIMESYNCD_TEMPLATE) + + with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf): + cc_ntp.handle('notimportant', cfg, mycloud, None, None) + + content = util.read_file_or_url('file://' + tsyncd_conf).contents + self.assertEqual( + "[Time]\nNTP=%s %s \n" % (" ".join(servers), " ".join(pools)), + content.decode()) + def test_ntp_handler_real_distro_templates(self): """Test ntp handler renders the shipped distro ntp.conf templates.""" pools = ['0.mycompany.pool.ntp.org', '3.mycompany.pool.ntp.org'] @@ -333,4 +404,30 @@ class TestNtp(FilesystemMockingTestCase): "pools ['0.mypool.org', '0.mypool.org']\n", content) + @mock.patch("cloudinit.config.cc_ntp.ntp_installable") + def test_ntp_handler_timesyncd(self, m_ntp_install): + """Test ntp handler configures timesyncd""" + m_ntp_install.return_value = False + distro = 'ubuntu' + cfg = { + 'servers': ['192.168.2.1', '192.168.2.2'], + 'pools': ['0.mypool.org'], + } + mycloud = self._get_cloud(distro) + tsyncd_conf = self.tmp_path("timesyncd.conf", self.new_root) + # Create timesyncd.conf.tmpl + template = '{0}.tmpl'.format(tsyncd_conf) + print(template) + with open(template, 'wb') as stream: + stream.write(TIMESYNCD_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.TIMESYNCD_CONF', tsyncd_conf): + cc_ntp.write_ntp_config_template(cfg, mycloud, tsyncd_conf, + template='timesyncd.conf') + + content = util.read_file_or_url('file://' + tsyncd_conf).contents + self.assertEqual( + "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n", + content.decode()) + + # vi: ts=4 expandtab -- cgit v1.2.3 From cc9762a2d737ead386ffb9f067adc5e543224560 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 22 Aug 2017 20:06:20 -0600 Subject: schema cli: Add schema subcommand to cloud-init cli and cc_runcmd schema This branch does a few things: - Add 'schema' subcommand to cloud-init CLI for validating cloud-config files against strict module jsonschema definitions - Add --annotate parameter to 'cloud-init schema' to annotate existing cloud-config file content with validation errors - Add jsonschema definition to cc_runcmd - Add unit test coverage for cc_runcmd - Update CLI capabilities documentation This branch only imports development (and analyze) subparsers when the specific subcommand is provided on the CLI to avoid adding costly unused file imports during cloud-init system boot. The schema command allows a person to quickly validate a cloud-config text file against cloud-init's known module schemas to avoid costly roundtrips deploying instances in their cloud of choice. As of this branch, only cc_ntp and cc_runcmd cloud-config modules define schemas. Schema validation will ignore all undefined config keys until all modules define a strict schema. To perform validation of runcmd and ntp sections of a cloud-config file: $ cat > cloud.cfg < Date: Wed, 23 Aug 2017 13:24:38 -0600 Subject: cc_landscape & cc_puppet: Fix six.StringIO use in writing configs Both landscape and puppet modules had issues with the way they wrote /etc/landscape/client.conf or /etc/puppet/puppet.conf in either python3 or python2. This branch adds initial unit tests for both modules which will get better exercise under both python2 and python3. The unit tests shed light on a few issues: - In the cc_landscape module py3 can't provide six.StringIO content to ConfigParser.write, so we need to use six.BytesIO instead - In the cc_puppet module, python <= 2.7 doesn't support using six.StringIO as a context manager, so we drop the context manager fanciness and directly set outputstream = StringIO(). - The docstring in cc_puppet is fixed to document the 'conf' sub-key requiring valid puppet section names for each key-value list. LP: #1699282 LP: #1710932 --- cloudinit/config/cc_landscape.py | 4 +- cloudinit/config/cc_puppet.py | 33 ++--- cloudinit/helpers.py | 14 +- .../test_handler/test_handler_landscape.py | 129 +++++++++++++++++++ .../unittests/test_handler/test_handler_puppet.py | 142 +++++++++++++++++++++ 5 files changed, 298 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_landscape.py create mode 100644 tests/unittests/test_handler/test_handler_puppet.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 86b71383..8f9f1abd 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -57,7 +57,7 @@ The following default client config is provided, but can be overridden:: import os -from six import StringIO +from six import BytesIO from configobj import ConfigObj @@ -109,7 +109,7 @@ def handle(_name, cfg, cloud, log, _args): ls_cloudcfg, ] merged = merge_together(merge_data) - contents = StringIO() + contents = BytesIO() merged.write(contents) util.ensure_dir(os.path.dirname(LSC_CLIENT_CFG_FILE)) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index dc11561b..28b1d568 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -15,21 +15,23 @@ This module handles puppet installation and configuration. If the ``puppet`` key does not exist in global configuration, no action will be taken. If a config entry for ``puppet`` is present, then by default the latest version of puppet will be installed. If ``install`` is set to ``false``, puppet will not -be installed. However, this may result in an error if puppet is not already +be installed. However, this will result in an error if puppet is not already present on the system. The version of puppet to be installed can be specified under ``version``, and defaults to ``none``, which selects the latest version in the repos. If the ``puppet`` config key exists in the config archive, this module will attempt to start puppet even if no installation was performed. -Puppet configuration can be specified under the ``conf`` key. The configuration -is specified as a dictionary which is converted into ``=`` format -and appended to ``puppet.conf`` under the ``[puppetd]`` section. The +Puppet configuration can be specified under the ``conf`` key. The +configuration is specified as a dictionary containing high-level ``
`` +keys and lists of ``=`` pairs within each section. Each section +name and ``=`` pair is written directly to ``puppet.conf``. As +such, section names should be one of: ``main``, ``master``, ``agent`` or +``user`` and keys should be valid puppet configuration options. The ``certname`` key supports string substitutions for ``%i`` and ``%f``, corresponding to the instance id and fqdn of the machine respectively. -If ``ca_cert`` is present under ``conf``, it will not be written to -``puppet.conf``, but instead will be used as the puppermaster certificate. -It should be specified in pem format as a multi-line string (using the ``|`` -yaml notation). +If ``ca_cert`` is present, it will not be written to ``puppet.conf``, but +instead will be used as the puppermaster certificate. It should be specified +in pem format as a multi-line string (using the ``|`` yaml notation). **Internal name:** ``cc_puppet`` @@ -43,12 +45,13 @@ yaml notation). install: version: conf: - server: "puppetmaster.example.org" - certname: "%i.%f" - ca_cert: | - -------BEGIN CERTIFICATE------- - - -------END CERTIFICATE------- + agent: + server: "puppetmaster.example.org" + certname: "%i.%f" + ca_cert: | + -------BEGIN CERTIFICATE------- + + -------END CERTIFICATE------- """ from six import StringIO @@ -127,7 +130,7 @@ def handle(name, cfg, cloud, log, _args): util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: - # Iterate throug the config items, we'll use ConfigParser.set + # Iterate through the config items, we'll use ConfigParser.set # to overwrite or create new items as needed for (o, v) in cfg.items(): if o == 'certname': diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index f01021aa..1979cd96 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -13,7 +13,7 @@ from time import time import contextlib import os -import six +from six import StringIO from six.moves.configparser import ( NoSectionError, NoOptionError, RawConfigParser) @@ -441,12 +441,12 @@ class DefaultingConfigParser(RawConfigParser): def stringify(self, header=None): contents = '' - with six.StringIO() as outputstream: - self.write(outputstream) - outputstream.flush() - contents = outputstream.getvalue() - if header: - contents = "\n".join([header, contents]) + outputstream = StringIO() + self.write(outputstream) + outputstream.flush() + contents = outputstream.getvalue() + if header: + contents = '\n'.join([header, contents, '']) return contents # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py new file mode 100644 index 00000000..7c247fa9 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -0,0 +1,129 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_landscape +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call + +from configobj import ConfigObj +import logging + + +LOG = logging.getLogger(__name__) + + +class TestLandscape(FilesystemMockingTestCase): + + with_logs = True + + def setUp(self): + super(TestLandscape, self).setUp() + self.new_root = self.tmp_dir() + self.conf = self.tmp_path('client.conf', self.new_root) + self.default_file = self.tmp_path('default_landscape', self.new_root) + + def _get_cloud(self, distro): + self.patchUtils(self.new_root) + paths = helpers.Paths({'templates_dir': self.new_root}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skips_empty_landscape_cloudconfig(self): + """Empty landscape cloud-config section does no work.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'landscape': {}} + cc_landscape.handle('notimportant', cfg, mycloud, LOG, None) + self.assertFalse(mycloud.distro.install_packages.called) + + def test_handler_error_on_invalid_landscape_type(self): + """Raise an error when landscape configuraiton option is invalid.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': 'wrongtype'} + with self.assertRaises(RuntimeError) as context_manager: + cc_landscape.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "'landscape' key existed in config, but not a dict", + str(context_manager.exception)) + + @mock.patch('cloudinit.config.cc_landscape.util') + def test_handler_restarts_landscape_client(self, m_util): + """handler restarts lansdscape-client after install.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(['service', 'landscape-client', 'restart'])], + m_util.subp.call_args_list) + + def test_handler_installs_client_and_creates_config_file(self): + """Write landscape client.conf and install landscape-client.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client'}} + mycloud.distro = mock.MagicMock() + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}, + 'LS_DEFAULT_FILE': {'new': self.default_file}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call('landscape-client')], + mycloud.distro.install_packages.call_args) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) + default_content = util.load_file(self.default_file) + self.assertEqual('RUN=1\n', default_content) + + def test_handler_writes_merged_client_config_file_with_defaults(self): + """Merge and write options from LSC_CLIENT_CFG_FILE with defaults.""" + # Write existing sparse client.conf file + util.write_file(self.conf, '[client]\ncomputer_title = My PC\n') + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client', + 'computer_title': 'My PC'}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) + + def test_handler_writes_merged_provided_cloudconfig_with_defaults(self): + """Merge and write options from cloud-config options with defaults.""" + # Write empty sparse client.conf file + util.write_file(self.conf, '') + mycloud = self._get_cloud('ubuntu') + cfg = {'landscape': {'client': {'computer_title': 'My PC'}}} + expected = {'client': { + 'log_level': 'info', + 'url': 'https://landscape.canonical.com/message-system', + 'ping_url': 'http://landscape.canonical.com/ping', + 'data_path': '/var/lib/landscape/client', + 'computer_title': 'My PC'}} + wrap_and_call( + 'cloudinit.config.cc_landscape', + {'LSC_CLIENT_CFG_FILE': {'new': self.conf}}, + cc_landscape.handle, 'notimportant', cfg, mycloud, LOG, None) + self.assertEqual(expected, dict(ConfigObj(self.conf))) + self.assertIn( + 'Wrote landscape config file to {0}'.format(self.conf), + self.logs.getvalue()) diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py new file mode 100644 index 00000000..805c76ba --- /dev/null +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -0,0 +1,142 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_puppet +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from ..helpers import CiTestCase, mock + +import logging + + +LOG = logging.getLogger(__name__) + + +@mock.patch('cloudinit.config.cc_puppet.util') +@mock.patch('cloudinit.config.cc_puppet.os') +class TestAutostartPuppet(CiTestCase): + + with_logs = True + + def test_wb_autostart_puppet_updates_puppet_default(self, m_os, m_util): + """Update /etc/default/puppet to autostart if it exists.""" + + def _fake_exists(path): + return path == '/etc/default/puppet' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + self.assertEqual( + [mock.call(['sed', '-i', '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False)], + m_util.subp.call_args_list) + + def test_wb_autostart_pupppet_enables_puppet_systemctl(self, m_os, m_util): + """If systemctl is present, enable puppet via systemctl.""" + + def _fake_exists(path): + return path == '/bin/systemctl' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + expected_calls = [mock.call( + ['/bin/systemctl', 'enable', 'puppet.service'], capture=False)] + self.assertEqual(expected_calls, m_util.subp.call_args_list) + + def test_wb_autostart_pupppet_enables_puppet_chkconfig(self, m_os, m_util): + """If chkconfig is present, enable puppet via checkcfg.""" + + def _fake_exists(path): + return path == '/sbin/chkconfig' + + m_os.path.exists.side_effect = _fake_exists + cc_puppet._autostart_puppet(LOG) + expected_calls = [mock.call( + ['/sbin/chkconfig', 'puppet', 'on'], capture=False)] + self.assertEqual(expected_calls, m_util.subp.call_args_list) + + +@mock.patch('cloudinit.config.cc_puppet._autostart_puppet') +class TestPuppetHandle(CiTestCase): + + with_logs = True + + def setUp(self): + super(TestPuppetHandle, self).setUp() + self.new_root = self.tmp_dir() + self.conf = self.tmp_path('puppet.conf') + + def _get_cloud(self, distro): + paths = helpers.Paths({'templates_dir': self.new_root}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skips_missing_puppet_key_in_cloudconfig(self, m_auto): + """Cloud-config containing no 'puppet' key is skipped.""" + mycloud = self._get_cloud('ubuntu') + cfg = {} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "no 'puppet' configuration found", self.logs.getvalue()) + self.assertEqual(0, m_auto.call_count) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_starts_puppet_service(self, m_subp, m_auto): + """Cloud-config 'puppet' configuration starts puppet.""" + mycloud = self._get_cloud('ubuntu') + cfg = {'puppet': {'install': False}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual(1, m_auto.call_count) + self.assertEqual( + [mock.call(['service', 'puppet', 'start'], capture=False)], + m_subp.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_empty_puppet_config_installs_puppet(self, m_subp, m_auto): + """Cloud-config empty 'puppet' configuration installs latest puppet.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', None))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_installs_puppet_on_true(self, m_subp, _): + """Cloud-config with 'puppet' key installs when 'install' is True.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {'install': True}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', None))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_installs_puppet_version(self, m_subp, _): + """Cloud-config 'puppet' configuration can specify a version.""" + mycloud = self._get_cloud('ubuntu') + mycloud.distro = mock.MagicMock() + cfg = {'puppet': {'version': '3.8'}} + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + self.assertEqual( + [mock.call(('puppet', '3.8'))], + mycloud.distro.install_packages.call_args_list) + + @mock.patch('cloudinit.config.cc_puppet.util.subp') + def test_handler_puppet_config_updates_puppet_conf(self, m_subp, m_auto): + """When 'conf' is provided update values in PUPPET_CONF_PATH.""" + mycloud = self._get_cloud('ubuntu') + cfg = { + 'puppet': { + 'conf': {'agent': {'server': 'puppetmaster.example.org'}}}} + util.write_file(self.conf, '[agent]\nserver = origpuppet\nother = 3') + puppet_conf_path = 'cloudinit.config.cc_puppet.PUPPET_CONF_PATH' + mycloud.distro = mock.MagicMock() + with mock.patch(puppet_conf_path, self.conf): + cc_puppet.handle('notimportant', cfg, mycloud, LOG, None) + content = util.load_file(self.conf) + expected = '[agent]\nserver = puppetmaster.example.org\nother = 3\n\n' + self.assertEqual(expected, content) -- cgit v1.2.3 From cbda576a7bbf846710ad55940bf8ca1f2d2194b9 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Fri, 25 Aug 2017 11:13:58 -0400 Subject: suse: Add support for openSUSE and return SLES to a working state. This gets initial opensuse and SLES support back to a working state. Still missing is more complete network file writing and unit tests. --- cloudinit/config/cc_resolv_conf.py | 2 +- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/opensuse.py | 212 +++++++++++++++++++++ cloudinit/distros/sles.py | 160 +--------------- templates/hosts.opensuse.tmpl | 26 +++ templates/hosts.suse.tmpl | 3 - tests/unittests/test_distros/test_opensuse.py | 12 ++ tests/unittests/test_distros/test_sles.py | 12 ++ .../unittests/test_handler/test_handler_locale.py | 12 +- .../test_handler/test_handler_set_hostname.py | 5 +- tox.ini | 16 ++ 11 files changed, 297 insertions(+), 165 deletions(-) create mode 100644 cloudinit/distros/opensuse.py create mode 100644 templates/hosts.opensuse.tmpl create mode 100644 tests/unittests/test_distros/test_opensuse.py create mode 100644 tests/unittests/test_distros/test_sles.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 2548d1f1..9812562a 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -55,7 +55,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -distros = ['fedora', 'rhel', 'sles'] +distros = ['fedora', 'opensuse', 'rhel', 'sles'] def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 1fd48a7b..807b3ea2 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -35,7 +35,7 @@ OSFAMILIES = { 'redhat': ['centos', 'fedora', 'rhel'], 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], - 'suse': ['sles'], + 'suse': ['opensuse', 'sles'], 'arch': ['arch'], } diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py new file mode 100644 index 00000000..a219e9fb --- /dev/null +++ b/cloudinit/distros/opensuse.py @@ -0,0 +1,212 @@ +# Copyright (C) 2017 SUSE LLC +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Author: Robert Schweikert +# Author: Juerg Haefliger +# +# Leaning very heavily on the RHEL and Debian implementation +# +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit import distros + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.distros import net_util +from cloudinit.distros import rhel_util as rhutil +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + clock_conf_fn = '/etc/sysconfig/clock' + hostname_conf_fn = '/etc/HOSTNAME' + init_cmd = ['service'] + locale_conf_fn = '/etc/sysconfig/language' + network_conf_fn = '/etc/sysconfig/network' + network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' + resolve_conf_fn = '/etc/resolv.conf' + route_conf_tpl = '/etc/sysconfig/network/ifroute-%s' + systemd_hostname_conf_fn = '/etc/hostname' + systemd_locale_conf_fn = '/etc/locale.conf' + tz_local_fn = '/etc/localtime' + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + self._runner = helpers.Runners(paths) + self.osfamily = 'suse' + cfg['ssh_svcname'] = 'sshd' + if self.uses_systemd(): + self.init_cmd = ['systemctl'] + cfg['ssh_svcname'] = 'sshd.service' + + def apply_locale(self, locale, out_fn=None): + if self.uses_systemd(): + if not out_fn: + out_fn = self.systemd_locale_conf_fn + locale_cfg = {'LANG': locale} + else: + if not out_fn: + out_fn = self.locale_conf_fn + locale_cfg = {'RC_LANG': locale} + rhutil.update_sysconfig_file(out_fn, locale_cfg) + + def install_packages(self, pkglist): + self.package_command( + 'install', + args='--auto-agree-with-licenses', + pkgs=pkglist + ) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['zypper'] + # No user interaction possible, enable non-interactive mode + cmd.append('--non-interactive') + + # Comand is the operation, such as install + if command == 'upgrade': + command = 'update' + cmd.append(command) + + # args are the arguments to the command, not global options + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, capture=False) + + def set_timezone(self, tz): + tz_file = self._find_tz_file(tz) + if self.uses_systemd(): + # Currently, timedatectl complains if invoked during startup + # so for compatibility, create the link manually. + util.del_file(self.tz_local_fn) + util.sym_link(tz_file, self.tz_local_fn) + else: + # Adjust the sysconfig clock zone setting + clock_cfg = { + 'TIMEZONE': str(tz), + } + rhutil.update_sysconfig_file(self.clock_conf_fn, clock_cfg) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ['refresh'], freq=PER_INSTANCE) + + def _bring_up_interfaces(self, device_names): + if device_names and 'all' in device_names: + raise RuntimeError(('Distro %s can not translate ' + 'the device name "all"') % (self.name)) + return distros.Distro._bring_up_interfaces(self, device_names) + + def _read_hostname(self, filename, default=None): + if self.uses_systemd() and filename.endswith('/previous-hostname'): + return util.load_file(filename).strip() + elif self.uses_systemd(): + (out, _err) = util.subp(['hostname']) + if len(out): + return out + else: + return default + else: + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_system_hostname(self): + if self.uses_systemd(): + host_fn = self.systemd_hostname_conf_fn + else: + host_fn = self.hostname_conf_fn + return (host_fn, self._read_hostname(host_fn)) + + def _write_hostname(self, hostname, out_fn): + if self.uses_systemd() and out_fn.endswith('/previous-hostname'): + util.write_file(out_fn, hostname) + elif self.uses_systemd(): + util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + else: + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(hostname) + util.write_file(out_fn, str(conf), 0o644) + + def _write_network(self, settings): + # Convert debian settings to ifcfg format + entries = net_util.translate_network(settings) + LOG.debug("Translated ubuntu style network settings %s into %s", + settings, entries) + # Make the intermediate format as the suse format... + nameservers = [] + searchservers = [] + dev_names = entries.keys() + for (dev, info) in entries.items(): + net_fn = self.network_script_tpl % (dev) + route_fn = self.route_conf_tpl % (dev) + mode = None + if info.get('auto', None): + mode = 'auto' + else: + mode = 'manual' + bootproto = info.get('bootproto', None) + gateway = info.get('gateway', None) + net_cfg = { + 'BOOTPROTO': bootproto, + 'BROADCAST': info.get('broadcast'), + 'GATEWAY': gateway, + 'IPADDR': info.get('address'), + 'LLADDR': info.get('hwaddress'), + 'NETMASK': info.get('netmask'), + 'STARTMODE': mode, + 'USERCONTROL': 'no' + } + if dev != 'lo': + net_cfg['ETHTOOL_OPTIONS'] = '' + else: + net_cfg['FIREWALL'] = 'no' + rhutil.update_sysconfig_file(net_fn, net_cfg, True) + if gateway and bootproto == 'static': + default_route = 'default %s' % gateway + util.write_file(route_fn, default_route, 0o644) + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + if 'dns-search' in info: + searchservers.extend(info['dns-search']) + if nameservers or searchservers: + rhutil.update_resolve_conf_file(self.resolve_conf_fn, + nameservers, searchservers) + return dev_names + +# vi: ts=4 expandtab diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index dbec2edf..6e336cbf 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -1,167 +1,17 @@ -# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2017 SUSE LLC # -# Author: Juerg Haefliger +# Author: Robert Schweikert # # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit import distros +from cloudinit.distros import opensuse -from cloudinit.distros.parsers.hostname import HostnameConf - -from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util - -from cloudinit.distros import net_util -from cloudinit.distros import rhel_util -from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) -class Distro(distros.Distro): - clock_conf_fn = '/etc/sysconfig/clock' - locale_conf_fn = '/etc/sysconfig/language' - network_conf_fn = '/etc/sysconfig/network' - hostname_conf_fn = '/etc/HOSTNAME' - network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' - resolve_conf_fn = '/etc/resolv.conf' - tz_local_fn = '/etc/localtime' - - def __init__(self, name, cfg, paths): - distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain - # calls from repeatly happening (when they - # should only happen say once per instance...) - self._runner = helpers.Runners(paths) - self.osfamily = 'suse' - - def install_packages(self, pkglist): - self.package_command('install', args='-l', pkgs=pkglist) - - def _write_network(self, settings): - # Convert debian settings to ifcfg format - entries = net_util.translate_network(settings) - LOG.debug("Translated ubuntu style network settings %s into %s", - settings, entries) - # Make the intermediate format as the suse format... - nameservers = [] - searchservers = [] - dev_names = entries.keys() - for (dev, info) in entries.items(): - net_fn = self.network_script_tpl % (dev) - mode = info.get('auto') - if mode and mode.lower() == 'true': - mode = 'auto' - else: - mode = 'manual' - net_cfg = { - 'BOOTPROTO': info.get('bootproto'), - 'BROADCAST': info.get('broadcast'), - 'GATEWAY': info.get('gateway'), - 'IPADDR': info.get('address'), - 'LLADDR': info.get('hwaddress'), - 'NETMASK': info.get('netmask'), - 'STARTMODE': mode, - 'USERCONTROL': 'no' - } - if dev != 'lo': - net_cfg['ETHERDEVICE'] = dev - net_cfg['ETHTOOL_OPTIONS'] = '' - else: - net_cfg['FIREWALL'] = 'no' - rhel_util.update_sysconfig_file(net_fn, net_cfg, True) - if 'dns-nameservers' in info: - nameservers.extend(info['dns-nameservers']) - if 'dns-search' in info: - searchservers.extend(info['dns-search']) - if nameservers or searchservers: - rhel_util.update_resolve_conf_file(self.resolve_conf_fn, - nameservers, searchservers) - return dev_names - - def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn - locale_cfg = { - 'RC_LANG': locale, - } - rhel_util.update_sysconfig_file(out_fn, locale_cfg) - - def _write_hostname(self, hostname, out_fn): - conf = None - try: - # Try to update the previous one - # so lets see if we can read it first. - conf = self._read_hostname_conf(out_fn) - except IOError: - pass - if not conf: - conf = HostnameConf('') - conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0o644) - - def _read_system_hostname(self): - host_fn = self.hostname_conf_fn - return (host_fn, self._read_hostname(host_fn)) - - def _read_hostname_conf(self, filename): - conf = HostnameConf(util.load_file(filename)) - conf.parse() - return conf - - def _read_hostname(self, filename, default=None): - hostname = None - try: - conf = self._read_hostname_conf(filename) - hostname = conf.hostname - except IOError: - pass - if not hostname: - return default - return hostname - - def _bring_up_interfaces(self, device_names): - if device_names and 'all' in device_names: - raise RuntimeError(('Distro %s can not translate ' - 'the device name "all"') % (self.name)) - return distros.Distro._bring_up_interfaces(self, device_names) - - def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Adjust the sysconfig clock zone setting - clock_cfg = { - 'TIMEZONE': str(tz), - } - rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) - - def package_command(self, command, args=None, pkgs=None): - if pkgs is None: - pkgs = [] - - cmd = ['zypper'] - # No user interaction possible, enable non-interactive mode - cmd.append('--non-interactive') - - # Comand is the operation, such as install - cmd.append(command) - - # args are the arguments to the command, not global options - if args and isinstance(args, str): - cmd.append(args) - elif args and isinstance(args, list): - cmd.extend(args) - - pkglist = util.expand_package_list('%s-%s', pkgs) - cmd.extend(pkglist) - - # Allow the output of this to flow outwards (ie not be captured) - util.subp(cmd, capture=False) - - def update_package_sources(self): - self._runner.run("update-sources", self.package_command, - ['refresh'], freq=PER_INSTANCE) +class Distro(opensuse.Distro): + pass # vi: ts=4 expandtab diff --git a/templates/hosts.opensuse.tmpl b/templates/hosts.opensuse.tmpl new file mode 100644 index 00000000..655da3f7 --- /dev/null +++ b/templates/hosts.opensuse.tmpl @@ -0,0 +1,26 @@ +* + This file /etc/cloud/templates/hosts.opensuse.tmpl is only utilized + if enabled in cloud-config. Specifically, in order to enable it + you need to add the following to config: + manage_etc_hosts: True +*# +# Your system has configured 'manage_etc_hosts' as True. +# As a result, if you wish for changes to this file to persist +# then you will need to either +# a.) make changes to the master file in +# /etc/cloud/templates/hosts.opensuse.tmpl +# b.) change or remove the value of 'manage_etc_hosts' in +# /etc/cloud/cloud.cfg or cloud-config from user-data +# +# The following lines are desirable for IPv4 capable hosts +127.0.0.1 localhost + +# The following lines are desirable for IPv6 capable hosts +::1 localhost ipv6-localhost ipv6-loopback +fe00::0 ipv6-localnet + +ff00::0 ipv6-mcastprefix +ff02::1 ipv6-allnodes +ff02::2 ipv6-allrouters +ff02::3 ipv6-allhosts + diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl index 399ec9b4..b6082692 100644 --- a/templates/hosts.suse.tmpl +++ b/templates/hosts.suse.tmpl @@ -14,12 +14,9 @@ you need to add the following to config: # # The following lines are desirable for IPv4 capable hosts 127.0.0.1 localhost -127.0.0.1 {{fqdn}} {{hostname}} - # The following lines are desirable for IPv6 capable hosts ::1 localhost ipv6-localhost ipv6-loopback -::1 {{fqdn}} {{hostname}} fe00::0 ipv6-localnet ff00::0 ipv6-mcastprefix diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py new file mode 100644 index 00000000..bdb1d633 --- /dev/null +++ b/tests/unittests/test_distros/test_opensuse.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import CiTestCase + +from . import _get_distro + + +class TestopenSUSE(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("opensuse") + self.assertEqual(distro.osfamily, 'suse') diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py new file mode 100644 index 00000000..c656aacc --- /dev/null +++ b/tests/unittests/test_distros/test_sles.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import CiTestCase + +from . import _get_distro + + +class TestSLES(CiTestCase): + + def test_get_distro(self): + distro = _get_distro("sles") + self.assertEqual(distro.osfamily, 'suse') diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index e9a810c5..aaf6c762 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -49,9 +49,15 @@ class TestLocale(t_help.FilesystemMockingTestCase): } cc = self._get_cloud('sles') cc_locale.handle('cc_locale', cfg, cc, LOG, []) - - contents = util.load_file('/etc/sysconfig/language', decode=False) + if cc.distro.uses_systemd: + locale_conf = cc.distro.systemd_locale_conf_fn + else: + locale_conf = cc.distro.locale_conf_fn + contents = util.load_file(locale_conf, decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) + if cc.distro.uses_systemd(): + self.assertEqual({'LANG': cfg['locale']}, dict(n_cfg)) + else: + self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 4b18de75..8165bf9a 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -70,7 +70,8 @@ class TestHostname(t_help.FilesystemMockingTestCase): cc = cloud.Cloud(ds, paths, {}, distro, None) self.patchUtils(self.tmp) cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) - contents = util.load_file("/etc/HOSTNAME") - self.assertEqual('blah', contents.strip()) + if not distro.uses_systemd(): + contents = util.load_file(distro.hostname_conf_fn) + self.assertEqual('blah', contents.strip()) # vi: ts=4 expandtab diff --git a/tox.ini b/tox.ini index 1e7ca2d3..a17156ce 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,22 @@ deps = six==1.9.0 -r{toxinidir}/test-requirements.txt +[testenv:opensusel42] +basepython = python2.7 +commands = nosetests {posargs:tests/unittests} +deps = + # requirements + argparse==1.3.0 + jinja2==2.8 + PyYAML==3.11 + PrettyTable==0.7.2 + oauthlib==0.7.2 + configobj==5.0.6 + requests==2.11.1 + jsonpatch==1.11 + six==1.9.0 + -r{toxinidir}/test-requirements.txt + [testenv:tip-pycodestyle] commands = {envpython} -m pycodestyle {posargs:cloudinit/ tests/ tools/} deps = pycodestyle -- cgit v1.2.3 From 300e4516f78dbb0a9533749aa84f7e366b023d04 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 30 Aug 2017 20:58:27 -0400 Subject: tests: fix two recently added tests for sles distro. test_set_locale_sles and test_set_locale_sles_default were incorrectly testing for truth of .uses_systemd rather than calling that function and checking its result. The error was only seen if the system running the tests was not using systemd. --- tests/unittests/test_handler/test_handler_locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/unittests/test_handler') diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index aaf6c762..cba5cae8 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -49,7 +49,7 @@ class TestLocale(t_help.FilesystemMockingTestCase): } cc = self._get_cloud('sles') cc_locale.handle('cc_locale', cfg, cc, LOG, []) - if cc.distro.uses_systemd: + if cc.distro.uses_systemd(): locale_conf = cc.distro.systemd_locale_conf_fn else: locale_conf = cc.distro.locale_conf_fn -- cgit v1.2.3 From 7e76c57b590c7c2c13f7b1a2a8b5b7d4f2d18396 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 16 Aug 2017 16:50:07 -0500 Subject: distro: allow distro to specify a default locale Currently the cloud-init default locale (en_US.UTF-8) is set by the base datasource class. This patch allows a distro to overide the fallback value with one that's available in the distro but continues to respect an image which has preconfigured a locale. - Distro object now has a get_locale method which will return a preconfigure locale setting by checking the distros locale system configuration file. If not set or not present, return the default locale of en_US.UTF-8 which retains behavior of all previous cloud-init releases. - Apply locale now handles regenerating locales or system configuration files as needed. - Adjust apply_locale logic to skip locale-regen if the specified LANG value is C.UTF-8,C, or POSIX; they do not require regeneration. - Further add unittests to exercise the default paths for Ubuntu and non-ubuntu paths to validate they get the LANG expected. --- cloudinit/distros/__init__.py | 3 + cloudinit/distros/debian.py | 94 ++++++++++++++++------ cloudinit/sources/__init__.py | 9 ++- tests/unittests/test_distros/test_debian.py | 66 +++++++++------ tests/unittests/test_distros/test_generic.py | 16 ++++ tests/unittests/test_handler/test_handler_debug.py | 11 ++- .../unittests/test_handler/test_handler_locale.py | 48 +++++++++++ 7 files changed, 195 insertions(+), 52 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 807b3ea2..b714b9ab 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -188,6 +188,9 @@ class Distro(object): def _get_localhost_ip(self): return "127.0.0.1" + def get_locale(self): + raise NotImplementedError() + @abc.abstractmethod def _read_hostname(self, filename, default=None): raise NotImplementedError() diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index abfb81f4..33cc0bf1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -61,11 +61,49 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' + self.default_locale = 'en_US.UTF-8' + self.system_locale = None - def apply_locale(self, locale, out_fn=None): + def get_locale(self): + """Return the default locale if set, else use default locale""" + + # read system locale value + if not self.system_locale: + self.system_locale = read_system_locale() + + # Return system_locale setting if valid, else use default locale + return (self.system_locale if self.system_locale else + self.default_locale) + + def apply_locale(self, locale, out_fn=None, keyname='LANG'): + """Apply specified locale to system, regenerate if specified locale + differs from system default.""" if not out_fn: out_fn = LOCALE_CONF_FN - apply_locale(locale, out_fn) + + if not locale: + raise ValueError('Failed to provide locale value.') + + # Only call locale regeneration if needed + # Update system locale config with specified locale if needed + distro_locale = self.get_locale() + conf_fn_exists = os.path.exists(out_fn) + sys_locale_unset = False if self.system_locale else True + need_regen = (locale.lower() != distro_locale.lower() or + not conf_fn_exists or sys_locale_unset) + need_conf = not conf_fn_exists or need_regen or sys_locale_unset + + if need_regen: + regenerate_locale(locale, out_fn, keyname=keyname) + else: + LOG.debug( + "System has '%s=%s' requested '%s', skipping regeneration.", + keyname, self.system_locale, locale) + + if need_conf: + update_locale_conf(locale, out_fn, keyname=keyname) + # once we've updated the system config, invalidate cache + self.system_locale = None def install_packages(self, pkglist): self.update_package_sources() @@ -218,37 +256,47 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): LOG.warning(msg) -def apply_locale(locale, sys_path=LOCALE_CONF_FN, keyname='LANG'): - """Apply the locale. - - Run locale-gen for the provided locale and set the default - system variable `keyname` appropriately in the provided `sys_path`. - - If sys_path indicates that `keyname` is already set to `locale` - then no changes will be made and locale-gen not called. - This allows images built with a locale already generated to not re-run - locale-gen which can be very heavy. - """ - if not locale: - raise ValueError('Failed to provide locale value.') - +def read_system_locale(sys_path=LOCALE_CONF_FN, keyname='LANG'): + """Read system default locale setting, if present""" + sys_val = "" if not sys_path: raise ValueError('Invalid path: %s' % sys_path) if os.path.exists(sys_path): locale_content = util.load_file(sys_path) - # if LANG isn't present, regen sys_defaults = util.load_shell_content(locale_content) sys_val = sys_defaults.get(keyname, "") - if sys_val.lower() == locale.lower(): - LOG.debug( - "System has '%s=%s' requested '%s', skipping regeneration.", - keyname, sys_val, locale) - return - util.subp(['locale-gen', locale], capture=False) + return sys_val + + +def update_locale_conf(locale, sys_path, keyname='LANG'): + """Update system locale config""" + LOG.debug('Updating %s with locale setting %s=%s', + sys_path, keyname, locale) util.subp( ['update-locale', '--locale-file=' + sys_path, '%s=%s' % (keyname, locale)], capture=False) + +def regenerate_locale(locale, sys_path, keyname='LANG'): + """ + Run locale-gen for the provided locale and set the default + system variable `keyname` appropriately in the provided `sys_path`. + + """ + # special case for locales which do not require regen + # % locale -a + # C + # C.UTF-8 + # POSIX + if locale.lower() in ['c', 'c.utf-8', 'posix']: + LOG.debug('%s=%s does not require rengeneration', keyname, locale) + return + + # finally, trigger regeneration + LOG.debug('Generating locales for %s', locale) + util.subp(['locale-gen', locale], capture=False) + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 952caf35..9a43fbee 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -44,6 +44,7 @@ class DataSourceNotFoundException(Exception): class DataSource(object): dsmode = DSMODE_NETWORK + default_locale = 'en_US.UTF-8' def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg @@ -150,7 +151,13 @@ class DataSource(object): return None def get_locale(self): - return 'en_US.UTF-8' + """Default locale is en_US.UTF-8, but allow distros to override""" + locale = self.default_locale + try: + locale = self.distro.get_locale() + except NotImplementedError: + pass + return locale @property def availability_zone(self): diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index 2330ad52..72d3aad6 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -1,67 +1,85 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import (CiTestCase, mock) - -from cloudinit.distros.debian import apply_locale +from cloudinit import distros from cloudinit import util +from ..helpers import (FilesystemMockingTestCase, mock) @mock.patch("cloudinit.distros.debian.util.subp") -class TestDebianApplyLocale(CiTestCase): +class TestDebianApplyLocale(FilesystemMockingTestCase): + + def setUp(self): + super(TestDebianApplyLocale, self).setUp() + self.new_root = self.tmp_dir() + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + self.spath = self.tmp_path('etc/default/locale', self.new_root) + cls = distros.fetch("debian") + self.distro = cls("debian", {}, None) + def test_no_rerun(self, m_subp): """If system has defined locale, no re-run is expected.""" - spath = self.tmp_path("default-locale") m_subp.return_value = (None, None) locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=%s\n' % locale, omode="w") - apply_locale(locale, sys_path=spath) + util.write_file(self.spath, 'LANG=%s\n' % locale, omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) m_subp.assert_not_called() + def test_no_regen_on_c_utf8(self, m_subp): + """If locale is set to C.UTF8, do not attempt to call locale-gen""" + m_subp.return_value = (None, None) + locale = 'C.UTF-8' + util.write_file(self.spath, 'LANG=%s\n' % 'en_US.UTF-8', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) + self.assertEqual( + [['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + def test_rerun_if_different(self, m_subp): """If system has different locale, locale-gen should be called.""" - spath = self.tmp_path("default-locale") m_subp.return_value = (None, None) locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=fr_FR.UTF-8', omode="w") - apply_locale(locale, sys_path=spath) + util.write_file(self.spath, 'LANG=fr_FR.UTF-8', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + ['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) def test_rerun_if_no_file(self, m_subp): """If system has no locale file, locale-gen should be called.""" - spath = self.tmp_path("default-locale") m_subp.return_value = (None, None) locale = 'en_US.UTF-8' - apply_locale(locale, sys_path=spath) + self.distro.apply_locale(locale, out_fn=self.spath) self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + ['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) def test_rerun_on_unset_system_locale(self, m_subp): """If system has unset locale, locale-gen should be called.""" m_subp.return_value = (None, None) - spath = self.tmp_path("default-locale") locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=', omode="w") - apply_locale(locale, sys_path=spath) + util.write_file(self.spath, 'LANG=', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath) self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + ['update-locale', '--locale-file=' + self.spath, + 'LANG=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) def test_rerun_on_mismatched_keys(self, m_subp): """If key is LC_ALL and system has only LANG, rerun is expected.""" m_subp.return_value = (None, None) - spath = self.tmp_path("default-locale") locale = 'en_US.UTF-8' - util.write_file(spath, 'LANG=', omode="w") - apply_locale(locale, sys_path=spath, keyname='LC_ALL') + util.write_file(self.spath, 'LANG=', omode="w") + self.distro.apply_locale(locale, out_fn=self.spath, keyname='LC_ALL') self.assertEqual( [['locale-gen', locale], - ['update-locale', '--locale-file=' + spath, + ['update-locale', '--locale-file=' + self.spath, 'LC_ALL=%s' % locale]], [p[0][0] for p in m_subp.call_args_list]) @@ -69,14 +87,14 @@ class TestDebianApplyLocale(CiTestCase): """locale as None or "" is invalid and should raise ValueError.""" with self.assertRaises(ValueError) as ctext_m: - apply_locale(None) + self.distro.apply_locale(None) m_subp.assert_not_called() self.assertEqual( 'Failed to provide locale value.', str(ctext_m.exception)) with self.assertRaises(ValueError) as ctext_m: - apply_locale("") + self.distro.apply_locale("") m_subp.assert_not_called() self.assertEqual( 'Failed to provide locale value.', str(ctext_m.exception)) diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index c9be277e..b355a19e 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -228,5 +228,21 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): os.symlink('/', '/run/systemd/system') self.assertFalse(d.uses_systemd()) + @mock.patch('cloudinit.distros.debian.read_system_locale') + def test_get_locale_ubuntu(self, m_locale): + """Test ubuntu distro returns locale set to C.UTF-8""" + m_locale.return_value = 'C.UTF-8' + cls = distros.fetch("ubuntu") + d = cls("ubuntu", {}, None) + locale = d.get_locale() + self.assertEqual('C.UTF-8', locale) + + def test_get_locale_rhel(self): + """Test rhel distro returns NotImplementedError exception""" + cls = distros.fetch("rhel") + d = cls("rhel", {}, None) + with self.assertRaises(NotImplementedError): + d.get_locale() + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index 929f786e..1873c3e1 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -11,7 +11,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from ..helpers import (FilesystemMockingTestCase, mock) import logging import shutil @@ -20,7 +20,8 @@ import tempfile LOG = logging.getLogger(__name__) -class TestDebug(t_help.FilesystemMockingTestCase): +@mock.patch('cloudinit.distros.debian.read_system_locale') +class TestDebug(FilesystemMockingTestCase): def setUp(self): super(TestDebug, self).setUp() self.new_root = tempfile.mkdtemp() @@ -36,7 +37,8 @@ class TestDebug(t_help.FilesystemMockingTestCase): ds.metadata.update(metadata) return cloud.Cloud(ds, paths, {}, d, None) - def test_debug_write(self): + def test_debug_write(self, m_locale): + m_locale.return_value = 'en_US.UTF-8' cfg = { 'abc': '123', 'c': u'\u20a0', @@ -54,7 +56,8 @@ class TestDebug(t_help.FilesystemMockingTestCase): for k in cfg.keys(): self.assertIn(k, contents) - def test_debug_no_write(self): + def test_debug_no_write(self, m_locale): + m_locale.return_value = 'en_US.UTF-8' cfg = { 'abc': '123', 'debug': { diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index cba5cae8..a789db32 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -20,6 +20,8 @@ from configobj import ConfigObj from six import BytesIO import logging +import mock +import os import shutil import tempfile @@ -27,6 +29,9 @@ LOG = logging.getLogger(__name__) class TestLocale(t_help.FilesystemMockingTestCase): + + with_logs = True + def setUp(self): super(TestLocale, self).setUp() self.new_root = tempfile.mkdtemp() @@ -60,4 +65,47 @@ class TestLocale(t_help.FilesystemMockingTestCase): else: self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) + def test_set_locale_sles_default(self): + cfg = {} + cc = self._get_cloud('sles') + cc_locale.handle('cc_locale', cfg, cc, LOG, []) + + if cc.distro.uses_systemd(): + locale_conf = cc.distro.systemd_locale_conf_fn + keyname = 'LANG' + else: + locale_conf = cc.distro.locale_conf_fn + keyname = 'RC_LANG' + + contents = util.load_file(locale_conf, decode=False) + n_cfg = ConfigObj(BytesIO(contents)) + self.assertEqual({keyname: 'en_US.UTF-8'}, dict(n_cfg)) + + def test_locale_update_config_if_different_than_default(self): + """Test cc_locale writes updates conf if different than default""" + locale_conf = os.path.join(self.new_root, "etc/default/locale") + util.write_file(locale_conf, 'LANG="en_US.UTF-8"\n') + cfg = {'locale': 'C.UTF-8'} + cc = self._get_cloud('ubuntu') + with mock.patch('cloudinit.distros.debian.util.subp') as m_subp: + with mock.patch('cloudinit.distros.debian.LOCALE_CONF_FN', + locale_conf): + cc_locale.handle('cc_locale', cfg, cc, LOG, []) + m_subp.assert_called_with(['update-locale', + '--locale-file=%s' % locale_conf, + 'LANG=C.UTF-8'], capture=False) + + def test_locale_rhel_defaults_en_us_utf8(self): + """Test cc_locale gets en_US.UTF-8 from distro get_locale fallback""" + cfg = {} + cc = self._get_cloud('rhel') + update_sysconfig = 'cloudinit.distros.rhel_util.update_sysconfig_file' + with mock.patch.object(cc.distro, 'uses_systemd') as m_use_sd: + m_use_sd.return_value = True + with mock.patch(update_sysconfig) as m_update_syscfg: + cc_locale.handle('cc_locale', cfg, cc, LOG, []) + m_update_syscfg.assert_called_with('/etc/locale.conf', + {'LANG': 'en_US.UTF-8'}) + + # vi: ts=4 expandtab -- cgit v1.2.3 From a3649e03206a3596131413956ea7ecc18790ec73 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 5 Sep 2017 11:03:59 -0600 Subject: relocate tests/unittests/helpers.py to cloudinit/tests This moves the base test case classes into into cloudinit/tests and updates all the corresponding imports. --- cloudinit/analyze/tests/test_dump.py | 2 +- cloudinit/net/tests/test_dhcp.py | 2 +- cloudinit/net/tests/test_init.py | 2 +- cloudinit/tests/helpers.py | 395 +++++++++++++++++++++ cloudinit/tests/test_url_helper.py | 2 +- tests/unittests/helpers.py | 391 -------------------- tests/unittests/test__init__.py | 2 +- tests/unittests/test_atomic_helper.py | 2 +- tests/unittests/test_builtin_handlers.py | 2 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_cs_util.py | 2 +- tests/unittests/test_data.py | 2 +- tests/unittests/test_datasource/test_aliyun.py | 2 +- tests/unittests/test_datasource/test_altcloud.py | 2 +- tests/unittests/test_datasource/test_azure.py | 6 +- .../unittests/test_datasource/test_azure_helper.py | 2 +- tests/unittests/test_datasource/test_cloudsigma.py | 2 +- tests/unittests/test_datasource/test_cloudstack.py | 2 +- tests/unittests/test_datasource/test_common.py | 2 +- .../unittests/test_datasource/test_configdrive.py | 2 +- .../unittests/test_datasource/test_digitalocean.py | 2 +- tests/unittests/test_datasource/test_ec2.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_maas.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 2 +- tests/unittests/test_datasource/test_ovf.py | 2 +- tests/unittests/test_datasource/test_scaleway.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 2 +- tests/unittests/test_distros/test_arch.py | 2 +- tests/unittests/test_distros/test_create_users.py | 2 +- tests/unittests/test_distros/test_debian.py | 2 +- tests/unittests/test_distros/test_generic.py | 2 +- tests/unittests/test_distros/test_netconfig.py | 2 +- tests/unittests/test_distros/test_opensuse.py | 2 +- tests/unittests/test_distros/test_resolv.py | 2 +- tests/unittests/test_distros/test_sles.py | 2 +- tests/unittests/test_distros/test_sysconfig.py | 2 +- .../test_distros/test_user_data_normalize.py | 2 +- tests/unittests/test_ds_identify.py | 3 +- tests/unittests/test_ec2_util.py | 2 +- tests/unittests/test_filters/test_launch_index.py | 2 +- .../test_handler/test_handler_apt_conf_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v3.py | 2 +- .../test_handler/test_handler_apt_source_v1.py | 2 +- .../test_handler/test_handler_apt_source_v3.py | 2 +- .../test_handler/test_handler_ca_certs.py | 2 +- tests/unittests/test_handler/test_handler_chef.py | 2 +- tests/unittests/test_handler/test_handler_debug.py | 2 +- .../test_handler/test_handler_disk_setup.py | 2 +- .../test_handler/test_handler_growpart.py | 2 +- .../test_handler/test_handler_landscape.py | 5 +- .../unittests/test_handler/test_handler_locale.py | 2 +- tests/unittests/test_handler/test_handler_lxd.py | 2 +- .../test_handler/test_handler_mcollective.py | 2 +- .../unittests/test_handler/test_handler_mounts.py | 2 +- tests/unittests/test_handler/test_handler_ntp.py | 2 +- .../test_handler/test_handler_power_state.py | 4 +- .../unittests/test_handler/test_handler_puppet.py | 2 +- .../unittests/test_handler/test_handler_rsyslog.py | 2 +- .../unittests/test_handler/test_handler_runcmd.py | 2 +- .../test_handler/test_handler_seed_random.py | 2 +- .../test_handler/test_handler_set_hostname.py | 2 +- .../unittests/test_handler/test_handler_snappy.py | 4 +- .../test_handler/test_handler_spacewalk.py | 2 +- .../test_handler/test_handler_timezone.py | 2 +- .../test_handler/test_handler_write_files.py | 2 +- .../test_handler/test_handler_yum_add_repo.py | 2 +- tests/unittests/test_handler/test_schema.py | 2 +- tests/unittests/test_helpers.py | 2 +- tests/unittests/test_log.py | 2 +- tests/unittests/test_merging.py | 2 +- tests/unittests/test_net.py | 8 +- tests/unittests/test_pathprefix2dict.py | 2 +- tests/unittests/test_registry.py | 2 +- tests/unittests/test_reporting.py | 2 +- tests/unittests/test_rh_subscription.py | 2 +- tests/unittests/test_runs/test_merge_run.py | 2 +- tests/unittests/test_runs/test_simple_run.py | 2 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 2 +- tests/unittests/test_util.py | 2 +- tests/unittests/test_version.py | 2 +- tests/unittests/test_vmware_config_file.py | 2 +- 86 files changed, 491 insertions(+), 482 deletions(-) create mode 100644 cloudinit/tests/helpers.py delete mode 100644 tests/unittests/helpers.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/analyze/tests/test_dump.py b/cloudinit/analyze/tests/test_dump.py index 2c0885d0..f4c42841 100644 --- a/cloudinit/analyze/tests/test_dump.py +++ b/cloudinit/analyze/tests/test_dump.py @@ -6,7 +6,7 @@ from textwrap import dedent from cloudinit.analyze.dump import ( dump_events, parse_ci_logline, parse_timestamp) from cloudinit.util import subp, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseTimestamp(CiTestCase): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 47d8d461..4a37e98a 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -8,7 +8,7 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery) from cloudinit.util import ensure_file, write_file -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestParseDHCPLeasesFile(CiTestCase): diff --git a/cloudinit/net/tests/test_init.py b/cloudinit/net/tests/test_init.py index cc052a7d..8cb4114e 100644 --- a/cloudinit/net/tests/test_init.py +++ b/cloudinit/net/tests/test_init.py @@ -7,7 +7,7 @@ import os import cloudinit.net as net from cloudinit.util import ensure_file, write_file, ProcessExecutionError -from tests.unittests.helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestSysDevPath(CiTestCase): diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py new file mode 100644 index 00000000..28e26622 --- /dev/null +++ b/cloudinit/tests/helpers.py @@ -0,0 +1,395 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from __future__ import print_function + +import functools +import json +import logging +import os +import shutil +import sys +import tempfile +import unittest + +import mock +import six +import unittest2 + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + +from cloudinit import helpers as ch +from cloudinit import util + +# Used for skipping tests +SkipTest = unittest2.SkipTest + +# Used for detecting different python versions +PY2 = False +PY26 = False +PY27 = False +PY3 = False + +_PY_VER = sys.version_info +_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] +if (_PY_MAJOR, _PY_MINOR) <= (2, 6): + if (_PY_MAJOR, _PY_MINOR) == (2, 6): + PY26 = True + if (_PY_MAJOR, _PY_MINOR) >= (2, 0): + PY2 = True +else: + if (_PY_MAJOR, _PY_MINOR) == (2, 7): + PY27 = True + PY2 = True + if (_PY_MAJOR, _PY_MINOR) >= (3, 0): + PY3 = True + + +# Makes the old path start +# with new base instead of whatever +# it previously had +def rebase_path(old_path, new_base): + if old_path.startswith(new_base): + # Already handled... + return old_path + # Retarget the base of that path + # to the new base instead of the + # old one... + path = os.path.join(new_base, old_path.lstrip("/")) + path = os.path.abspath(path) + return path + + +# Can work on anything that takes a path as arguments +def retarget_many_wrapper(new_base, am, old_func): + def wrapper(*args, **kwds): + n_args = list(args) + nam = am + if am == -1: + nam = len(n_args) + for i in range(0, nam): + path = args[i] + # patchOS() wraps various os and os.path functions, however in + # Python 3 some of these now accept file-descriptors (integers). + # That breaks rebase_path() so in lieu of a better solution, just + # don't rebase if we get a fd. + if isinstance(path, six.string_types): + n_args[i] = rebase_path(path, new_base) + return old_func(*n_args, **kwds) + return wrapper + + +class TestCase(unittest2.TestCase): + + def reset_global_state(self): + """Reset any global state to its original settings. + + cloudinit caches some values in cloudinit.util. Unit tests that + involved those cached paths were then subject to failure if the order + of invocation changed (LP: #1703697). + + This function resets any of these global state variables to their + initial state. + + In the future this should really be done with some registry that + can then be cleaned in a more obvious way. + """ + util.PROC_CMDLINE = None + util._DNS_REDIRECT_IP = None + util._LSB_RELEASE = {} + + def setUp(self): + super(TestCase, self).setUp() + self.reset_global_state() + + +class CiTestCase(TestCase): + """This is the preferred test case base class unless user + needs other test case classes below.""" + + # Subclass overrides for specific test behavior + # Whether or not a unit test needs logfile setup + with_logs = False + + def setUp(self): + super(CiTestCase, self).setUp() + if self.with_logs: + # Create a log handler so unit tests can search expected logs. + self.logger = logging.getLogger() + self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') + handler = logging.StreamHandler(self.logs) + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] + + def tearDown(self): + if self.with_logs: + # Remove the handler we setup + logging.getLogger().handlers = self.old_handlers + super(CiTestCase, self).tearDown() + + def tmp_dir(self, dir=None, cleanup=True): + # return a full path to a temporary directory that will be cleaned up. + if dir is None: + tmpd = tempfile.mkdtemp( + prefix="ci-%s." % self.__class__.__name__) + else: + tmpd = tempfile.mkdtemp(dir=dir) + self.addCleanup(functools.partial(shutil.rmtree, tmpd)) + return tmpd + + def tmp_path(self, path, dir=None): + # return an absolute path to 'path' under dir. + # if dir is None, one will be created with tmp_dir() + # the file is not created or modified. + if dir is None: + dir = self.tmp_dir() + return os.path.normpath(os.path.abspath(os.path.join(dir, path))) + + +class ResourceUsingTestCase(CiTestCase): + + def setUp(self): + super(ResourceUsingTestCase, self).setUp() + self.resource_path = None + + def resourceLocation(self, subname=None): + if self.resource_path is None: + paths = [ + os.path.join('tests', 'data'), + os.path.join('data'), + os.path.join(os.pardir, 'tests', 'data'), + os.path.join(os.pardir, 'data'), + ] + for p in paths: + if os.path.isdir(p): + self.resource_path = p + break + self.assertTrue((self.resource_path and + os.path.isdir(self.resource_path)), + msg="Unable to locate test resource data path!") + if not subname: + return self.resource_path + return os.path.join(self.resource_path, subname) + + def readResource(self, name): + where = self.resourceLocation(name) + with open(where, 'r') as fh: + return fh.read() + + def getCloudPaths(self, ds=None): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + cp = ch.Paths({'cloud_dir': tmpdir, + 'templates_dir': self.resourceLocation()}, + ds=ds) + return cp + + +class FilesystemMockingTestCase(ResourceUsingTestCase): + + def setUp(self): + super(FilesystemMockingTestCase, self).setUp() + self.patched_funcs = ExitStack() + + def tearDown(self): + self.patched_funcs.close() + ResourceUsingTestCase.tearDown(self) + + def replicateTestRoot(self, example_root, target_root): + real_root = self.resourceLocation() + real_root = os.path.join(real_root, 'roots', example_root) + for (dir_path, _dirnames, filenames) in os.walk(real_root): + real_path = dir_path + make_path = rebase_path(real_path[len(real_root):], target_root) + util.ensure_dir(make_path) + for f in filenames: + real_path = util.abs_join(real_path, f) + make_path = util.abs_join(make_path, f) + shutil.copy(real_path, make_path) + + def patchUtils(self, new_root): + patch_funcs = { + util: [('write_file', 1), + ('append_file', 1), + ('load_file', 1), + ('ensure_dir', 1), + ('chmod', 1), + ('delete_dir_contents', 1), + ('del_file', 1), + ('sym_link', -1), + ('copy', -1)], + } + for (mod, funcs) in patch_funcs.items(): + for (f, am) in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, am, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + # Handle subprocess calls + func = getattr(util, 'subp') + + def nsubp(*_args, **_kwargs): + return ('', '') + + self.patched_funcs.enter_context( + mock.patch.object(util, 'subp', nsubp)) + + def null_func(*_args, **_kwargs): + return None + + for f in ['chownbyid', 'chownbyname']: + self.patched_funcs.enter_context( + mock.patch.object(util, f, null_func)) + + def patchOS(self, new_root): + patch_funcs = { + os.path: [('isfile', 1), ('exists', 1), + ('islink', 1), ('isdir', 1)], + os: [('listdir', 1), ('mkdir', 1), + ('lstat', 1), ('symlink', 2)], + } + for (mod, funcs) in patch_funcs.items(): + for f, nargs in funcs: + func = getattr(mod, f) + trap_func = retarget_many_wrapper(new_root, nargs, func) + self.patched_funcs.enter_context( + mock.patch.object(mod, f, trap_func)) + + def patchOpen(self, new_root): + trap_func = retarget_many_wrapper(new_root, 1, open) + name = 'builtins.open' if PY3 else '__builtin__.open' + self.patched_funcs.enter_context(mock.patch(name, trap_func)) + + def patchStdoutAndStderr(self, stdout=None, stderr=None): + if stdout is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stdout', stdout)) + if stderr is not None: + self.patched_funcs.enter_context( + mock.patch.object(sys, 'stderr', stderr)) + + def reRoot(self, root=None): + if root is None: + root = self.tmp_dir() + self.patchUtils(root) + self.patchOS(root) + return root + + +class HttprettyTestCase(CiTestCase): + # necessary as http_proxy gets in the way of httpretty + # https://github.com/gabrielfalcao/HTTPretty/issues/122 + + def setUp(self): + self.restore_proxy = os.environ.get('http_proxy') + if self.restore_proxy is not None: + del os.environ['http_proxy'] + super(HttprettyTestCase, self).setUp() + + def tearDown(self): + if self.restore_proxy: + os.environ['http_proxy'] = self.restore_proxy + super(HttprettyTestCase, self).tearDown() + + +def populate_dir(path, files): + if not os.path.exists(path): + os.makedirs(path) + ret = [] + for (name, content) in files.items(): + p = os.path.sep.join([path, name]) + util.ensure_dir(os.path.dirname(p)) + with open(p, "wb") as fp: + if isinstance(content, six.binary_type): + fp.write(content) + else: + fp.write(content.encode('utf-8')) + fp.close() + ret.append(p) + + return ret + + +def dir2dict(startdir, prefix=None): + flist = {} + if prefix is None: + prefix = startdir + for root, dirs, files in os.walk(startdir): + for fname in files: + fpath = os.path.join(root, fname) + key = fpath[len(prefix):] + flist[key] = util.load_file(fpath) + return flist + + +def json_dumps(data): + # print data in nicely formatted json. + return json.dumps(data, indent=1, sort_keys=True, + separators=(',', ': ')) + + +def wrap_and_call(prefix, mocks, func, *args, **kwargs): + """ + call func(args, **kwargs) with mocks applied, then unapplies mocks + nicer to read than repeating dectorators on each function + + prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None + mocks: dictionary of names (under 'prefix') to mock and either + a return value or a dictionary to pass to the mock.patch call + func: function to call with mocks applied + *args,**kwargs: arguments for 'func' + + return_value: return from 'func' + """ + delim = '.' + if prefix is None: + prefix = '' + prefix = prefix.rstrip(delim) + unwraps = [] + for fname, kw in mocks.items(): + if prefix: + fname = delim.join((prefix, fname)) + if not isinstance(kw, dict): + kw = {'return_value': kw} + p = mock.patch(fname, **kw) + p.start() + unwraps.append(p) + try: + return func(*args, **kwargs) + finally: + for p in unwraps: + p.stop() + + +try: + skipIf = unittest.skipIf +except AttributeError: + # Python 2.6. Doesn't have to be high fidelity. + def skipIf(condition, reason): + def decorator(func): + def wrapper(*args, **kws): + if condition: + return func(*args, **kws) + else: + print(reason, file=sys.stderr) + return wrapper + return decorator + + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + +# vi: ts=4 expandtab diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py index 349110d9..b778a3a7 100644 --- a/cloudinit/tests/test_url_helper.py +++ b/cloudinit/tests/test_url_helper.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.url_helper import oauth_headers -from tests.unittests.helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf try: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py deleted file mode 100644 index bf1dc5df..00000000 --- a/tests/unittests/helpers.py +++ /dev/null @@ -1,391 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -from __future__ import print_function - -import functools -import json -import logging -import os -import shutil -import sys -import tempfile -import unittest - -import mock -import six -import unittest2 - -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - -from cloudinit import helpers as ch -from cloudinit import util - -# Used for skipping tests -SkipTest = unittest2.SkipTest - -# Used for detecting different python versions -PY2 = False -PY26 = False -PY27 = False -PY3 = False - -_PY_VER = sys.version_info -_PY_MAJOR, _PY_MINOR, _PY_MICRO = _PY_VER[0:3] -if (_PY_MAJOR, _PY_MINOR) <= (2, 6): - if (_PY_MAJOR, _PY_MINOR) == (2, 6): - PY26 = True - if (_PY_MAJOR, _PY_MINOR) >= (2, 0): - PY2 = True -else: - if (_PY_MAJOR, _PY_MINOR) == (2, 7): - PY27 = True - PY2 = True - if (_PY_MAJOR, _PY_MINOR) >= (3, 0): - PY3 = True - - -# Makes the old path start -# with new base instead of whatever -# it previously had -def rebase_path(old_path, new_base): - if old_path.startswith(new_base): - # Already handled... - return old_path - # Retarget the base of that path - # to the new base instead of the - # old one... - path = os.path.join(new_base, old_path.lstrip("/")) - path = os.path.abspath(path) - return path - - -# Can work on anything that takes a path as arguments -def retarget_many_wrapper(new_base, am, old_func): - def wrapper(*args, **kwds): - n_args = list(args) - nam = am - if am == -1: - nam = len(n_args) - for i in range(0, nam): - path = args[i] - # patchOS() wraps various os and os.path functions, however in - # Python 3 some of these now accept file-descriptors (integers). - # That breaks rebase_path() so in lieu of a better solution, just - # don't rebase if we get a fd. - if isinstance(path, six.string_types): - n_args[i] = rebase_path(path, new_base) - return old_func(*n_args, **kwds) - return wrapper - - -class TestCase(unittest2.TestCase): - def reset_global_state(self): - """Reset any global state to its original settings. - - cloudinit caches some values in cloudinit.util. Unit tests that - involved those cached paths were then subject to failure if the order - of invocation changed (LP: #1703697). - - This function resets any of these global state variables to their - initial state. - - In the future this should really be done with some registry that - can then be cleaned in a more obvious way. - """ - util.PROC_CMDLINE = None - util._DNS_REDIRECT_IP = None - util._LSB_RELEASE = {} - - def setUp(self): - super(unittest2.TestCase, self).setUp() - self.reset_global_state() - - -class CiTestCase(TestCase): - """This is the preferred test case base class unless user - needs other test case classes below.""" - - # Subclass overrides for specific test behavior - # Whether or not a unit test needs logfile setup - with_logs = False - - def setUp(self): - super(CiTestCase, self).setUp() - if self.with_logs: - # Create a log handler so unit tests can search expected logs. - self.logger = logging.getLogger() - self.logs = six.StringIO() - formatter = logging.Formatter('%(levelname)s: %(message)s') - handler = logging.StreamHandler(self.logs) - handler.setFormatter(formatter) - self.old_handlers = self.logger.handlers - self.logger.handlers = [handler] - - def tearDown(self): - if self.with_logs: - # Remove the handler we setup - logging.getLogger().handlers = self.old_handlers - super(CiTestCase, self).tearDown() - - def tmp_dir(self, dir=None, cleanup=True): - # return a full path to a temporary directory that will be cleaned up. - if dir is None: - tmpd = tempfile.mkdtemp( - prefix="ci-%s." % self.__class__.__name__) - else: - tmpd = tempfile.mkdtemp(dir=dir) - self.addCleanup(functools.partial(shutil.rmtree, tmpd)) - return tmpd - - def tmp_path(self, path, dir=None): - # return an absolute path to 'path' under dir. - # if dir is None, one will be created with tmp_dir() - # the file is not created or modified. - if dir is None: - dir = self.tmp_dir() - return os.path.normpath(os.path.abspath(os.path.join(dir, path))) - - -class ResourceUsingTestCase(CiTestCase): - def setUp(self): - super(ResourceUsingTestCase, self).setUp() - self.resource_path = None - - def resourceLocation(self, subname=None): - if self.resource_path is None: - paths = [ - os.path.join('tests', 'data'), - os.path.join('data'), - os.path.join(os.pardir, 'tests', 'data'), - os.path.join(os.pardir, 'data'), - ] - for p in paths: - if os.path.isdir(p): - self.resource_path = p - break - self.assertTrue((self.resource_path and - os.path.isdir(self.resource_path)), - msg="Unable to locate test resource data path!") - if not subname: - return self.resource_path - return os.path.join(self.resource_path, subname) - - def readResource(self, name): - where = self.resourceLocation(name) - with open(where, 'r') as fh: - return fh.read() - - def getCloudPaths(self, ds=None): - tmpdir = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, tmpdir) - cp = ch.Paths({'cloud_dir': tmpdir, - 'templates_dir': self.resourceLocation()}, - ds=ds) - return cp - - -class FilesystemMockingTestCase(ResourceUsingTestCase): - def setUp(self): - super(FilesystemMockingTestCase, self).setUp() - self.patched_funcs = ExitStack() - - def tearDown(self): - self.patched_funcs.close() - ResourceUsingTestCase.tearDown(self) - - def replicateTestRoot(self, example_root, target_root): - real_root = self.resourceLocation() - real_root = os.path.join(real_root, 'roots', example_root) - for (dir_path, _dirnames, filenames) in os.walk(real_root): - real_path = dir_path - make_path = rebase_path(real_path[len(real_root):], target_root) - util.ensure_dir(make_path) - for f in filenames: - real_path = util.abs_join(real_path, f) - make_path = util.abs_join(make_path, f) - shutil.copy(real_path, make_path) - - def patchUtils(self, new_root): - patch_funcs = { - util: [('write_file', 1), - ('append_file', 1), - ('load_file', 1), - ('ensure_dir', 1), - ('chmod', 1), - ('delete_dir_contents', 1), - ('del_file', 1), - ('sym_link', -1), - ('copy', -1)], - } - for (mod, funcs) in patch_funcs.items(): - for (f, am) in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, am, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - # Handle subprocess calls - func = getattr(util, 'subp') - - def nsubp(*_args, **_kwargs): - return ('', '') - - self.patched_funcs.enter_context( - mock.patch.object(util, 'subp', nsubp)) - - def null_func(*_args, **_kwargs): - return None - - for f in ['chownbyid', 'chownbyname']: - self.patched_funcs.enter_context( - mock.patch.object(util, f, null_func)) - - def patchOS(self, new_root): - patch_funcs = { - os.path: [('isfile', 1), ('exists', 1), - ('islink', 1), ('isdir', 1)], - os: [('listdir', 1), ('mkdir', 1), - ('lstat', 1), ('symlink', 2)], - } - for (mod, funcs) in patch_funcs.items(): - for f, nargs in funcs: - func = getattr(mod, f) - trap_func = retarget_many_wrapper(new_root, nargs, func) - self.patched_funcs.enter_context( - mock.patch.object(mod, f, trap_func)) - - def patchOpen(self, new_root): - trap_func = retarget_many_wrapper(new_root, 1, open) - name = 'builtins.open' if PY3 else '__builtin__.open' - self.patched_funcs.enter_context(mock.patch(name, trap_func)) - - def patchStdoutAndStderr(self, stdout=None, stderr=None): - if stdout is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stdout', stdout)) - if stderr is not None: - self.patched_funcs.enter_context( - mock.patch.object(sys, 'stderr', stderr)) - - def reRoot(self, root=None): - if root is None: - root = self.tmp_dir() - self.patchUtils(root) - self.patchOS(root) - return root - - -class HttprettyTestCase(CiTestCase): - # necessary as http_proxy gets in the way of httpretty - # https://github.com/gabrielfalcao/HTTPretty/issues/122 - def setUp(self): - self.restore_proxy = os.environ.get('http_proxy') - if self.restore_proxy is not None: - del os.environ['http_proxy'] - super(HttprettyTestCase, self).setUp() - - def tearDown(self): - if self.restore_proxy: - os.environ['http_proxy'] = self.restore_proxy - super(HttprettyTestCase, self).tearDown() - - -def populate_dir(path, files): - if not os.path.exists(path): - os.makedirs(path) - ret = [] - for (name, content) in files.items(): - p = os.path.sep.join([path, name]) - util.ensure_dir(os.path.dirname(p)) - with open(p, "wb") as fp: - if isinstance(content, six.binary_type): - fp.write(content) - else: - fp.write(content.encode('utf-8')) - fp.close() - ret.append(p) - - return ret - - -def dir2dict(startdir, prefix=None): - flist = {} - if prefix is None: - prefix = startdir - for root, dirs, files in os.walk(startdir): - for fname in files: - fpath = os.path.join(root, fname) - key = fpath[len(prefix):] - flist[key] = util.load_file(fpath) - return flist - - -def json_dumps(data): - # print data in nicely formatted json. - return json.dumps(data, indent=1, sort_keys=True, - separators=(',', ': ')) - - -def wrap_and_call(prefix, mocks, func, *args, **kwargs): - """ - call func(args, **kwargs) with mocks applied, then unapplies mocks - nicer to read than repeating dectorators on each function - - prefix: prefix for mock names (e.g. 'cloudinit.stages.util') or None - mocks: dictionary of names (under 'prefix') to mock and either - a return value or a dictionary to pass to the mock.patch call - func: function to call with mocks applied - *args,**kwargs: arguments for 'func' - - return_value: return from 'func' - """ - delim = '.' - if prefix is None: - prefix = '' - prefix = prefix.rstrip(delim) - unwraps = [] - for fname, kw in mocks.items(): - if prefix: - fname = delim.join((prefix, fname)) - if not isinstance(kw, dict): - kw = {'return_value': kw} - p = mock.patch(fname, **kw) - p.start() - unwraps.append(p) - try: - return func(*args, **kwargs) - finally: - for p in unwraps: - p.stop() - - -try: - skipIf = unittest.skipIf -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skipIf(condition, reason): - def decorator(func): - def wrapper(*args, **kws): - if condition: - return func(*args, **kws) - else: - print(reason, file=sys.stderr) - return wrapper - return decorator - - -# older versions of mock do not have the useful 'assert_not_called' -if not hasattr(mock.Mock, 'assert_not_called'): - def __mock_assert_not_called(mmock): - if mmock.call_count != 0: - msg = ("[citest] Expected '%s' to not have been called. " - "Called %s times." % - (mmock._mock_name or 'mock', mmock.call_count)) - raise AssertionError(msg) - mock.Mock.assert_not_called = __mock_assert_not_called - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 781f6d54..25878d7a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -12,7 +12,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase, CiTestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, CiTestCase, ExitStack, mock class FakeModule(handlers.Handler): diff --git a/tests/unittests/test_atomic_helper.py b/tests/unittests/test_atomic_helper.py index 515919d8..0101b0e3 100644 --- a/tests/unittests/test_atomic_helper.py +++ b/tests/unittests/test_atomic_helper.py @@ -6,7 +6,7 @@ import stat from cloudinit import atomic_helper -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase class TestAtomicHelper(CiTestCase): diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index dd9d0357..9751ed95 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -11,7 +11,7 @@ try: except ImportError: import mock -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 12f01852..495bdc9f 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -2,7 +2,7 @@ import six -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cmd import main as cli diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index b8f5031c..ee88520d 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -2,7 +2,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.cs_utils import Cepko diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4ad86bb6..6d621d26 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,7 +27,7 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -from . import helpers +from cloudinit.tests import helpers INSTANCE_ID = "i-testing" diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 996560e4..82ee9714 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -5,9 +5,9 @@ import httpretty import mock import os -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceAliYun as ay +from cloudinit.tests import helpers as test_helpers DEFAULT_METADATA = { 'instance-id': 'aliyun-test-vm-00', diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 9c46abc1..3b274d90 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -18,7 +18,7 @@ import tempfile from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import cloudinit.sources.DataSourceAltCloud as dsac diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 20e70fb7..0a117771 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -6,8 +6,8 @@ from cloudinit.sources import DataSourceAzure as dsaz from cloudinit.util import find_freebsd_part from cloudinit.util import get_path_dev_freebsd -from ..helpers import (CiTestCase, TestCase, populate_dir, mock, - ExitStack, PY26, SkipTest) +from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock, + ExitStack, PY26, SkipTest) import crypt import os @@ -871,6 +871,7 @@ class TestLoadAzureDsDir(CiTestCase): class TestReadAzureOvf(TestCase): + def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(dsaz.BrokenAzureDataSource, @@ -1079,6 +1080,7 @@ class TestCanDevBeReformatted(CiTestCase): class TestAzureNetExists(CiTestCase): + def test_azure_net_must_exist_for_legacy_objpkl(self): """DataSourceAzureNet must exist for old obj.pkl files that reference it.""" diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b2d2971b..80ce003d 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -3,7 +3,7 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import ExitStack, mock, TestCase +from cloudinit.tests.helpers import ExitStack, mock, TestCase GOAL_STATE_TEMPLATE = """\ diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 5997102c..e4c59907 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -6,7 +6,7 @@ from cloudinit.cs_utils import Cepko from cloudinit import sources from cloudinit.sources import DataSourceCloudSigma -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers SERVER_CONTEXT = { "cpu": 1000, diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index e94aad61..2dc90305 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase, mock, ExitStack +from cloudinit.tests.helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py index 4802f105..80b9c650 100644 --- a/tests/unittests/test_datasource/test_common.py +++ b/tests/unittests/test_datasource/test_common.py @@ -24,7 +24,7 @@ from cloudinit.sources import ( ) from cloudinit.sources import DataSourceNone as DSNone -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers DEFAULT_LOCAL = [ Azure.DataSourceAzure, diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 337be667..237c189b 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -15,7 +15,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase, ExitStack, mock +from cloudinit.tests.helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index e97a679a..f264f361 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -13,7 +13,7 @@ from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from cloudinit.sources.helpers import digitalocean -from ..helpers import mock, TestCase +from cloudinit.tests.helpers import mock, TestCase DO_MULTIPLE_KEYS = ["ssh-rsa AAAAB3NzaC1yc2EAAAA... test1@do.co", "ssh-rsa AAAAB3NzaC1yc2EAAAA... test2@do.co"] diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index b7a84e21..9fb90483 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -4,9 +4,9 @@ import copy import httpretty import mock -from .. import helpers as test_helpers from cloudinit import helpers from cloudinit.sources import DataSourceEc2 as ec2 +from cloudinit.tests import helpers as test_helpers # collected from api version 2016-09-02/ with diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index ad608bec..50e49a10 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceGCE -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers GCE_META = { diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index c1911bf4..289c6a40 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -8,7 +8,7 @@ import yaml from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from ..helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir try: from unittest import mock diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index ff294395..fea9156b 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir, mock, ExitStack +from cloudinit.tests.helpers import TestCase, populate_dir, mock, ExitStack import os import shutil diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index b0f8e435..e7d55692 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,7 +3,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import mock, populate_dir, TestCase +from cloudinit.tests.helpers import mock, populate_dir, TestCase import os import pwd diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c2905d1a..177e9808 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -9,7 +9,7 @@ import httpretty as hp import json import re -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from six.moves.urllib.parse import urlparse from six import StringIO diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 477cf8ed..9dbf4dd9 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -6,7 +6,7 @@ import base64 -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit.sources import DataSourceOVF as dsovf diff --git a/tests/unittests/test_datasource/test_scaleway.py b/tests/unittests/test_datasource/test_scaleway.py index 65d83ad7..436df9ee 100644 --- a/tests/unittests/test_datasource/test_scaleway.py +++ b/tests/unittests/test_datasource/test_scaleway.py @@ -9,7 +9,7 @@ from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceScaleway -from ..helpers import mock, HttprettyTestCase, TestCase +from cloudinit.tests.helpers import mock, HttprettyTestCase, TestCase class DataResponses(object): diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index e3c99bbb..933d5b63 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -33,7 +33,7 @@ import six from cloudinit import helpers as c_helpers from cloudinit.util import b64e -from ..helpers import mock, FilesystemMockingTestCase, TestCase +from cloudinit.tests.helpers import mock, FilesystemMockingTestCase, TestCase SDC_NICS = json.loads(""" [ diff --git a/tests/unittests/test_distros/test_arch.py b/tests/unittests/test_distros/test_arch.py index 3d4c9a70..a95ba3b5 100644 --- a/tests/unittests/test_distros/test_arch.py +++ b/tests/unittests/test_distros/test_arch.py @@ -3,7 +3,7 @@ from cloudinit.distros.arch import _render_network from cloudinit import util -from ..helpers import (CiTestCase, dir2dict) +from cloudinit.tests.helpers import (CiTestCase, dir2dict) from . import _get_distro diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 1d02f7bd..aa13670a 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit import distros -from ..helpers import (TestCase, mock) +from cloudinit.tests.helpers import (TestCase, mock) class MyBaseDistro(distros.Distro): diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py index 72d3aad6..da16a797 100644 --- a/tests/unittests/test_distros/test_debian.py +++ b/tests/unittests/test_distros/test_debian.py @@ -2,7 +2,7 @@ from cloudinit import distros from cloudinit import util -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) @mock.patch("cloudinit.distros.debian.util.subp") diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index b355a19e..791fe612 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -3,7 +3,7 @@ from cloudinit import distros from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import os import shutil diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 6d89dba8..c4bd11bc 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -12,7 +12,7 @@ try: except ImportError: from contextlib2 import ExitStack -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase from cloudinit import distros from cloudinit.distros.parsers.sys_conf import SysConf diff --git a/tests/unittests/test_distros/test_opensuse.py b/tests/unittests/test_distros/test_opensuse.py index bdb1d633..b9bb9b3e 100644 --- a/tests/unittests/test_distros/test_opensuse.py +++ b/tests/unittests/test_distros/test_opensuse.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 97168cf9..68ea0083 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -3,7 +3,7 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit.distros import rhel_util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import re import tempfile diff --git a/tests/unittests/test_distros/test_sles.py b/tests/unittests/test_distros/test_sles.py index c656aacc..33e3c457 100644 --- a/tests/unittests/test_distros/test_sles.py +++ b/tests/unittests/test_distros/test_sles.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from ..helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from . import _get_distro diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index 235ecebb..c1d5b693 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -4,7 +4,7 @@ import re from cloudinit.distros.parsers.sys_conf import SysConf -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase # Lots of good examples @ diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 88746e0a..0fa9cdb5 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -5,7 +5,7 @@ from cloudinit.distros import ug_util from cloudinit import helpers from cloudinit import settings -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import mock diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py index 8ccfe55c..1a81a89e 100644 --- a/tests/unittests/test_ds_identify.py +++ b/tests/unittests/test_ds_identify.py @@ -6,7 +6,8 @@ from uuid import uuid4 from cloudinit import safeyaml from cloudinit import util -from .helpers import CiTestCase, dir2dict, json_dumps, populate_dir +from cloudinit.tests.helpers import ( + CiTestCase, dir2dict, json_dumps, populate_dir) UNAME_MYSYS = ("Linux bart 4.4.0-62-generic #83-Ubuntu " "SMP Wed Jan 18 14:10:15 UTC 2017 x86_64 GNU/Linux") diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 65fdb519..af78997f 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -2,7 +2,7 @@ import httpretty as hp -from . import helpers +from cloudinit.tests import helpers from cloudinit import ec2_utils as eu from cloudinit import url_helper as uh diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 13137f6d..6364d38e 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -2,7 +2,7 @@ import copy -from .. import helpers +from cloudinit.tests import helpers from six.moves import filterfalse diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 554277ff..83f962a9 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import copy import os diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index f53ddbb2..d2b96f0b 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index 1ca915b4..f7608c28 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -24,7 +24,7 @@ from cloudinit.sources import DataSourceNone from cloudinit.distros.debian import Distro -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index 12502d05..3a3f95ca 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -20,7 +20,7 @@ from cloudinit.config import cc_apt_configure from cloudinit import gpg from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 292d3f59..7bb1b7c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -28,7 +28,7 @@ from cloudinit import util from cloudinit.config import cc_apt_configure from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index 7cee2c3f..06e14db0 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -5,7 +5,7 @@ from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 6a152ea2..e5785cfd 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -14,7 +14,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test_handler/test_handler_debug.py b/tests/unittests/test_handler/test_handler_debug.py index 1873c3e1..787ba350 100644 --- a/tests/unittests/test_handler/test_handler_debug.py +++ b/tests/unittests/test_handler/test_handler_debug.py @@ -11,7 +11,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from ..helpers import (FilesystemMockingTestCase, mock) +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock) import logging import shutil diff --git a/tests/unittests/test_handler/test_handler_disk_setup.py b/tests/unittests/test_handler/test_handler_disk_setup.py index 8a6d49ed..5afcacaf 100644 --- a/tests/unittests/test_handler/test_handler_disk_setup.py +++ b/tests/unittests/test_handler/test_handler_disk_setup.py @@ -3,7 +3,7 @@ import random from cloudinit.config import cc_disk_setup -from ..helpers import CiTestCase, ExitStack, mock, TestCase +from cloudinit.tests.helpers import CiTestCase, ExitStack, mock, TestCase class TestIsDiskUsed(TestCase): diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index c5fc8c9b..a3e46351 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -4,7 +4,7 @@ from cloudinit import cloud from cloudinit.config import cc_growpart from cloudinit import util -from ..helpers import TestCase +from cloudinit.tests.helpers import TestCase import errno import logging diff --git a/tests/unittests/test_handler/test_handler_landscape.py b/tests/unittests/test_handler/test_handler_landscape.py index 7c247fa9..db92a7e2 100644 --- a/tests/unittests/test_handler/test_handler_landscape.py +++ b/tests/unittests/test_handler/test_handler_landscape.py @@ -1,9 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. from cloudinit.config import cc_landscape -from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, wrap_and_call +from cloudinit.sources import DataSourceNone +from cloudinit.tests.helpers import (FilesystemMockingTestCase, mock, + wrap_and_call) from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index a789db32..e29a06f9 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 351226bf..f132a778 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_lxd from cloudinit.sources import DataSourceNoCloud from cloudinit import (distros, helpers, cloud) -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_mcollective.py b/tests/unittests/test_handler/test_handler_mcollective.py index 2a9f3823..7eec7352 100644 --- a/tests/unittests/test_handler/test_handler_mcollective.py +++ b/tests/unittests/test_handler/test_handler_mcollective.py @@ -4,7 +4,7 @@ from cloudinit import (cloud, distros, helpers, util) from cloudinit.config import cc_mcollective from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import configobj import logging diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py index 650ca0ec..fe492d4b 100644 --- a/tests/unittests/test_handler/test_handler_mounts.py +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -6,7 +6,7 @@ import tempfile from cloudinit.config import cc_mounts -from .. import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers try: from unittest import mock diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 83d5faa2..4f291248 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_ntp from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, mock, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock, skipIf import os diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index e382210d..85a0fe0a 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,8 +4,8 @@ import sys from cloudinit.config import cc_power_state_change as psc -from .. import helpers as t_help -from ..helpers import mock +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_puppet.py b/tests/unittests/test_handler/test_handler_puppet.py index 805c76ba..0b6e3b58 100644 --- a/tests/unittests/test_handler/test_handler_puppet.py +++ b/tests/unittests/test_handler/test_handler_puppet.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_puppet from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import CiTestCase, mock +from cloudinit.tests.helpers import CiTestCase, mock import logging diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index cca06678..8c8e2838 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -9,7 +9,7 @@ from cloudinit.config.cc_rsyslog import ( parse_remotes_line, remotes_to_rsyslog_cfg) from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help class TestLoadConfig(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_runcmd.py b/tests/unittests/test_handler/test_handler_runcmd.py index 7880ee72..374c1d31 100644 --- a/tests/unittests/test_handler/test_handler_runcmd.py +++ b/tests/unittests/test_handler/test_handler_runcmd.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_runcmd from cloudinit.sources import DataSourceNone from cloudinit import (distros, helpers, cloud, util) -from ..helpers import FilesystemMockingTestCase, skipIf +from cloudinit.tests.helpers import FilesystemMockingTestCase, skipIf import logging import os diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index e5e607fb..f60dedc2 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -22,7 +22,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 8165bf9a..abdc17e7 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -7,7 +7,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import util -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index e4d07622..76b79c29 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -7,9 +7,9 @@ from cloudinit.config.cc_snap_config import ( from cloudinit import (distros, helpers, cloud, util) from cloudinit.config.cc_snap_config import handle as snap_handle from cloudinit.sources import DataSourceNone -from ..helpers import FilesystemMockingTestCase, mock +from cloudinit.tests.helpers import FilesystemMockingTestCase, mock -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help import logging import os diff --git a/tests/unittests/test_handler/test_handler_spacewalk.py b/tests/unittests/test_handler/test_handler_spacewalk.py index 28b5892a..ddbf4a79 100644 --- a/tests/unittests/test_handler/test_handler_spacewalk.py +++ b/tests/unittests/test_handler/test_handler_spacewalk.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_spacewalk from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers import logging diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index c30fbdfe..27eedded 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -13,7 +13,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from .. import helpers as t_help +from cloudinit.tests import helpers as t_help from configobj import ConfigObj import logging diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index 1129e77d..7fa8fd21 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -4,7 +4,7 @@ from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import CiTestCase, FilesystemMockingTestCase +from cloudinit.tests.helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index c4396df5..b7adbe50 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -3,7 +3,7 @@ from cloudinit.config import cc_yum_add_repo from cloudinit import util -from .. import helpers +from cloudinit.tests import helpers try: from configparser import ConfigParser diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 640f11d4..6137e3cf 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -6,7 +6,7 @@ from cloudinit.config.schema import ( validate_cloudconfig_schema, main) from cloudinit.util import write_file -from ..helpers import CiTestCase, mock, skipIf +from cloudinit.tests.helpers import CiTestCase, mock, skipIf from copy import copy from six import StringIO diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py index f1979e89..2e4582a0 100644 --- a/tests/unittests/test_helpers.py +++ b/tests/unittests/test_helpers.py @@ -4,7 +4,7 @@ import os -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers from cloudinit import sources diff --git a/tests/unittests/test_log.py b/tests/unittests/test_log.py index 68fb4b8d..cd6296d6 100644 --- a/tests/unittests/test_log.py +++ b/tests/unittests/test_log.py @@ -2,9 +2,9 @@ """Tests for cloudinit.log """ -from .helpers import CiTestCase from cloudinit.analyze.dump import CLOUD_INIT_ASCTIME_FMT from cloudinit import log as ci_logging +from cloudinit.tests.helpers import CiTestCase import datetime import logging import six diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 0658b6b4..f51358da 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from . import helpers +from cloudinit.tests import helpers from cloudinit.handlers import cloud_config from cloudinit.handlers import (CONTENT_START, CONTENT_END) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index f251024b..c10ef905 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -11,10 +11,10 @@ from cloudinit.net import sysconfig from cloudinit.sources.helpers import openstack from cloudinit import util -from .helpers import CiTestCase -from .helpers import dir2dict -from .helpers import mock -from .helpers import populate_dir +from cloudinit.tests.helpers import CiTestCase +from cloudinit.tests.helpers import dir2dict +from cloudinit.tests.helpers import mock +from cloudinit.tests.helpers import populate_dir import base64 import copy diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index a4ae284f..abbb29b8 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -2,7 +2,7 @@ from cloudinit import util -from .helpers import TestCase, populate_dir +from cloudinit.tests.helpers import TestCase, populate_dir import shutil import tempfile diff --git a/tests/unittests/test_registry.py b/tests/unittests/test_registry.py index acf0bf4f..2b625026 100644 --- a/tests/unittests/test_registry.py +++ b/tests/unittests/test_registry.py @@ -2,7 +2,7 @@ from cloudinit.registry import DictRegistry -from .helpers import (mock, TestCase) +from cloudinit.tests.helpers import (mock, TestCase) class TestDictRegistry(TestCase): diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index f3b8f992..571420ed 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -8,7 +8,7 @@ from cloudinit.reporting import handlers import mock -from .helpers import TestCase +from cloudinit.tests.helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index ca14cd46..e9d5702a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -7,7 +7,7 @@ import logging from cloudinit.config import cc_rh_subscription from cloudinit import util -from .helpers import TestCase, mock +from cloudinit.tests.helpers import TestCase, mock class GoodTests(TestCase): diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 65895273..add93653 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 55f15b55..5cf666fe 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -4,7 +4,7 @@ import os import shutil import tempfile -from .. import helpers +from cloudinit.tests import helpers from cloudinit.settings import PER_INSTANCE from cloudinit import stages diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 991f45a6..2a8e6abe 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -2,8 +2,8 @@ from mock import patch -from . import helpers as test_helpers from cloudinit import ssh_util +from cloudinit.tests import helpers as test_helpers VALID_CONTENT = { @@ -57,6 +57,7 @@ TEST_OPTIONS = ( class TestAuthKeyLineParser(test_helpers.TestCase): + def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) parser = ssh_util.AuthKeyLineParser() diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 4e627826..b911d929 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -6,7 +6,7 @@ from __future__ import print_function -from . import helpers as test_helpers +from cloudinit.tests import helpers as test_helpers import textwrap from cloudinit import templater diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 5f11c88f..3e4154ca 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -12,7 +12,7 @@ import six import yaml from cloudinit import importer, util -from . import helpers +from cloudinit.tests import helpers try: from unittest import mock diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py index 1662ce09..d012f69d 100644 --- a/tests/unittests/test_version.py +++ b/tests/unittests/test_version.py @@ -1,6 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. -from .helpers import CiTestCase +from cloudinit.tests.helpers import CiTestCase from cloudinit import version diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 03b36d31..d8651077 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,10 +8,10 @@ import logging import sys -from .helpers import CiTestCase from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) logger = logging.getLogger(__name__) -- cgit v1.2.3 From ed8f1b159174715403cb1ffa200ff6d080770152 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Sat, 2 Sep 2017 01:51:29 -0600 Subject: schema and docs: Add jsonschema to resizefs and bootcmd modules Add schema definitions to both cc_resizefs and cc_bootcmd modules. Extend schema.py to parse and document enumerated json types. Schema definitions are used to generate module documention and log warnings for schema infractions. This branch also does the following: - drops vestigial 'resize_rootfs_tmp' option from cc_resizefs. That option only created the specified directory and didn't make use of that directory for any resize operations. - Drop yaml.dumps calls from schema documentation generation to avoid yaml import costs on module load - Add __doc__ = get_schema_doc(schema) definitions it each module to supplement python help() calls for cc_runcmd, cc_bootcmd, cc_ntp and cc_resizefs - Add a SCHEMA_EXAMPLES_SPACER_TEMPLATE string to docs for modules which contain more than one example --- cloudinit/config/cc_bootcmd.py | 87 ++++--- cloudinit/config/cc_ntp.py | 48 +--- cloudinit/config/cc_resizefs.py | 149 +++++++----- cloudinit/config/cc_runcmd.py | 5 +- cloudinit/config/schema.py | 19 +- .../unittests/test_handler/test_handler_bootcmd.py | 145 ++++++++++++ .../test_handler/test_handler_resizefs.py | 257 ++++++++++++++++++++- tests/unittests/test_handler/test_schema.py | 44 ++-- 8 files changed, 586 insertions(+), 168 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_bootcmd.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 9c0476af..233da1ef 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -3,45 +3,73 @@ # # Author: Scott Moser # Author: Juerg Haefliger +# Author: Chad Smith # # This file is part of cloud-init. See LICENSE file for license information. -""" -Bootcmd -------- -**Summary:** run commands early in boot process - -This module runs arbitrary commands very early in the boot process, -only slightly after a boothook would run. This is very similar to a -boothook, but more user friendly. The environment variable ``INSTANCE_ID`` -will be set to the current instance id for all run commands. Commands can be -specified either as lists or strings. For invocation details, see ``runcmd``. - -.. note:: - bootcmd should only be used for things that could not be done later in the - boot process. - -**Internal name:** ``cc_bootcmd`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - bootcmd: - - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts - - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] -""" +"""Bootcmd: run arbitrary commands early in the boot process.""" import os +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import temp_utils from cloudinit import util frequency = PER_ALWAYS +# The schema definition for each cloud-config module is a strict contract for +# describing supported configuration parameters for each cloud-config section. +# It allows cloud-config to validate and alert users to invalid or ignored +# configuration options before actually attempting to deploy with said +# configuration. + +distros = ['all'] + +schema = { + 'id': 'cc_bootcmd', + 'name': 'Bootcmd', + 'title': 'Run arbitrary commands early in the boot process', + 'description': dedent("""\ + This module runs arbitrary commands very early in the boot process, + only slightly after a boothook would run. This is very similar to a + boothook, but more user friendly. The environment variable + ``INSTANCE_ID`` will be set to the current instance id for all run + commands. Commands can be specified either as lists or strings. For + invocation details, see ``runcmd``. + + .. note:: + bootcmd should only be used for things that could not be done later + in the boot process."""), + 'distros': distros, + 'examples': [dedent("""\ + bootcmd: + - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts + - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] + """)], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'bootcmd': { + 'type': 'array', + 'items': { + 'oneOf': [ + {'type': 'array', 'items': {'type': 'string'}}, + {'type': 'string'}] + }, + 'additionalItems': False, # Reject items of non-string non-list + 'additionalProperties': False, + 'minItems': 1, + 'required': [], + 'uniqueItems': True + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): @@ -50,13 +78,14 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return + validate_cloudconfig_schema(cfg, schema) with temp_utils.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except Exception: - util.logexc(log, "Failed to shellify bootcmd") + except Exception as e: + util.logexc(log, "Failed to shellify bootcmd: %s", str(e)) raise try: diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index a02b4bf1..15ae1ecd 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -4,39 +4,10 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -NTP ---- -**Summary:** enable and configure ntp - -Handle ntp configuration. If ntp is not installed on the system and ntp -configuration is specified, ntp will be installed. If there is a default ntp -config file in the image or one is present in the distro's ntp package, it will -be copied to ``/etc/ntp.conf.dist`` before any changes are made. A list of ntp -pools and ntp servers can be provided under the ``ntp`` config key. If no ntp -servers or pools are provided, 4 pools will be used in the format -``{0-3}.{distro}.pool.ntp.org``. - -**Internal name:** ``cc_ntp`` - -**Module frequency:** per instance - -**Supported distros:** centos, debian, fedora, opensuse, ubuntu - -**Config keys**:: - - ntp: - pools: - - 0.company.pool.ntp.org - - 1.company.pool.ntp.org - - ntp.myorg.org - servers: - - my.ntp.server.local - - ntp.ubuntu.com - - 192.168.23.2 -""" +"""NTP: enable and configure ntp""" -from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit import log as logging from cloudinit.settings import PER_INSTANCE from cloudinit import templater @@ -76,10 +47,13 @@ schema = { ``{0-3}.{distro}.pool.ntp.org``."""), 'distros': distros, 'examples': [ - {'ntp': {'pools': ['0.company.pool.ntp.org', '1.company.pool.ntp.org', - 'ntp.myorg.org'], - 'servers': ['my.ntp.server.local', 'ntp.ubuntu.com', - '192.168.23.2']}}], + dedent("""\ + ntp: + pools: [0.int.pool.ntp.org, 1.int.pool.ntp.org, ntp.myorg.org] + servers: + - ntp.server.local + - ntp.ubuntu.com + - 192.168.23.2""")], 'frequency': PER_INSTANCE, 'type': 'object', 'properties': { @@ -117,6 +91,8 @@ schema = { } } +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): """Enable and configure ntp.""" diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index ceee952b..f14d3836 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -6,31 +6,8 @@ # # This file is part of cloud-init. See LICENSE file for license information. -""" -Resizefs --------- -**Summary:** resize filesystem +"""Resizefs: cloud-config module which resizes the filesystem""" -Resize a filesystem to use all avaliable space on partition. This module is -useful along with ``cc_growpart`` and will ensure that if the root partition -has been resized the root filesystem will be resized along with it. By default, -``cc_resizefs`` will resize the root partition and will block the boot process -while the resize command is running. Optionally, the resize operation can be -performed in the background while cloud-init continues running modules. This -can be enabled by setting ``resize_rootfs`` to ``true``. This module can be -disabled altogether by setting ``resize_rootfs`` to ``false``. - -**Internal name:** ``cc_resizefs`` - -**Module frequency:** per always - -**Supported distros:** all - -**Config keys**:: - - resize_rootfs: - resize_rootfs_tmp: -""" import errno import getopt @@ -38,11 +15,47 @@ import os import re import shlex import stat +from textwrap import dedent +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_ALWAYS from cloudinit import util +NOBLOCK = "noblock" + frequency = PER_ALWAYS +distros = ['all'] + +schema = { + 'id': 'cc_resizefs', + 'name': 'Resizefs', + 'title': 'Resize filesystem', + 'description': dedent("""\ + Resize a filesystem to use all avaliable space on partition. This + module is useful along with ``cc_growpart`` and will ensure that if the + root partition has been resized the root filesystem will be resized + along with it. By default, ``cc_resizefs`` will resize the root + partition and will block the boot process while the resize command is + running. Optionally, the resize operation can be performed in the + background while cloud-init continues running modules. This can be + enabled by setting ``resize_rootfs`` to ``true``. This module can be + disabled altogether by setting ``resize_rootfs`` to ``false``."""), + 'distros': distros, + 'examples': [ + 'resize_rootfs: false # disable root filesystem resize operation'], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'resize_rootfs': { + 'enum': [True, False, NOBLOCK], + 'description': dedent("""\ + Whether to resize the root partition. Default: 'true'""") + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() def _resize_btrfs(mount_point, devpth): @@ -131,8 +144,6 @@ RESIZE_FS_PRECHECK_CMDS = { 'ufs': _can_skip_resize_ufs } -NOBLOCK = "noblock" - def rootdev_from_cmdline(cmdline): found = None @@ -161,71 +172,81 @@ def can_skip_resize(fs_type, resize_what, devpth): return False -def handle(name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = args[0] - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if not util.translate_bool(resize_root, addons=[NOBLOCK]): - log.debug("Skipping module named %s, resizing disabled", name) - return - - # TODO(harlowja) is the directory ok to be used?? - resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") - util.ensure_dir(resize_root_d) +def is_device_path_writable_block(devpath, info, log): + """Return True if devpath is a writable block device. - # TODO(harlowja): allow what is to be resized to be configurable?? - resize_what = "/" - result = util.get_mount_info(resize_what, log) - if not result: - log.warn("Could not determine filesystem type of %s", resize_what) - return - - (devpth, fs_type, mount_point) = result - - info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) - log.debug("resize_info: %s" % info) + @param devpath: Path to the root device we want to resize. + @param info: String representing information about the requested device. + @param log: Logger to which logs will be added upon error. + @returns Boolean True if block device is writable + """ container = util.is_container() # Ensure the path is a block device. - if (devpth == "/dev/root" and not os.path.exists(devpth) and + if (devpath == "/dev/root" and not os.path.exists(devpath) and not container): - devpth = util.rootdev_from_cmdline(util.get_cmdline()) - if devpth is None: + devpath = util.rootdev_from_cmdline(util.get_cmdline()) + if devpath is None: log.warn("Unable to find device '/dev/root'") - return - log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth) + return False + log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) try: - statret = os.stat(devpth) + statret = os.stat(devpath) except OSError as exc: if container and exc.errno == errno.ENOENT: log.debug("Device '%s' did not exist in container. " - "cannot resize: %s", devpth, info) + "cannot resize: %s", devpath, info) elif exc.errno == errno.ENOENT: log.warn("Device '%s' did not exist. cannot resize: %s", - devpth, info) + devpath, info) else: raise exc - return + return False - if not os.access(devpth, os.W_OK): + if not os.access(devpath, os.W_OK): if container: log.debug("'%s' not writable in container. cannot resize: %s", - devpth, info) + devpath, info) else: - log.warn("'%s' not writable. cannot resize: %s", devpth, info) + log.warn("'%s' not writable. cannot resize: %s", devpath, info) return if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." - " cannot resize: %s" % (devpth, info)) + " cannot resize: %s" % (devpath, info)) else: log.warn("device '%s' not a block device. cannot resize: %s" % - (devpth, info)) + (devpath, info)) + return False + return True + + +def handle(name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = args[0] + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + validate_cloudconfig_schema(cfg, schema) + if not util.translate_bool(resize_root, addons=[NOBLOCK]): + log.debug("Skipping module named %s, resizing disabled", name) + return + + # TODO(harlowja): allow what is to be resized to be configurable?? + resize_what = "/" + result = util.get_mount_info(resize_what, log) + if not result: + log.warn("Could not determine filesystem type of %s", resize_what) + return + + (devpth, fs_type, mount_point) = result + + info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) + log.debug("resize_info: %s" % info) + + if not is_device_path_writable_block(devpth, info, log): return resizer = None diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 7c3ccd41..7f995693 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -8,7 +8,8 @@ """Runcmd: run arbitrary commands at rc.local with output to the console""" -from cloudinit.config.schema import validate_cloudconfig_schema +from cloudinit.config.schema import ( + get_schema_doc, validate_cloudconfig_schema) from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -67,6 +68,8 @@ schema = { } } +__doc__ = get_schema_doc(schema) # Supplement python help() + def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 73dd5c2e..c17d973e 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -14,6 +14,7 @@ import re import sys import yaml +_YAML_MAP = {True: 'true', False: 'false', None: 'null'} SCHEMA_UNDEFINED = b'UNDEFINED' CLOUD_CONFIG_HEADER = b'#cloud-config' SCHEMA_DOC_TMPL = """ @@ -34,6 +35,8 @@ SCHEMA_DOC_TMPL = """ {examples} """ SCHEMA_PROPERTY_TMPL = '{prefix}**{prop_name}:** ({type}) {description}' +SCHEMA_EXAMPLES_HEADER = '\n**Examples**::\n\n' +SCHEMA_EXAMPLES_SPACER_TEMPLATE = '\n # --- Example{0} ---' class SchemaValidationError(ValueError): @@ -212,6 +215,9 @@ def _schemapath_for_cloudconfig(config, original_content): def _get_property_type(property_dict): """Return a string representing a property type from a given jsonschema.""" property_type = property_dict.get('type', SCHEMA_UNDEFINED) + if property_type == SCHEMA_UNDEFINED and property_dict.get('enum'): + property_type = [ + str(_YAML_MAP.get(k, k)) for k in property_dict['enum']] if isinstance(property_type, list): property_type = '/'.join(property_type) items = property_dict.get('items', {}) @@ -249,15 +255,14 @@ def _get_schema_examples(schema, prefix=''): examples = schema.get('examples') if not examples: return '' - rst_content = '\n**Examples**::\n\n' - for example in examples: - if isinstance(example, str): - example_content = example - else: - example_content = yaml.dump(example, default_flow_style=False) + rst_content = SCHEMA_EXAMPLES_HEADER + for count, example in enumerate(examples): # Python2.6 is missing textwrapper.indent - lines = example_content.split('\n') + lines = example.split('\n') indented_lines = [' {0}'.format(line) for line in lines] + if rst_content != SCHEMA_EXAMPLES_HEADER: + indented_lines.insert( + 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1)) rst_content += '\n'.join(indented_lines) return rst_content diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py new file mode 100644 index 00000000..580017ed --- /dev/null +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -0,0 +1,145 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_bootcmd +from cloudinit.sources import DataSourceNone +from cloudinit import (distros, helpers, cloud, util) +from cloudinit.tests.helpers import CiTestCase, mock, skipIf + +import logging +import tempfile + +try: + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False +except ImportError: + _missing_jsonschema_dep = True + +LOG = logging.getLogger(__name__) + + +class FakeExtendedTempFile(object): + def __init__(self, suffix): + self.suffix = suffix + self.handle = tempfile.NamedTemporaryFile( + prefix="ci-%s." % self.__class__.__name__, delete=False) + + def __enter__(self): + return self.handle + + def __exit__(self, exc_type, exc_value, traceback): + self.handle.close() + + +class TestBootcmd(CiTestCase): + + with_logs = True + + _etmpfile_path = ('cloudinit.config.cc_bootcmd.temp_utils.' + 'ExtendedTemporaryFile') + + def setUp(self): + super(TestBootcmd, self).setUp() + self.subp = util.subp + self.new_root = self.tmp_dir() + + def _get_cloud(self, distro): + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + paths.datasource = myds + return cloud.Cloud(myds, paths, {}, mydist, None) + + def test_handler_skip_if_no_bootcmd(self): + """When the provided config doesn't contain bootcmd, skip it.""" + cfg = {} + mycloud = self._get_cloud('ubuntu') + cc_bootcmd.handle('notimportant', cfg, mycloud, LOG, None) + self.assertIn( + "Skipping module named notimportant, no 'bootcmd' key", + self.logs.getvalue()) + + def test_handler_invalid_command_set(self): + """Commands which can't be converted to shell will raise errors.""" + invalid_config = {'bootcmd': 1} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError) as context_manager: + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + self.assertIn('Failed to shellify bootcmd', self.logs.getvalue()) + self.assertEqual( + "'int' object is not iterable", + str(context_manager.exception)) + + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + def test_handler_schema_validation_warns_non_array_type(self): + """Schema validation warns of non-array type for bootcmd key. + + Schema validation is not strict, so bootcmd attempts to shellify the + invalid content. + """ + invalid_config = {'bootcmd': 1} + cc = self._get_cloud('ubuntu') + with self.assertRaises(TypeError): + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + self.assertIn( + 'Invalid config:\nbootcmd: 1 is not of type \'array\'', + self.logs.getvalue()) + self.assertIn('Failed to shellify', self.logs.getvalue()) + + @skipIf(_missing_jsonschema_dep, 'No python-jsonschema dependency') + def test_handler_schema_validation_warns_non_array_item_type(self): + """Schema validation warns of non-array or string bootcmd items. + + Schema validation is not strict, so bootcmd attempts to shellify the + invalid content. + """ + invalid_config = { + 'bootcmd': ['ls /', 20, ['wget', 'http://stuff/blah'], {'a': 'n'}]} + cc = self._get_cloud('ubuntu') + with self.assertRaises(RuntimeError) as context_manager: + cc_bootcmd.handle('cc_bootcmd', invalid_config, cc, LOG, []) + expected_warnings = [ + 'bootcmd.1: 20 is not valid under any of the given schemas', + 'bootcmd.3: {\'a\': \'n\'} is not valid under any of the given' + ' schema' + ] + logs = self.logs.getvalue() + for warning in expected_warnings: + self.assertIn(warning, logs) + self.assertIn('Failed to shellify', logs) + self.assertEqual( + 'Unable to shellify type int which is not a list or string', + str(context_manager.exception)) + + def test_handler_creates_and_runs_bootcmd_script_with_instance_id(self): + """Valid schema runs a bootcmd script with INSTANCE_ID in the env.""" + cc = self._get_cloud('ubuntu') + out_file = self.tmp_path('bootcmd.out', self.new_root) + my_id = "b6ea0f59-e27d-49c6-9f87-79f19765a425" + valid_config = {'bootcmd': [ + 'echo {0} $INSTANCE_ID > {1}'.format(my_id, out_file)]} + + with mock.patch(self._etmpfile_path, FakeExtendedTempFile): + cc_bootcmd.handle('cc_bootcmd', valid_config, cc, LOG, []) + self.assertEqual(my_id + ' iid-datasource-none\n', + util.load_file(out_file)) + + def test_handler_runs_bootcmd_script_with_error(self): + """When a valid script generates an error, that error is raised.""" + cc = self._get_cloud('ubuntu') + valid_config = {'bootcmd': ['exit 1']} # Script with error + + with mock.patch(self._etmpfile_path, FakeExtendedTempFile): + with self.assertRaises(util.ProcessExecutionError) as ctxt_manager: + cc_bootcmd.handle('does-not-matter', valid_config, cc, LOG, []) + self.assertIn( + 'Unexpected error while running command.\n' + "Command: ['/bin/sh',", + str(ctxt_manager.exception)) + self.assertIn( + 'Failed to run bootcmd module does-not-matter', + self.logs.getvalue()) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 52591b8b..76dddbf8 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -1,17 +1,30 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config import cc_resizefs +from cloudinit.config.cc_resizefs import ( + can_skip_resize, handle, is_device_path_writable_block, + rootdev_from_cmdline) +import logging import textwrap -import unittest + +from cloudinit.tests.helpers import (CiTestCase, mock, skipIf, util, + wrap_and_call) + + +LOG = logging.getLogger(__name__) + try: - from unittest import mock + import jsonschema + assert jsonschema # avoid pyflakes error F401: import unused + _missing_jsonschema_dep = False except ImportError: - import mock + _missing_jsonschema_dep = True + +class TestResizefs(CiTestCase): + with_logs = True -class TestResizefs(unittest.TestCase): def setUp(self): super(TestResizefs, self).setUp() self.name = "resizefs" @@ -34,7 +47,7 @@ class TestResizefs(unittest.TestCase): 58720296 3145728 3 freebsd-swap (1.5G) 61866024 1048496 - free - (512M) """) - res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth) + res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') @@ -52,8 +65,238 @@ class TestResizefs(unittest.TestCase): => 34 297086 da0 GPT (145M) 34 297086 1 freebsd-ufs (145M) """) - res = cc_resizefs.can_skip_resize(fs_type, resize_what, devpth) + res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) + def test_handle_noops_on_disabled(self): + """The handle function logs when the configuration disables resize.""" + cfg = {'resize_rootfs': False} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + self.assertIn( + 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', + self.logs.getvalue()) + + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") + def test_handle_schema_validation_logs_invalid_resize_rootfs_value(self): + """The handle reports json schema violations as a warning. + + Invalid values for resize_rootfs result in disabling the module. + """ + cfg = {'resize_rootfs': 'junk'} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + logs = self.logs.getvalue() + self.assertIn( + "WARNING: Invalid config:\nresize_rootfs: 'junk' is not one of" + " [True, False, 'noblock']", + logs) + self.assertIn( + 'DEBUG: Skipping module named cc_resizefs, resizing disabled\n', + logs) + + @mock.patch('cloudinit.config.cc_resizefs.util.get_mount_info') + def test_handle_warns_on_unknown_mount_info(self, m_get_mount_info): + """handle warns when get_mount_info sees unknown filesystem for /.""" + m_get_mount_info.return_value = None + cfg = {'resize_rootfs': True} + handle('cc_resizefs', cfg, _cloud=None, log=LOG, args=[]) + logs = self.logs.getvalue() + self.assertNotIn("WARNING: Invalid config:\nresize_rootfs:", logs) + self.assertIn( + 'WARNING: Could not determine filesystem type of /\n', + logs) + self.assertEqual( + [mock.call('/', LOG)], + m_get_mount_info.call_args_list) + + def test_handle_warns_on_undiscoverable_root_path_in_commandline(self): + """handle noops when the root path is not found on the commandline.""" + cfg = {'resize_rootfs': True} + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + + def fake_mount_info(path, log): + self.assertEqual('/', path) + self.assertEqual(LOG, log) + return ('/dev/root', 'ext4', '/') + + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'get_mount_info': {'side_effect': fake_mount_info}, + 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, + handle, 'cc_resizefs', cfg, _cloud=None, log=LOG, + args=[]) + logs = self.logs.getvalue() + self.assertIn("WARNING: Unable to find device '/dev/root'", logs) + + +class TestRootDevFromCmdline(CiTestCase): + + def test_rootdev_from_cmdline_with_no_root(self): + """Return None from rootdev_from_cmdline when root is not present.""" + invalid_cases = [ + 'BOOT_IMAGE=/adsf asdfa werasef root adf', 'BOOT_IMAGE=/adsf', ''] + for case in invalid_cases: + self.assertIsNone(rootdev_from_cmdline(case)) + + def test_rootdev_from_cmdline_with_root_startswith_dev(self): + """Return the cmdline root when the path starts with /dev.""" + self.assertEqual( + '/dev/this', rootdev_from_cmdline('asdf root=/dev/this')) + + def test_rootdev_from_cmdline_with_root_without_dev_prefix(self): + """Add /dev prefix to cmdline root when the path lacks the prefix.""" + self.assertEqual('/dev/this', rootdev_from_cmdline('asdf root=this')) + + def test_rootdev_from_cmdline_with_root_with_label(self): + """When cmdline root contains a LABEL, our root is disk/by-label.""" + self.assertEqual( + '/dev/disk/by-label/unique', + rootdev_from_cmdline('asdf root=LABEL=unique')) + + def test_rootdev_from_cmdline_with_root_with_uuid(self): + """When cmdline root contains a UUID, our root is disk/by-uuid.""" + self.assertEqual( + '/dev/disk/by-uuid/adsfdsaf-adsf', + rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf')) + + +class TestIsDevicePathWritableBlock(CiTestCase): + + with_logs = True + + def test_is_device_path_writable_block_warns_missing_cmdline_root(self): + """When root does not exist isn't in the cmdline, log warning.""" + info = 'does not matter' + + def fake_mount_info(path, log): + self.assertEqual('/', path) + self.assertEqual(LOG, log) + return ('/dev/root', 'ext4', '/') + + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'get_mount_info': {'side_effect': fake_mount_info}, + 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, + is_device_path_writable_block, '/dev/root', info, LOG) + self.assertFalse(is_valid) + logs = self.logs.getvalue() + self.assertIn("WARNING: Unable to find device '/dev/root'", logs) + + def test_is_device_path_writable_block_does_not_exist(self): + """When devpath does not exist, a warning is logged.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "WARNING: Device '/I/dont/exist' did not exist." + ' cannot resize: %s' % info, + self.logs.getvalue()) + + def test_is_device_path_writable_block_does_not_exist_in_container(self): + """When devpath does not exist in a container, log a debug message.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: Device '/I/dont/exist' did not exist in container." + ' cannot resize: %s' % info, + self.logs.getvalue()) + + def test_is_device_path_writable_block_raises_oserror(self): + """When unexpected OSError is raises by os.stat it is reraised.""" + info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' + with self.assertRaises(OSError) as context_manager: + wrap_and_call( + 'cloudinit.config.cc_resizefs', + {'util.is_container': {'return_value': True}, + 'os.stat': {'side_effect': OSError('Something unexpected')}}, + is_device_path_writable_block, '/I/dont/exist', info, LOG) + self.assertEqual( + 'Something unexpected', str(context_manager.exception)) + + def test_is_device_path_writable_block_readonly(self): + """When root device is readonly, emit a warning and return False.""" + fake_devpath = self.tmp_path('dev/readonly') + util.write_file(fake_devpath, '', mode=0o400) # read-only + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' + with mock.patch(exists_mock_path) as m_exists: + m_exists.return_value = False + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}, + 'rootdev_from_cmdline': {'return_value': fake_devpath}}, + is_device_path_writable_block, '/dev/root', info, LOG) + self.assertFalse(is_valid) + logs = self.logs.getvalue() + self.assertIn( + "Converted /dev/root to '{0}' per kernel cmdline".format( + fake_devpath), + logs) + self.assertIn( + "WARNING: '{0}' not writable. cannot resize".format(fake_devpath), + logs) + + def test_is_device_path_writable_block_readonly_in_container(self): + """When root device is readonly, emit debug log and return False.""" + fake_devpath = self.tmp_path('dev/readonly') + util.write_file(fake_devpath, '', mode=0o400) # read-only + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: '{0}' not writable in container. cannot resize".format( + fake_devpath), + self.logs.getvalue()) + + def test_is_device_path_writable_block_non_block(self): + """When device is not a block device, emit warning return False.""" + fake_devpath = self.tmp_path('dev/readwrite') + util.write_file(fake_devpath, '', mode=0o600) # read-write + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "WARNING: device '{0}' not a block device. cannot resize".format( + fake_devpath), + self.logs.getvalue()) + + def test_is_device_path_writable_block_non_block_on_container(self): + """When device is non-block device in container, emit debug log.""" + fake_devpath = self.tmp_path('dev/readwrite') + util.write_file(fake_devpath, '', mode=0o600) # read-write + info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) + + is_valid = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': True}}, + is_device_path_writable_block, fake_devpath, info, LOG) + self.assertFalse(is_valid) + self.assertIn( + "DEBUG: device '{0}' not a block device in container." + ' cannot resize'.format(fake_devpath), + self.logs.getvalue()) + # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 6137e3cf..745bb0ff 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,7 +27,7 @@ class GetSchemaTest(CiTestCase): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() self.assertItemsEqual( - ['cc_ntp', 'cc_runcmd'], + ['cc_bootcmd', 'cc_ntp', 'cc_resizefs', 'cc_runcmd'], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) self.assertEqual( @@ -205,6 +205,17 @@ class GetSchemaDocTest(CiTestCase): '**prop1:** (string/integer) prop-description', get_schema_doc(full_schema)) + def test_get_schema_doc_handles_enum_types(self): + """get_schema_doc converts enum types to yaml and delimits with '/'.""" + full_schema = copy(self.required_schema) + full_schema.update( + {'properties': { + 'prop1': {'enum': [True, False, 'stuff'], + 'description': 'prop-description'}}}) + self.assertIn( + '**prop1:** (true/false/stuff) prop-description', + get_schema_doc(full_schema)) + def test_get_schema_doc_handles_nested_oneof_property_types(self): """get_schema_doc describes array items oneOf declarations in type.""" full_schema = copy(self.required_schema) @@ -219,29 +230,11 @@ class GetSchemaDocTest(CiTestCase): '**prop1:** (array of (string)/(integer)) prop-description', get_schema_doc(full_schema)) - def test_get_schema_doc_returns_restructured_text_with_examples(self): - """get_schema_doc returns indented examples when present in schema.""" - full_schema = copy(self.required_schema) - full_schema.update( - {'examples': [{'ex1': [1, 2, 3]}], - 'properties': { - 'prop1': {'type': 'array', 'description': 'prop-description', - 'items': {'type': 'integer'}}}}) - self.assertIn( - dedent(""" - **Config schema**: - **prop1:** (array of integer) prop-description - - **Examples**:: - - ex1"""), - get_schema_doc(full_schema)) - - def test_get_schema_doc_handles_unstructured_examples(self): - """get_schema_doc properly indented examples which as just strings.""" + def test_get_schema_doc_handles_string_examples(self): + """get_schema_doc properly indented examples as a list of strings.""" full_schema = copy(self.required_schema) full_schema.update( - {'examples': ['My example:\n [don\'t, expand, "this"]'], + {'examples': ['ex1:\n [don\'t, expand, "this"]', 'ex2: true'], 'properties': { 'prop1': {'type': 'array', 'description': 'prop-description', 'items': {'type': 'integer'}}}}) @@ -252,8 +245,11 @@ class GetSchemaDocTest(CiTestCase): **Examples**:: - My example: - [don't, expand, "this"]"""), + ex1: + [don't, expand, "this"] + # --- Example2 --- + ex2: true + """), get_schema_doc(full_schema)) def test_get_schema_doc_raises_key_errors(self): -- cgit v1.2.3 From cf10a2ff2e2f666d9370f38297a5a105e809ea3c Mon Sep 17 00:00:00 2001 From: Ethan Apodaca Date: Wed, 13 Sep 2017 22:18:26 -0600 Subject: chef: Add option to pin chef omnibus install version Most users of chef will want to pin the version that is installed. Typically new versions of chef have to be evaluated for breakage etc. This change proposes a new optional `omnibus_version` field to the chef configuration. The changeset also adds documentation referencing the new field. LP: #1462693 --- cloudinit/config/cc_chef.py | 45 ++++++++---- cloudinit/util.py | 25 +++++++ doc/examples/cloud-config-chef.txt | 4 ++ tests/unittests/test_handler/test_handler_chef.py | 88 +++++++++++++++++++---- 4 files changed, 138 insertions(+), 24 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index c192dd32..46abedd1 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,6 +58,9 @@ file). log_level: log_location: node_name: + omnibus_url: + omnibus_url_retries: + omnibus_version: pid_file: server_url: show_time: @@ -71,7 +74,6 @@ import itertools import json import os -from cloudinit import temp_utils from cloudinit import templater from cloudinit import url_helper from cloudinit import util @@ -280,6 +282,31 @@ def run_chef(chef_cfg, log): util.subp(cmd, capture=False) +def install_chef_from_omnibus(url=None, retries=None, omnibus_version=None): + """Install an omnibus unified package from url. + + @param url: URL where blob of chef content may be downloaded. Defaults to + OMNIBUS_URL. + @param retries: Number of retries to perform when attempting to read url. + Defaults to OMNIBUS_URL_RETRIES + @param omnibus_version: Optional version string to require for omnibus + install. + """ + if url is None: + url = OMNIBUS_URL + if retries is None: + retries = OMNIBUS_URL_RETRIES + + if omnibus_version is None: + args = [] + else: + args = ['-v', omnibus_version] + content = url_helper.readurl(url=url, retries=retries).contents + return util.subp_blob_in_tempfile( + blob=content, args=args, + basename='chef-omnibus-install', capture=False) + + def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', @@ -298,17 +325,11 @@ def install_chef(cloud, chef_cfg, log): # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) elif install_type == 'omnibus': - # This will install as a omnibus unified package - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - retries = max(0, util.get_cfg_option_int(chef_cfg, - "omnibus_url_retries", - default=OMNIBUS_URL_RETRIES)) - content = url_helper.readurl(url=url, retries=retries).contents - with temp_utils.tempdir() as tmpd: - # Use tmpdir over tmpfile to avoid 'text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, content, mode=0o700) - util.subp([tmpf], capture=False) + omnibus_version = util.get_cfg_option_str(chef_cfg, "omnibus_version") + install_chef_from_omnibus( + url=util.get_cfg_option_str(chef_cfg, "omnibus_url"), + retries=util.get_cfg_option_int(chef_cfg, "omnibus_url_retries"), + omnibus_version=omnibus_version) else: log.warn("Unknown chef install type '%s'", install_type) run = False diff --git a/cloudinit/util.py b/cloudinit/util.py index ae5cda8d..7e9d94fc 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1742,6 +1742,31 @@ def delete_dir_contents(dirname): del_file(node_fullpath) +def subp_blob_in_tempfile(blob, *args, **kwargs): + """Write blob to a tempfile, and call subp with args, kwargs. Then cleanup. + + 'basename' as a kwarg allows providing the basename for the file. + The 'args' argument to subp will be updated with the full path to the + filename as the first argument. + """ + basename = kwargs.pop('basename', "subp_blob") + + if len(args) == 0 and 'args' not in kwargs: + args = [tuple()] + + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + with temp_utils.tempdir() as tmpd: + tmpf = os.path.join(tmpd, basename) + if 'args' in kwargs: + kwargs['args'] = [tmpf] + list(kwargs['args']) + else: + args = list(args) + args[0] = [tmpf] + args[0] + + write_file(tmpf, blob, mode=0o700) + return subp(*args, **kwargs) + + def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, logstring=False, decode="replace", target=None, update_env=None): diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index 9d235817..58d5fdc7 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -94,6 +94,10 @@ chef: # if install_type is 'omnibus', change the url to download omnibus_url: "https://www.chef.io/chef/install.sh" + # if install_type is 'omnibus', pass pinned version string + # to the install script + omnibus_version: "12.3.0" + # Capture all subprocess output into a logfile # Useful for troubleshooting cloud-init issues diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index e5785cfd..0136a93d 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,11 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. +import httpretty import json import logging import os -import shutil import six -import tempfile from cloudinit import cloud from cloudinit.config import cc_chef @@ -14,18 +13,83 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNone from cloudinit import util -from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import ( + CiTestCase, FilesystemMockingTestCase, mock, skipIf) LOG = logging.getLogger(__name__) CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) -class TestChef(t_help.FilesystemMockingTestCase): +class TestInstallChefOmnibus(CiTestCase): + + def setUp(self): + self.new_root = self.tmp_dir() + + @httpretty.activate + def test_install_chef_from_omnibus_runs_chef_url_content(self): + """install_chef_from_omnibus runs downloaded OMNIBUS_URL as script.""" + chef_outfile = self.tmp_path('chef.out', self.new_root) + response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + httpretty.register_uri( + httpretty.GET, cc_chef.OMNIBUS_URL, body=response, status=200) + cc_chef.install_chef_from_omnibus() + self.assertEqual('Hi Mom\n', util.load_file(chef_outfile)) + + @mock.patch('cloudinit.config.cc_chef.url_helper.readurl') + @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + def test_install_chef_from_omnibus_retries_url(self, m_subp_blob, m_rdurl): + """install_chef_from_omnibus retries OMNIBUS_URL upon failure.""" + + class FakeURLResponse(object): + contents = '#!/bin/bash\necho "Hi Mom" > {0}/chef.out'.format( + self.new_root) + + m_rdurl.return_value = FakeURLResponse() + + cc_chef.install_chef_from_omnibus() + expected_kwargs = {'retries': cc_chef.OMNIBUS_URL_RETRIES, + 'url': cc_chef.OMNIBUS_URL} + self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[0][1]) + cc_chef.install_chef_from_omnibus(retries=10) + expected_kwargs = {'retries': 10, + 'url': cc_chef.OMNIBUS_URL} + self.assertItemsEqual(expected_kwargs, m_rdurl.call_args_list[1][1]) + expected_subp_kwargs = { + 'args': ['-v', '2.0'], + 'basename': 'chef-omnibus-install', + 'blob': m_rdurl.return_value.contents, + 'capture': False + } + self.assertItemsEqual( + expected_subp_kwargs, + m_subp_blob.call_args_list[0][1]) + + @httpretty.activate + @mock.patch('cloudinit.config.cc_chef.util.subp_blob_in_tempfile') + def test_install_chef_from_omnibus_has_omnibus_version(self, m_subp_blob): + """install_chef_from_omnibus provides version arg to OMNIBUS_URL.""" + chef_outfile = self.tmp_path('chef.out', self.new_root) + response = '#!/bin/bash\necho "Hi Mom" > {0}'.format(chef_outfile) + httpretty.register_uri( + httpretty.GET, cc_chef.OMNIBUS_URL, body=response) + cc_chef.install_chef_from_omnibus(omnibus_version='2.0') + + called_kwargs = m_subp_blob.call_args_list[0][1] + expected_kwargs = { + 'args': ['-v', '2.0'], + 'basename': 'chef-omnibus-install', + 'blob': response, + 'capture': False + } + self.assertItemsEqual(expected_kwargs, called_kwargs) + + +class TestChef(FilesystemMockingTestCase): + def setUp(self): super(TestChef, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) + self.tmp = self.tmp_dir() def fetch_cloud(self, distro_kind): cls = distros.fetch(distro_kind) @@ -43,8 +107,8 @@ class TestChef(t_help.FilesystemMockingTestCase): for d in cc_chef.CHEF_DIRS: self.assertFalse(os.path.isdir(d)) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_basic_config(self): """ test basic config looks sane @@ -122,8 +186,8 @@ class TestChef(t_help.FilesystemMockingTestCase): 'c': 'd', }, json.loads(c)) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_template_deletes(self): tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) @@ -143,8 +207,8 @@ class TestChef(t_help.FilesystemMockingTestCase): self.assertNotIn('json_attribs', c) self.assertNotIn('Formatter.show_time', c) - @t_help.skipIf(not os.path.isfile(CLIENT_TEMPL), - CLIENT_TEMPL + " is not available") + @skipIf(not os.path.isfile(CLIENT_TEMPL), + CLIENT_TEMPL + " is not available") def test_validation_cert_and_validation_key(self): # test validation_cert content is written to validation_key path tpl_file = util.load_file('templates/chef_client.rb.tmpl') -- cgit v1.2.3 From 024ecc1f758d36cc0aa5ebce65704eed6bd66d45 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 14 Sep 2017 14:05:13 -0600 Subject: resizefs: Drop check for read-only device file, do not warn on overlayroot. As root user, os.access(, os.W_OK) will always return True so that path will never get executed. Also avoid a warning if the root is overlayroot, which is the common case on a MAAS booted 'ephemeral' system. --- cloudinit/config/cc_resizefs.py | 12 ++-- .../test_handler/test_handler_resizefs.py | 72 +++++++--------------- 2 files changed, 26 insertions(+), 58 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index f42b6a63..f774baa3 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -192,6 +192,10 @@ def is_device_path_writable_block(devpath, info, log): return False log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) + if devpath == 'overlayroot': + log.debug("Not attempting to resize devpath '%s': %s", devpath, info) + return False + try: statret = os.stat(devpath) except OSError as exc: @@ -205,14 +209,6 @@ def is_device_path_writable_block(devpath, info, log): raise exc return False - if not os.access(devpath, os.W_OK): - if container: - log.debug("'%s' not writable in container. cannot resize: %s", - devpath, info) - else: - log.warn("'%s' not writable. cannot resize: %s", devpath, info) - return - if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): if container: log.debug("device '%s' not a block device in container." diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index 76dddbf8..3e5d436c 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -166,6 +166,18 @@ class TestIsDevicePathWritableBlock(CiTestCase): with_logs = True + def test_is_device_path_writable_block_false_on_overlayroot(self): + """When devpath is overlayroot (on MAAS), is_dev_writable is False.""" + info = 'does not matter' + is_writable = wrap_and_call( + 'cloudinit.config.cc_resizefs.util', + {'is_container': {'return_value': False}}, + is_device_path_writable_block, 'overlayroot', info, LOG) + self.assertFalse(is_writable) + self.assertIn( + "Not attempting to resize devpath 'overlayroot'", + self.logs.getvalue()) + def test_is_device_path_writable_block_warns_missing_cmdline_root(self): """When root does not exist isn't in the cmdline, log warning.""" info = 'does not matter' @@ -178,24 +190,24 @@ class TestIsDevicePathWritableBlock(CiTestCase): exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' with mock.patch(exists_mock_path) as m_exists: m_exists.return_value = False - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}, 'get_mount_info': {'side_effect': fake_mount_info}, 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, is_device_path_writable_block, '/dev/root', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) logs = self.logs.getvalue() self.assertIn("WARNING: Unable to find device '/dev/root'", logs) def test_is_device_path_writable_block_does_not_exist(self): """When devpath does not exist, a warning is logged.""" info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}}, is_device_path_writable_block, '/I/dont/exist', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "WARNING: Device '/I/dont/exist' did not exist." ' cannot resize: %s' % info, @@ -204,11 +216,11 @@ class TestIsDevicePathWritableBlock(CiTestCase): def test_is_device_path_writable_block_does_not_exist_in_container(self): """When devpath does not exist in a container, log a debug message.""" info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': True}}, is_device_path_writable_block, '/I/dont/exist', info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "DEBUG: Device '/I/dont/exist' did not exist in container." ' cannot resize: %s' % info, @@ -226,57 +238,17 @@ class TestIsDevicePathWritableBlock(CiTestCase): self.assertEqual( 'Something unexpected', str(context_manager.exception)) - def test_is_device_path_writable_block_readonly(self): - """When root device is readonly, emit a warning and return False.""" - fake_devpath = self.tmp_path('dev/readonly') - util.write_file(fake_devpath, '', mode=0o400) # read-only - info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - - exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' - with mock.patch(exists_mock_path) as m_exists: - m_exists.return_value = False - is_valid = wrap_and_call( - 'cloudinit.config.cc_resizefs.util', - {'is_container': {'return_value': False}, - 'rootdev_from_cmdline': {'return_value': fake_devpath}}, - is_device_path_writable_block, '/dev/root', info, LOG) - self.assertFalse(is_valid) - logs = self.logs.getvalue() - self.assertIn( - "Converted /dev/root to '{0}' per kernel cmdline".format( - fake_devpath), - logs) - self.assertIn( - "WARNING: '{0}' not writable. cannot resize".format(fake_devpath), - logs) - - def test_is_device_path_writable_block_readonly_in_container(self): - """When root device is readonly, emit debug log and return False.""" - fake_devpath = self.tmp_path('dev/readonly') - util.write_file(fake_devpath, '', mode=0o400) # read-only - info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - - is_valid = wrap_and_call( - 'cloudinit.config.cc_resizefs.util', - {'is_container': {'return_value': True}}, - is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) - self.assertIn( - "DEBUG: '{0}' not writable in container. cannot resize".format( - fake_devpath), - self.logs.getvalue()) - def test_is_device_path_writable_block_non_block(self): """When device is not a block device, emit warning return False.""" fake_devpath = self.tmp_path('dev/readwrite') util.write_file(fake_devpath, '', mode=0o600) # read-write info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': False}}, is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "WARNING: device '{0}' not a block device. cannot resize".format( fake_devpath), @@ -288,11 +260,11 @@ class TestIsDevicePathWritableBlock(CiTestCase): util.write_file(fake_devpath, '', mode=0o600) # read-write info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) - is_valid = wrap_and_call( + is_writable = wrap_and_call( 'cloudinit.config.cc_resizefs.util', {'is_container': {'return_value': True}}, is_device_path_writable_block, fake_devpath, info, LOG) - self.assertFalse(is_valid) + self.assertFalse(is_writable) self.assertIn( "DEBUG: device '{0}' not a block device in container." ' cannot resize'.format(fake_devpath), -- cgit v1.2.3 From 79ce0a234584a50b1c6e2b664b9ccf7a5d1fca58 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 21 Sep 2017 16:59:59 -0400 Subject: tests: remove a temp file used in bootcmd tests. The bootcmd test was leaving files in the tmpdir named ci-FakeExtendedTempFile.XXXXXX. This cleans those up. --- tests/unittests/test_handler/test_handler_bootcmd.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tests/unittests/test_handler') diff --git a/tests/unittests/test_handler/test_handler_bootcmd.py b/tests/unittests/test_handler/test_handler_bootcmd.py index 580017ed..dbf43e0d 100644 --- a/tests/unittests/test_handler/test_handler_bootcmd.py +++ b/tests/unittests/test_handler/test_handler_bootcmd.py @@ -29,6 +29,7 @@ class FakeExtendedTempFile(object): def __exit__(self, exc_type, exc_value, traceback): self.handle.close() + util.del_file(self.handle.name) class TestBootcmd(CiTestCase): -- cgit v1.2.3 From cc1475d07b9d0727012634ee9c7a914d67b051f5 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Thu, 21 Sep 2017 11:58:28 -0400 Subject: suse: Support addition of zypper repos via cloud-config. This adds a config module so support for adding zypper repositories via cloud-config. LP: #1718675 --- cloudinit/config/cc_zypper_add_repo.py | 218 +++++++++++++++++++ config/cloud.cfg.tmpl | 3 + .../test_handler/test_handler_zypper_add_repo.py | 237 +++++++++++++++++++++ tests/unittests/test_handler/test_schema.py | 8 +- 4 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 cloudinit/config/cc_zypper_add_repo.py create mode 100644 tests/unittests/test_handler/test_handler_zypper_add_repo.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_zypper_add_repo.py b/cloudinit/config/cc_zypper_add_repo.py new file mode 100644 index 00000000..aba26952 --- /dev/null +++ b/cloudinit/config/cc_zypper_add_repo.py @@ -0,0 +1,218 @@ +# +# Copyright (C) 2017 SUSE LLC. +# +# This file is part of cloud-init. See LICENSE file for license information. + +"""zypper_add_repo: Add zyper repositories to the system""" + +import configobj +import os +from six import string_types +from textwrap import dedent + +from cloudinit.config.schema import get_schema_doc +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util + +distros = ['opensuse', 'sles'] + +schema = { + 'id': 'cc_zypper_add_repo', + 'name': 'ZypperAddRepo', + 'title': 'Configure zypper behavior and add zypper repositories', + 'description': dedent("""\ + Configure zypper behavior by modifying /etc/zypp/zypp.conf. The + configuration writer is "dumb" and will simply append the provided + configuration options to the configuration file. Option settings + that may be duplicate will be resolved by the way the zypp.conf file + is parsed. The file is in INI format. + Add repositories to the system. No validation is performed on the + repository file entries, it is assumed the user is familiar with + the zypper repository file format."""), + 'distros': distros, + 'examples': [dedent("""\ + zypper: + repos: + - id: opensuse-oss + name: os-oss + baseurl: http://dl.opensuse.org/dist/leap/v/repo/oss/ + enabled: 1 + autorefresh: 1 + - id: opensuse-oss-update + name: os-oss-up + baseurl: http://dl.opensuse.org/dist/leap/v/update + # any setting per + # https://en.opensuse.org/openSUSE:Standards_RepoInfo + # enable and autorefresh are on by default + config: + reposdir: /etc/zypp/repos.dir + servicesdir: /etc/zypp/services.d + download.use_deltarpm: true + # any setting in /etc/zypp/zypp.conf + """)], + 'frequency': PER_ALWAYS, + 'type': 'object', + 'properties': { + 'zypper': { + 'type': 'object', + 'properties': { + 'repos': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'string', + 'description': dedent("""\ + The unique id of the repo, used when + writing + /etc/zypp/repos.d/.repo.""") + }, + 'baseurl': { + 'type': 'string', + 'format': 'uri', # built-in format type + 'description': 'The base repositoy URL' + } + }, + 'required': ['id', 'baseurl'], + 'additionalProperties': True + }, + 'minItems': 1 + }, + 'config': { + 'type': 'object', + 'description': dedent("""\ + Any supported zypo.conf key is written to + /etc/zypp/zypp.conf'""") + } + }, + 'required': [], + 'minProperties': 1, # Either config or repo must be provided + 'additionalProperties': False, # only repos and config allowed + } + } +} + +__doc__ = get_schema_doc(schema) # Supplement python help() + +LOG = logging.getLogger(__name__) + + +def _canonicalize_id(repo_id): + repo_id = repo_id.replace(" ", "_") + return repo_id + + +def _format_repo_value(val): + if isinstance(val, bool): + # zypp prefers 1/0 + return 1 if val else 0 + if isinstance(val, (list, tuple)): + return "\n ".join([_format_repo_value(v) for v in val]) + if not isinstance(val, string_types): + return str(val) + return val + + +def _format_repository_config(repo_id, repo_config): + to_be = configobj.ConfigObj() + to_be[repo_id] = {} + # Do basic translation of the items -> values + for (k, v) in repo_config.items(): + # For now assume that people using this know the format + # of zypper repos and don't verify keys/values further + to_be[repo_id][k] = _format_repo_value(v) + lines = to_be.write() + return "\n".join(lines) + + +def _write_repos(repos, repo_base_path): + """Write the user-provided repo definition files + @param repos: A list of repo dictionary objects provided by the user's + cloud config. + @param repo_base_path: The directory path to which repo definitions are + written. + """ + + if not repos: + return + valid_repos = {} + for index, user_repo_config in enumerate(repos): + # Skip on absent required keys + missing_keys = set(['id', 'baseurl']).difference(set(user_repo_config)) + if missing_keys: + LOG.warning( + "Repo config at index %d is missing required config keys: %s", + index, ",".join(missing_keys)) + continue + repo_id = user_repo_config.get('id') + canon_repo_id = _canonicalize_id(repo_id) + repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) + if os.path.exists(repo_fn_pth): + LOG.info("Skipping repo %s, file %s already exists!", + repo_id, repo_fn_pth) + continue + elif repo_id in valid_repos: + LOG.info("Skipping repo %s, file %s already pending!", + repo_id, repo_fn_pth) + continue + + # Do some basic key formatting + repo_config = dict( + (k.lower().strip().replace("-", "_"), v) + for k, v in user_repo_config.items() + if k and k != 'id') + + # Set defaults if not present + for field in ['enabled', 'autorefresh']: + if field not in repo_config: + repo_config[field] = '1' + + valid_repos[repo_id] = (repo_fn_pth, repo_config) + + for (repo_id, repo_data) in valid_repos.items(): + repo_blob = _format_repository_config(repo_id, repo_data[-1]) + util.write_file(repo_data[0], repo_blob) + + +def _write_zypp_config(zypper_config): + """Write to the default zypp configuration file /etc/zypp/zypp.conf""" + if not zypper_config: + return + zypp_config = '/etc/zypp/zypp.conf' + zypp_conf_content = util.load_file(zypp_config) + new_settings = ['# Added via cloud.cfg'] + for setting, value in zypper_config.items(): + if setting == 'configdir': + msg = 'Changing the location of the zypper configuration is ' + msg += 'not supported, skipping "configdir" setting' + LOG.warning(msg) + continue + if value: + new_settings.append('%s=%s' % (setting, value)) + if len(new_settings) > 1: + new_config = zypp_conf_content + '\n'.join(new_settings) + else: + new_config = zypp_conf_content + util.write_file(zypp_config, new_config) + + +def handle(name, cfg, _cloud, log, _args): + zypper_section = cfg.get('zypper') + if not zypper_section: + LOG.debug(("Skipping module named %s," + " no 'zypper' relevant configuration found"), name) + return + repos = zypper_section.get('repos') + if not repos: + LOG.debug(("Skipping module named %s," + " no 'repos' configuration found"), name) + return + zypper_config = zypper_section.get('config', {}) + repo_base_path = zypper_config.get('reposdir', '/etc/zypp/repos.d/') + + _write_zypp_config(zypper_config) + _write_repos(repos, repo_base_path) + +# vi: ts=4 expandtab diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 50e3bd86..32de9c9b 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -84,6 +84,9 @@ cloud_config_modules: - apt-pipelining - apt-configure {% endif %} +{% if variant in ["suse"] %} + - zypper-add-repo +{% endif %} {% if variant not in ["freebsd"] %} - ntp {% endif %} diff --git a/tests/unittests/test_handler/test_handler_zypper_add_repo.py b/tests/unittests/test_handler/test_handler_zypper_add_repo.py new file mode 100644 index 00000000..315c2a5e --- /dev/null +++ b/tests/unittests/test_handler/test_handler_zypper_add_repo.py @@ -0,0 +1,237 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +import glob +import os + +from cloudinit.config import cc_zypper_add_repo +from cloudinit import util + +from cloudinit.tests import helpers +from cloudinit.tests.helpers import mock + +try: + from configparser import ConfigParser +except ImportError: + from ConfigParser import ConfigParser +import logging +from six import StringIO + +LOG = logging.getLogger(__name__) + + +class TestConfig(helpers.FilesystemMockingTestCase): + def setUp(self): + super(TestConfig, self).setUp() + self.tmp = self.tmp_dir() + self.zypp_conf = 'etc/zypp/zypp.conf' + + def test_bad_repo_config(self): + """Config has no baseurl, no file should be written""" + cfg = { + 'repos': [ + { + 'id': 'foo', + 'name': 'suse-test', + 'enabled': '1' + }, + ] + } + self.patchUtils(self.tmp) + cc_zypper_add_repo._write_repos(cfg['repos'], '/etc/zypp/repos.d') + self.assertRaises(IOError, util.load_file, + "/etc/zypp/repos.d/foo.repo") + + def test_write_repos(self): + """Verify valid repos get written""" + cfg = self._get_base_config_repos() + root_d = self.tmp_dir() + cc_zypper_add_repo._write_repos(cfg['zypper']['repos'], root_d) + repos = glob.glob('%s/*.repo' % root_d) + expected_repos = ['testing-foo.repo', 'testing-bar.repo'] + if len(repos) != 2: + assert 'Number of repos written is "%d" expected 2' % len(repos) + for repo in repos: + repo_name = os.path.basename(repo) + if repo_name not in expected_repos: + assert 'Found repo with name "%s"; unexpected' % repo_name + # Validation that the content gets properly written is in another test + + def test_write_repo(self): + """Verify the content of a repo file""" + cfg = { + 'repos': [ + { + 'baseurl': 'http://foo', + 'name': 'test-foo', + 'id': 'testing-foo' + }, + ] + } + root_d = self.tmp_dir() + cc_zypper_add_repo._write_repos(cfg['repos'], root_d) + contents = util.load_file("%s/testing-foo.repo" % root_d) + parser = ConfigParser() + parser.readfp(StringIO(contents)) + expected = { + 'testing-foo': { + 'name': 'test-foo', + 'baseurl': 'http://foo', + 'enabled': '1', + 'autorefresh': '1' + } + } + for section in expected: + self.assertTrue(parser.has_section(section), + "Contains section {0}".format(section)) + for k, v in expected[section].items(): + self.assertEqual(parser.get(section, k), v) + + def test_config_write(self): + """Write valid configuration data""" + cfg = { + 'config': { + 'download.deltarpm': 'False', + 'reposdir': 'foo' + } + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg['config']) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + 'reposdir=foo' + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + + @mock.patch('cloudinit.log.logging') + def test_config_write_skip_configdir(self, mock_logging): + """Write configuration but skip writing 'configdir' setting""" + cfg = { + 'config': { + 'download.deltarpm': 'False', + 'reposdir': 'foo', + 'configdir': 'bar' + } + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg['config']) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + 'reposdir=foo' + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + # Not finding teh right path for mocking :( + # assert mock_logging.warning.called + + def test_empty_config_section_no_new_data(self): + """When the config section is empty no new data should be written to + zypp.conf""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = None + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_empty_config_value_no_new_data(self): + """When the config section is not empty but there are no values + no new data should be written to zypp.conf""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = { + 'download.deltarpm': None + } + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_handler_full_setup(self): + """Test that the handler ends up calling the renderers""" + cfg = self._get_base_config_repos() + cfg['zypper']['config'] = { + 'download.deltarpm': 'False', + } + root_d = self.tmp_dir() + os.makedirs('%s/etc/zypp/repos.d' % root_d) + helpers.populate_dir(root_d, {self.zypp_conf: '# Zypp config\n'}) + self.reRoot(root_d) + cc_zypper_add_repo.handle('zypper_add_repo', cfg, None, LOG, []) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + expected = [ + '# Zypp config', + '# Added via cloud.cfg', + 'download.deltarpm=False', + ] + for item in contents.split('\n'): + if item not in expected: + self.assertIsNone(item) + repos = glob.glob('%s/etc/zypp/repos.d/*.repo' % root_d) + expected_repos = ['testing-foo.repo', 'testing-bar.repo'] + if len(repos) != 2: + assert 'Number of repos written is "%d" expected 2' % len(repos) + for repo in repos: + repo_name = os.path.basename(repo) + if repo_name not in expected_repos: + assert 'Found repo with name "%s"; unexpected' % repo_name + + def test_no_config_section_no_new_data(self): + """When there is no config section no new data should be written to + zypp.conf""" + cfg = self._get_base_config_repos() + root_d = self.tmp_dir() + helpers.populate_dir(root_d, {self.zypp_conf: '# No data'}) + self.reRoot(root_d) + cc_zypper_add_repo._write_zypp_config(cfg.get('config', {})) + cfg_out = os.path.join(root_d, self.zypp_conf) + contents = util.load_file(cfg_out) + self.assertEqual(contents, '# No data') + + def test_no_repo_data(self): + """When there is no repo data nothing should happen""" + root_d = self.tmp_dir() + self.reRoot(root_d) + cc_zypper_add_repo._write_repos(None, root_d) + content = glob.glob('%s/*' % root_d) + self.assertEqual(len(content), 0) + + def _get_base_config_repos(self): + """Basic valid repo configuration""" + cfg = { + 'zypper': { + 'repos': [ + { + 'baseurl': 'http://foo', + 'name': 'test-foo', + 'id': 'testing-foo' + }, + { + 'baseurl': 'http://bar', + 'name': 'test-bar', + 'id': 'testing-bar' + } + ] + } + } + return cfg diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 745bb0ff..b8fc8930 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -27,7 +27,13 @@ class GetSchemaTest(CiTestCase): """Every cloudconfig module with schema is listed in allOf keyword.""" schema = get_schema() self.assertItemsEqual( - ['cc_bootcmd', 'cc_ntp', 'cc_resizefs', 'cc_runcmd'], + [ + 'cc_bootcmd', + 'cc_ntp', + 'cc_resizefs', + 'cc_runcmd', + 'cc_zypper_add_repo' + ], [subschema['id'] for subschema in schema['allOf']]) self.assertEqual('cloud-config-schema', schema['id']) self.assertEqual( -- cgit v1.2.3