From 0a448dd034883c07f85091dbfc9117de7227eb8d Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 25 May 2017 11:04:55 -0600 Subject: ntp: Add schema definition and passive schema validation. cloud-config files are very flexible and permissive. This adds a jsonsschema definition to the cc_ntp module and validation functions in cloudinit/config/schema which will log warnings about invalid configuration values in the ntp section. A cmdline tools/cloudconfig-schema is added which can be used in our dev environments to quickly attempt to exercise the ntp schema. It is also exposed as a main in cloudinit.config.schema. (python3 -m cloudinit.config.schema) LP: #1692916 --- tests/unittests/helpers.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 9ff15993..e78abce2 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -19,10 +19,6 @@ try: from contextlib import ExitStack except ImportError: from contextlib2 import ExitStack -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO from cloudinit import helpers as ch from cloudinit import util @@ -102,7 +98,7 @@ class CiTestCase(TestCase): if self.with_logs: # Create a log handler so unit tests can search expected logs. logger = logging.getLogger() - self.logs = StringIO() + self.logs = six.StringIO() handler = logging.StreamHandler(self.logs) self.old_handlers = logger.handlers logger.handlers = [handler] -- cgit v1.2.3 From 0fe6a0607408d387f4b0d4482b95afbc5d3f3909 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Fri, 5 Dec 2014 14:34:10 -0800 Subject: write_file(s): Print permissions as octal, not decimal Unix file modes are usually represented as octal, but they were being interpreted as decimal, for example 0o644 would be printed as '420'. Reviewed-by: Tom Kirchner --- cloudinit/config/cc_write_files.py | 10 ++++++- cloudinit/util.py | 6 ++++- tests/unittests/helpers.py | 8 +++--- tests/unittests/test_datasource/test_azure.py | 3 ++- tests/unittests/test_handler/test_handler_ntp.py | 3 ++- .../test_handler/test_handler_write_files.py | 31 ++++++++++++++++++++-- 6 files changed, 52 insertions(+), 9 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 72e1cdd6..1835a31b 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -53,6 +53,7 @@ import six from cloudinit.settings import PER_INSTANCE from cloudinit import util + frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" @@ -119,7 +120,14 @@ def decode_perms(perm, default, log): # Force to string and try octal conversion return int(str(perm), 8) except (TypeError, ValueError): - log.warn("Undecodable permissions %s, assuming %s", perm, default) + reps = [] + for r in (perm, default): + try: + reps.append("%o" % r) + except TypeError: + reps.append("%r" % r) + log.warning( + "Undecodable permissions {0}, returning default {1}".format(*reps)) return default diff --git a/cloudinit/util.py b/cloudinit/util.py index 415ca374..ec68925e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1751,8 +1751,12 @@ def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False): else: content = decode_binary(content) write_type = 'characters' + try: + mode_r = "%o" % mode + except TypeError: + mode_r = "%r" % mode LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode_r, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index e78abce2..569f1aef 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -97,11 +97,13 @@ class CiTestCase(TestCase): super(CiTestCase, self).setUp() if self.with_logs: # Create a log handler so unit tests can search expected logs. - logger = logging.getLogger() + self.logger = logging.getLogger() self.logs = six.StringIO() + formatter = logging.Formatter('%(levelname)s: %(message)s') handler = logging.StreamHandler(self.logs) - self.old_handlers = logger.handlers - logger.handlers = [handler] + handler.setFormatter(formatter) + self.old_handlers = self.logger.handlers + self.logger.handlers = [handler] def tearDown(self): if self.with_logs: diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 42f49e06..b17f389c 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -263,7 +263,8 @@ fdescfs /dev/fd fdescfs rw 0 0 {}, distro=None, paths=self.paths) self.assertFalse(dsrc.get_data()) self.assertEqual( - "Non-Azure DMI asset tag '{0}' discovered.\n".format(nonazure_tag), + "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format( + nonazure_tag), self.logs.getvalue()) def test_basic_seed_dir(self): diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 3a9f7f7e..c4299d94 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -216,7 +216,8 @@ class TestNtp(FilesystemMockingTestCase): """When no ntp section is defined handler logs a warning and noops.""" cc_ntp.handle('cc_ntp', {}, None, None, []) self.assertEqual( - 'Skipping module named cc_ntp, not present or disabled by cfg\n', + 'DEBUG: Skipping module named cc_ntp, ' + 'not present or disabled by cfg\n', self.logs.getvalue()) def test_ntp_handler_schema_validation_allows_empty_ntp_config(self): diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index fb252d1d..88a4742b 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -1,10 +1,10 @@ # This file is part of cloud-init. See LICENSE file for license information. -from cloudinit.config.cc_write_files import write_files +from cloudinit.config.cc_write_files import write_files, decode_perms from cloudinit import log as logging from cloudinit import util -from ..helpers import FilesystemMockingTestCase +from ..helpers import CiTestCase, FilesystemMockingTestCase import base64 import gzip @@ -98,6 +98,33 @@ class TestWriteFiles(FilesystemMockingTestCase): self.assertEqual(len(expected), flen_expected) +class TestDecodePerms(CiTestCase): + + with_logs = True + + def test_none_returns_default(self): + """If None is passed as perms, then default should be returned.""" + default = object() + found = decode_perms(None, default, self.logger) + self.assertEqual(default, found) + + def test_integer(self): + """A valid integer should return itself.""" + found = decode_perms(0o755, None, self.logger) + self.assertEqual(0o755, found) + + def test_valid_octal_string(self): + """A string should be read as octal.""" + found = decode_perms("644", None, self.logger) + self.assertEqual(0o644, found) + + def test_invalid_octal_string_returns_default_and_warns(self): + """A string with invalid octal should warn and return default.""" + found = decode_perms("999", None, self.logger) + self.assertIsNone(found) + self.assertIn("WARNING: Undecodable", self.logs.getvalue()) + + def _gzip_bytes(data): buf = six.BytesIO() fp = None -- cgit v1.2.3 From 865e941f3f88c7daeafbf1eab856e02ce2b6a5f7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Jul 2017 17:16:35 -0700 Subject: tests: fixes for issues uncovered when moving to python 3.6. This includes a few fixes found when testing with python 3.6. - fix eni renderer when target is None This just uses the util.target_path() in the event that target is None. - change test cases to not rely on the cached result of util.get_cmdline() and other cached globals. Update the base TestCase to unset that cache. - mock calls to system_is_snappy from the create_users test cases. - drop unused _pp_root in test_simple_run.py LP: #1703697 --- cloudinit/net/netplan.py | 3 ++- tests/unittests/helpers.py | 21 +++++++++++++++- tests/unittests/test_distros/test_create_users.py | 30 +++++++++-------------- tests/unittests/test_runs/test_simple_run.py | 18 -------------- 4 files changed, 34 insertions(+), 38 deletions(-) (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 67543305..9f35b72b 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -209,7 +209,8 @@ class Renderer(renderer.Renderer): # check network state for version # if v2, then extract network_state.config # else render_v2_from_state - fpnplan = os.path.join(target, self.netplan_path) + fpnplan = os.path.join(util.target_path(target), self.netplan_path) + util.ensure_dir(os.path.dirname(fpnplan)) header = self.netplan_header if self.netplan_header else "" diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 569f1aef..6691cf82 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -82,7 +82,26 @@ def retarget_many_wrapper(new_base, am, old_func): class TestCase(unittest2.TestCase): - pass + def reset_global_state(self): + """Reset any global state to its original settings. + + cloudinit caches some values in cloudinit.util. Unit tests that + involved those cached paths were then subject to failure if the order + of invocation changed (LP: #1703697). + + This function resets any of these global state variables to their + initial state. + + In the future this should really be done with some registry that + can then be cleaned in a more obvious way. + """ + util.PROC_CMDLINE = None + util._DNS_REDIRECT_IP = None + util._LSB_RELEASE = {} + + def setUp(self): + super(unittest2.TestCase, self).setUp() + self.reset_global_state() class CiTestCase(TestCase): diff --git a/tests/unittests/test_distros/test_create_users.py b/tests/unittests/test_distros/test_create_users.py index 9ded4f6c..1d02f7bd 100644 --- a/tests/unittests/test_distros/test_create_users.py +++ b/tests/unittests/test_distros/test_create_users.py @@ -38,6 +38,8 @@ class MyBaseDistro(distros.Distro): raise NotImplementedError() +@mock.patch("cloudinit.distros.util.system_is_snappy", return_value=False) +@mock.patch("cloudinit.distros.util.subp") class TestCreateUser(TestCase): def setUp(self): super(TestCase, self).setUp() @@ -53,8 +55,7 @@ class TestCreateUser(TestCase): logcmd[i + 1] = 'REDACTED' return mock.call(args, logstring=logcmd) - @mock.patch("cloudinit.distros.util.subp") - def test_basic(self, m_subp): + def test_basic(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user) self.assertEqual( @@ -62,8 +63,7 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '-m']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_no_home(self, m_subp): + def test_no_home(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user, no_create_home=True) self.assertEqual( @@ -71,8 +71,7 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '-M']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_system_user(self, m_subp): + def test_system_user(self, m_subp, m_is_snappy): # system user should have no home and get --system user = 'foouser' self.dist.create_user(user, system=True) @@ -81,8 +80,7 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '--system', '-M']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_explicit_no_home_false(self, m_subp): + def test_explicit_no_home_false(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user, no_create_home=False) self.assertEqual( @@ -90,16 +88,14 @@ class TestCreateUser(TestCase): [self._useradd2call([user, '-m']), mock.call(['passwd', '-l', user])]) - @mock.patch("cloudinit.distros.util.subp") - def test_unlocked(self, m_subp): + def test_unlocked(self, m_subp, m_is_snappy): user = 'foouser' self.dist.create_user(user, lock_passwd=False) self.assertEqual( m_subp.call_args_list, [self._useradd2call([user, '-m'])]) - @mock.patch("cloudinit.distros.util.subp") - def test_set_password(self, m_subp): + def test_set_password(self, m_subp, m_is_snappy): user = 'foouser' password = 'passfoo' self.dist.create_user(user, passwd=password) @@ -109,8 +105,7 @@ class TestCreateUser(TestCase): mock.call(['passwd', '-l', user])]) @mock.patch("cloudinit.distros.util.is_group") - @mock.patch("cloudinit.distros.util.subp") - def test_group_added(self, m_subp, m_is_group): + def test_group_added(self, m_is_group, m_subp, m_is_snappy): m_is_group.return_value = False user = 'foouser' self.dist.create_user(user, groups=['group1']) @@ -121,8 +116,7 @@ class TestCreateUser(TestCase): self.assertEqual(m_subp.call_args_list, expected) @mock.patch("cloudinit.distros.util.is_group") - @mock.patch("cloudinit.distros.util.subp") - def test_only_new_group_added(self, m_subp, m_is_group): + def test_only_new_group_added(self, m_is_group, m_subp, m_is_snappy): ex_groups = ['existing_group'] groups = ['group1', ex_groups[0]] m_is_group.side_effect = lambda m: m in ex_groups @@ -135,8 +129,8 @@ class TestCreateUser(TestCase): self.assertEqual(m_subp.call_args_list, expected) @mock.patch("cloudinit.distros.util.is_group") - @mock.patch("cloudinit.distros.util.subp") - def test_create_groups_with_whitespace_string(self, m_subp, m_is_group): + def test_create_groups_with_whitespace_string( + self, m_is_group, m_subp, m_is_snappy): # groups supported as a comma delimeted string even with white space m_is_group.return_value = False user = 'foouser' diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 31324204..55f15b55 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -16,24 +16,6 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.patchOS(root) self.patchUtils(root) - def _pp_root(self, root, repatch=True): - for (dirpath, dirnames, filenames) in os.walk(root): - print(dirpath) - for f in filenames: - joined = os.path.join(dirpath, f) - if os.path.islink(joined): - print("f %s - (symlink)" % (f)) - else: - print("f %s" % (f)) - for d in dirnames: - joined = os.path.join(dirpath, d) - if os.path.islink(joined): - print("d %s - (symlink)" % (d)) - else: - print("d %s" % (d)) - if repatch: - self._patchIn(root) - def test_none_ds(self): new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, new_root) -- cgit v1.2.3 From 0ef61b289472665f4e3059a24a8b9b91246f06ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Jun 2017 15:42:12 -0500 Subject: locale: Do not re-run locale-gen if provided locale is system default. If the system configure default in /etc/default/locale is set to the same value that is provided for cloud-init's "locale" setting, then do not re-run locale-gen. This allows images built with a locale already generated to not re-run locale-gen (which can be very heavy). Also here is a fix to invoke update-locale correctly and remove the internal writing of /etc/default/locale. We were calling update-locale This ends up having no affect. The more correct invocation is: update-locale LANG= Also added some support here should we ever want to change setting LANG to setting LC_ALL (or any other key). Lastly, a test change to allow us to use assert_not_called from mock. Versions of mock in CentOS 6 do not have assert_not_called. --- cloudinit/distros/debian.py | 48 +++++++++++++---- tests/unittests/helpers.py | 12 +++++ tests/unittests/test_distros/test_debian.py | 82 +++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 tests/unittests/test_distros/test_debian.py (limited to 'tests/unittests/helpers.py') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index d06d46a6..abfb81f4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -37,11 +37,11 @@ ENI_HEADER = """# This file is generated from information provided by """ NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" +LOCALE_CONF_FN = "/etc/default/locale" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" - locale_conf_fn = "/etc/default/locale" network_conf_fn = { "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", "netplan": "/etc/netplan/50-cloud-init.yaml" @@ -64,16 +64,8 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', locale], capture=False) - util.subp(['update-locale', locale], capture=False) - # "" provides trailing newline during join - lines = [ - util.make_header(), - 'LANG="%s"' % (locale), - "", - ] - util.write_file(out_fn, "\n".join(lines)) + out_fn = LOCALE_CONF_FN + apply_locale(locale, out_fn) def install_packages(self, pkglist): self.update_package_sources() @@ -225,4 +217,38 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): LOG.warning(msg) + +def apply_locale(locale, sys_path=LOCALE_CONF_FN, keyname='LANG'): + """Apply the locale. + + Run locale-gen for the provided locale and set the default + system variable `keyname` appropriately in the provided `sys_path`. + + If sys_path indicates that `keyname` is already set to `locale` + then no changes will be made and locale-gen not called. + This allows images built with a locale already generated to not re-run + locale-gen which can be very heavy. + """ + if not locale: + raise ValueError('Failed to provide locale value.') + + if not sys_path: + raise ValueError('Invalid path: %s' % sys_path) + + if os.path.exists(sys_path): + locale_content = util.load_file(sys_path) + # if LANG isn't present, regen + sys_defaults = util.load_shell_content(locale_content) + sys_val = sys_defaults.get(keyname, "") + if sys_val.lower() == locale.lower(): + LOG.debug( + "System has '%s=%s' requested '%s', skipping regeneration.", + keyname, sys_val, locale) + return + + util.subp(['locale-gen', locale], capture=False) + util.subp( + ['update-locale', '--locale-file=' + sys_path, + '%s=%s' % (keyname, locale)], capture=False) + # vi: ts=4 expandtab diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 6691cf82..08c5c469 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -376,4 +376,16 @@ except AttributeError: return wrapper return decorator + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py new file mode 100644 index 00000000..2330ad52 --- /dev/null +++ b/tests/unittests/test_distros/test_debian.py @@ -0,0 +1,82 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import (CiTestCase, mock) + +from cloudinit.distros.debian import apply_locale +from cloudinit import util + + +@mock.patch("cloudinit.distros.debian.util.subp") +class TestDebianApplyLocale(CiTestCase): + def test_no_rerun(self, m_subp): + """If system has defined locale, no re-run is expected.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=%s\n' % locale, omode="w") + apply_locale(locale, sys_path=spath) + m_subp.assert_not_called() + + def test_rerun_if_different(self, m_subp): + """If system has different locale, locale-gen should be called.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=fr_FR.UTF-8', omode="w") + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_if_no_file(self, m_subp): + """If system has no locale file, locale-gen should be called.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_on_unset_system_locale(self, m_subp): + """If system has unset locale, locale-gen should be called.""" + m_subp.return_value = (None, None) + spath = self.tmp_path("default-locale") + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=', omode="w") + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_on_mismatched_keys(self, m_subp): + """If key is LC_ALL and system has only LANG, rerun is expected.""" + m_subp.return_value = (None, None) + spath = self.tmp_path("default-locale") + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=', omode="w") + apply_locale(locale, sys_path=spath, keyname='LC_ALL') + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, + 'LC_ALL=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_falseish_locale_raises_valueerror(self, m_subp): + """locale as None or "" is invalid and should raise ValueError.""" + + with self.assertRaises(ValueError) as ctext_m: + apply_locale(None) + m_subp.assert_not_called() + + self.assertEqual( + 'Failed to provide locale value.', str(ctext_m.exception)) + + with self.assertRaises(ValueError) as ctext_m: + apply_locale("") + m_subp.assert_not_called() + self.assertEqual( + 'Failed to provide locale value.', str(ctext_m.exception)) -- cgit v1.2.3