From aa8b51a48a30e3a3c863ca0ddb8bc4667026d57a Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 27 Oct 2012 19:25:48 -0700 Subject: Helpful cleanups. 1. Remove the usage of the path.join function now that all code should be going through the util file methods (and they can be mocked out as needed). 2. Adjust all occurences of the above join function to either not use it or replace it with the standard os.path.join (which can also be mocked out as needed) 3. Fix pylint from complaining about the tests folder 'helpers.py' not being found 4. Add a pylintrc file that is used instead of the options hidden in the 'run_pylint' tool. --- cloudinit/distros/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'cloudinit/distros/__init__.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2fbb0e9b..869540d2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -122,8 +122,7 @@ class Distro(object): new_etchosts = StringIO() need_write = False need_change = True - hosts_ro_fn = self._paths.join(True, "/etc/hosts") - for line in util.load_file(hosts_ro_fn).splitlines(): + for line in util.load_file("/etc/hosts").splitlines(): if line.strip().startswith(header): continue if not line.strip() or line.strip().startswith("#"): @@ -147,8 +146,7 @@ class Distro(object): need_write = True if need_write: contents = new_etchosts.getvalue() - util.write_file(self._paths.join(False, "/etc/hosts"), - contents, mode=0644) + util.write_file("/etc/hosts", contents, mode=0644) def _bring_up_interface(self, device_name): cmd = ['ifup', device_name] @@ -262,7 +260,7 @@ class Distro(object): # Import SSH keys if 'ssh_authorized_keys' in kwargs: keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, None, self._paths) + ssh_util.setup_user_keys(keys, name, key_prefix=None) return True -- cgit v1.2.3 From b80c2401123e16b9038ff3fb6f6d660717ee68e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 8 Nov 2012 17:37:16 -0800 Subject: Fix the case where on a redhat based system the fully qualified domain name should end up in /etc/sysconfig/network by passing the fqdn to the update and set hostname methods and using it accordingly. LP: #1076759 --- cloudinit/config/cc_set_hostname.py | 10 ++++++---- cloudinit/config/cc_update_hostname.py | 8 +++++--- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/debian.py | 4 ++-- cloudinit/distros/rhel.py | 25 +++++++++++++++++-------- 5 files changed, 32 insertions(+), 19 deletions(-) (limited to 'cloudinit/distros/__init__.py') diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index b0f27ebf..2b32fc94 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -27,9 +27,11 @@ def handle(name, cfg, cloud, log, _args): " not setting the hostname in module %s"), name) return - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) try: - log.debug("Setting hostname to %s", hostname) - cloud.distro.set_hostname(hostname) + log.debug("Setting the hostname to %s (%s)", fqdn, hostname) + cloud.distro.set_hostname(hostname, fqdn) except Exception: - util.logexc(log, "Failed to set hostname to %s", hostname) + util.logexc(log, "Failed to set the hostname to %s (%s)", + fqdn, hostname) + raise diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 1d6679ea..52225cd8 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -32,10 +32,12 @@ def handle(name, cfg, cloud, log, _args): " not updating the hostname in module %s"), name) return - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) try: prev_fn = os.path.join(cloud.get_cpath('data'), "previous-hostname") - cloud.distro.update_hostname(hostname, prev_fn) + log.debug("Updating hostname to %s (%s)", fqdn, hostname) + cloud.distro.update_hostname(hostname, fqdn, prev_fn) except Exception: - util.logexc(log, "Failed to set the hostname to %s", hostname) + util.logexc(log, "Failed to update the hostname to %s (%s)", + fqdn, hostname) raise diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 869540d2..bd04ba79 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -58,11 +58,11 @@ class Distro(object): return self._cfg.get(opt_name, default) @abc.abstractmethod - def set_hostname(self, hostname): + def set_hostname(self, hostname, fqdn=None): raise NotImplementedError() @abc.abstractmethod - def update_hostname(self, hostname, prev_hostname_fn): + def update_hostname(self, hostname, fqdn, prev_hostname_fn): raise NotImplementedError() @abc.abstractmethod diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index cc7e53a0..ed4070b4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -67,7 +67,7 @@ class Distro(distros.Distro): else: return distros.Distro._bring_up_interfaces(self, device_names) - def set_hostname(self, hostname): + def set_hostname(self, hostname, fqdn=None): self._write_hostname(hostname, "/etc/hostname") LOG.debug("Setting hostname to %s", hostname) util.subp(['hostname', hostname]) @@ -76,7 +76,7 @@ class Distro(distros.Distro): # "" gives trailing newline. util.write_file(out_fn, "%s\n" % str(hostname), 0644) - def update_hostname(self, hostname, prev_fn): + def update_hostname(self, hostname, fqdn, prev_fn): hostname_prev = self._read_hostname(prev_fn) hostname_in_etc = self._read_hostname("/etc/hostname") update_files = [] diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index bf3c18d2..e4c27216 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -146,8 +146,13 @@ class Distro(distros.Distro): lines.insert(0, _make_header()) util.write_file(fn, "\n".join(lines), 0644) - def set_hostname(self, hostname): - self._write_hostname(hostname, '/etc/sysconfig/network') + def set_hostname(self, hostname, fqdn=None): + # See: http://bit.ly/TwitgL + # Should be fqdn if we can use it + sysconfig_hostname = fqdn + if not sysconfig_hostname: + sysconfig_hostname = hostname + self._write_hostname(sysconfig_hostname, '/etc/sysconfig/network') LOG.debug("Setting hostname to %s", hostname) util.subp(['hostname', hostname]) @@ -165,28 +170,32 @@ class Distro(distros.Distro): } self._update_sysconfig_file(out_fn, host_cfg) - def update_hostname(self, hostname, prev_file): + def update_hostname(self, hostname, fqdn, prev_file): + # See: http://bit.ly/TwitgL + # Should be fqdn if we can use it + sysconfig_hostname = fqdn + if not sysconfig_hostname: + sysconfig_hostname = hostname hostname_prev = self._read_hostname(prev_file) hostname_in_sys = self._read_hostname("/etc/sysconfig/network") update_files = [] - if not hostname_prev or hostname_prev != hostname: + if not hostname_prev or hostname_prev != sysconfig_hostname: update_files.append(prev_file) if (not hostname_in_sys or (hostname_in_sys == hostname_prev - and hostname_in_sys != hostname)): + and hostname_in_sys != sysconfig_hostname)): update_files.append("/etc/sysconfig/network") for fn in update_files: try: - self._write_hostname(hostname, fn) + self._write_hostname(sysconfig_hostname, fn) except: util.logexc(LOG, "Failed to write hostname %s to %s", - hostname, fn) + sysconfig_hostname, fn) if (hostname_in_sys and hostname_prev and hostname_in_sys != hostname_prev): LOG.debug(("%s differs from /etc/sysconfig/network." " Assuming user maintained hostname."), prev_file) if "/etc/sysconfig/network" in update_files: - # Only do this if we are running in non-adjusted root mode LOG.debug("Setting hostname to %s", hostname) util.subp(['hostname', hostname]) -- cgit v1.2.3 From 8c006684034c13719171672836edfc65bf02ebe9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Nov 2012 14:40:41 -0800 Subject: Fix pep8 warnings. --- cloudinit/distros/__init__.py | 2 +- tools/run-pep8 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/distros/__init__.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index bd04ba79..3392a065 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -81,7 +81,7 @@ class Distro(object): def _get_arch_package_mirror_info(self, arch=None): mirror_info = self.get_option("package_mirrors", []) - if arch == None: + if not arch: arch = self.get_primary_arch() return _get_arch_package_mirror_info(mirror_info, arch) diff --git a/tools/run-pep8 b/tools/run-pep8 index 1dfa92c3..20e594bc 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -31,6 +31,7 @@ IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent +IGNORE="$IGNORE,E502" # The backslash is redundant between brackets cmd=( ${base}/hacking.py -- cgit v1.2.3 From 1169dcc5f18fd9a5adbf353bec87e48d563550a5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 9 Nov 2012 15:28:35 -0800 Subject: Fix the merging of group configuration when that group configuration is a dict => members. LP: #1077245 --- cloudinit/distros/__init__.py | 32 +++++++++++++++++++--- .../test_distros/test_user_data_normalize.py | 22 +++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) (limited to 'cloudinit/distros/__init__.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3392a065..2d01efc3 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -425,12 +425,36 @@ def _get_arch_package_mirror_info(package_mirrors, arch): # is the standard form used in the rest # of cloud-init def _normalize_groups(grp_cfg): - if isinstance(grp_cfg, (str, basestring, list)): + if isinstance(grp_cfg, (str, basestring)): + grp_cfg = grp_cfg.strip().split(",") + if isinstance(grp_cfg, (list)): c_grp_cfg = {} - for i in util.uniq_merge(grp_cfg): - c_grp_cfg[i] = [] + for i in grp_cfg: + if isinstance(i, (dict)): + for k, v in i.items(): + if k not in c_grp_cfg: + if isinstance(v, (list)): + c_grp_cfg[k] = list(v) + elif isinstance(v, (basestring, str)): + c_grp_cfg[k] = [v] + else: + raise TypeError("Bad group member type %s" % + util.obj_name(v)) + else: + if isinstance(v, (list)): + c_grp_cfg[k].extend(v) + elif isinstance(v, (basestring, str)): + c_grp_cfg[k].append(v) + else: + raise TypeError("Bad group member type %s" % + util.obj_name(v)) + elif isinstance(i, (str, basestring)): + if i not in c_grp_cfg: + c_grp_cfg[i] = [] + else: + raise TypeError("Unknown group name type %s" % + util.obj_name(i)) grp_cfg = c_grp_cfg - groups = {} if isinstance(grp_cfg, (dict)): for (grp_name, grp_members) in grp_cfg.items(): diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 8f0d8896..50400c8a 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -30,6 +30,28 @@ class TestUGNormalize(MockerTestCase): def _norm(self, cfg, distro): return distros.normalize_users_groups(cfg, distro) + def test_group_dict(self): + distro = self._make_distro('ubuntu') + g = {'groups': [ + { + 'ubuntu': ['foo', 'bar'], + 'bob': 'users', + }, + 'cloud-users', + { + 'bob': 'users2', + }, + ] + } + (users, groups) = self._norm(g, distro) + self.assertIn('ubuntu', groups) + ub_members = groups['ubuntu'] + self.assertEquals(sorted(['foo', 'bar']), sorted(ub_members)) + self.assertIn('bob', groups) + b_members = groups['bob'] + self.assertEquals(sorted(['users', 'users2']), + sorted(b_members)) + def test_basic_groups(self): distro = self._make_distro('ubuntu') ug_cfg = { -- cgit v1.2.3 From a17a69c35c1de0a6bd6f054f76d3da9e4a9c5364 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 10 Nov 2012 10:15:16 -0800 Subject: Sudoers.d creation cleanups + tests. --- cloudinit/distros/__init__.py | 20 +++++++++++------ tests/unittests/helpers.py | 1 + tests/unittests/test_distros/test_generic.py | 32 +++++++++++++++++++++++++--- tests/unittests/test_runs/test_merge_run.py | 2 +- 4 files changed, 45 insertions(+), 10 deletions(-) (limited to 'cloudinit/distros/__init__.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2d01efc3..d2cb0a8b 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -283,8 +283,10 @@ class Distro(object): # Ensure the dir is included and that # it actually exists as a directory sudoers_contents = '' + base_exists = False if os.path.exists(sudo_base): sudoers_contents = util.load_file(sudo_base) + base_exists = True found_include = False for line in sudoers_contents.splitlines(): line = line.strip() @@ -299,18 +301,24 @@ class Distro(object): found_include = True break if not found_include: - sudoers_contents += "\n#includedir %s\n" % (path) try: - if not os.path.exists(sudo_base): + if not base_exists: + lines = [('# See sudoers(5) for more information' + ' on "#include" directives:'), '', + '# Added by cloud-init', + "#includedir %s" % (path), ''] + sudoers_contents = "\n".join(lines) util.write_file(sudo_base, sudoers_contents, 0440) else: - with open(sudo_base, 'a') as f: - f.write(sudoers_contents) - LOG.debug("added '#includedir %s' to %s" % (path, sudo_base)) + lines = ['', '# Added by cloud-init', + "#includedir %s" % (path), ''] + sudoers_contents = "\n".join(lines) + util.append_file(sudo_base, sudoers_contents) + LOG.debug("Added '#includedir %s' to %s" % (path, sudo_base)) except IOError as e: util.logexc(LOG, "Failed to write %s" % sudo_base, e) raise e - util.ensure_dir(path, 0755) + util.ensure_dir(path, 0750) def write_sudo_rules(self, user, diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 2c5dcad2..e8080668 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -103,6 +103,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): def patchUtils(self, new_root): patch_funcs = { util: [('write_file', 1), + ('append_file', 1), ('load_file', 1), ('ensure_dir', 1), ('chmod', 1), diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 2df4c2f0..704699b5 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -1,6 +1,9 @@ -from mocker import MockerTestCase - from cloudinit import distros +from cloudinit import util + +from tests.unittests import helpers + +import os unknown_arch_info = { 'arches': ['default'], @@ -27,7 +30,7 @@ gpmi = distros._get_package_mirror_info # pylint: disable=W0212 gapmi = distros._get_arch_package_mirror_info # pylint: disable=W0212 -class TestGenericDistro(MockerTestCase): +class TestGenericDistro(helpers.FilesystemMockingTestCase): def return_first(self, mlist): if not mlist: @@ -52,6 +55,29 @@ class TestGenericDistro(MockerTestCase): # Make a temp directoy for tests to use. self.tmp = self.makeDir() + def test_sudoers_ensure_new(self): + cls = distros.fetch("ubuntu") + d = cls("ubuntu", {}, None) + self.patchOS(self.tmp) + self.patchUtils(self.tmp) + d.ensure_sudo_dir("/b") + contents = util.load_file("/etc/sudoers") + self.assertIn("includedir /b", contents) + self.assertTrue(os.path.isdir("/b")) + + def test_sudoers_ensure_append(self): + cls = distros.fetch("ubuntu") + d = cls("ubuntu", {}, None) + self.patchOS(self.tmp) + self.patchUtils(self.tmp) + util.write_file("/etc/sudoers", "josh, josh\n") + d.ensure_sudo_dir("/b") + contents = util.load_file("/etc/sudoers") + self.assertIn("includedir /b", contents) + self.assertTrue(os.path.isdir("/b")) + self.assertIn("josh", contents) + self.assertEquals(2, contents.count("josh")) + def test_arch_package_mirror_info_unknown(self): """for an unknown arch, we should get back that with arch 'default'.""" arch_mirrors = gapmi(package_mirrors, arch="unknown") diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 04c03730..36de97ae 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -7,7 +7,7 @@ from cloudinit import stages from cloudinit import util -class TestSimpleRun(helpers.FilesystemMockingTestCase): +class TestMergeRun(helpers.FilesystemMockingTestCase): def _patchIn(self, root): self.restore() self.patchOS(root) -- cgit v1.2.3 From 3248ac9bbb2008e88a3bd9c030ba0fcbc14b7fce Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 10 Nov 2012 22:32:49 -0500 Subject: whitespace / indentation cleanups These changes were pulled out of the previous merge (cc_yum_add_repo) as they were unrelated there. Re-applying them here. --- cloudinit/distros/__init__.py | 38 +++++++++++++--------------- cloudinit/sources/DataSourceAltCloud.py | 2 +- cloudinit/util.py | 3 +-- tests/unittests/test_runs/test_simple_run.py | 6 +++-- tools/hacking.py | 4 +-- 5 files changed, 26 insertions(+), 27 deletions(-) (limited to 'cloudinit/distros/__init__.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index d2cb0a8b..8a98e334 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -187,23 +187,23 @@ class Distro(object): # inputs. If something goes wrong, we can end up with a system # that nobody can login to. adduser_opts = { - "gecos": '--comment', - "homedir": '--home', - "primary_group": '--gid', - "groups": '--groups', - "passwd": '--password', - "shell": '--shell', - "expiredate": '--expiredate', - "inactive": '--inactive', - "selinux_user": '--selinux-user', - } + "gecos": '--comment', + "homedir": '--home', + "primary_group": '--gid', + "groups": '--groups', + "passwd": '--password', + "shell": '--shell', + "expiredate": '--expiredate', + "inactive": '--inactive', + "selinux_user": '--selinux-user', + } adduser_opts_flags = { - "no_user_group": '--no-user-group', - "system": '--system', - "no_log_init": '--no-log-init', - "no_create_home": "-M", - } + "no_user_group": '--no-user-group', + "system": '--system', + "no_log_init": '--no-log-init', + "no_create_home": "-M", + } # Now check the value and create the command for option in kwargs: @@ -320,11 +320,9 @@ class Distro(object): raise e util.ensure_dir(path, 0750) - def write_sudo_rules(self, - user, - rules, - sudo_file="/etc/sudoers.d/90-cloud-init-users", - ): + def write_sudo_rules(self, user, rules, sudo_file=None): + if not sudo_file: + sudo_file = "/etc/sudoers.d/90-cloud-init-users" content_header = "# user rules for %s" % user content = "%s\n%s %s\n\n" % (content_header, user, rules) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index d7e1204f..9812bdcb 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -47,7 +47,7 @@ META_DATA_NOT_SUPPORTED = { 'instance-id': 455, 'local-hostname': 'localhost', 'placement': {}, - } +} def read_user_data_callback(mount_dir): diff --git a/cloudinit/util.py b/cloudinit/util.py index 7890a3d6..4f5b15ee 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1193,8 +1193,7 @@ def yaml_dumps(obj): indent=4, explicit_start=True, explicit_end=True, - default_flow_style=False, - ) + default_flow_style=False) return formatted diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 22d6cf2c..60ef812a 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -37,11 +37,13 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.replicateTestRoot('simple_ubuntu', new_root) cfg = { 'datasource_list': ['None'], - 'write_files': [{ + 'write_files': [ + { 'path': '/etc/blah.ini', 'content': 'blah', 'permissions': 0755, - }], + }, + ], 'cloud_init_modules': ['write-files'], } cloud_cfg = util.yaml_dumps(cfg) diff --git a/tools/hacking.py b/tools/hacking.py index 11163df3..26a07c53 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -66,8 +66,8 @@ def cloud_import_alphabetical(physical_line, line_number, lines): # handle import x # use .lower since capitalization shouldn't dictate order split_line = import_normalize(physical_line.strip()).lower().split() - split_previous = import_normalize(lines[line_number - 2] - ).strip().lower().split() + split_previous = import_normalize(lines[line_number - 2]) + split_previous = split_previous.strip().lower().split() # with or without "as y" length = [2, 4] if (len(split_line) in length and len(split_previous) in length and -- cgit v1.2.3