From 914822a765007be7e17539e456b3e6ff12b19442 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 7 Jun 2017 11:32:56 -0400 Subject: rhel/centos spec cleanups. Many changes here to get us able to build rpms on CentOS 5 or 6 and RHEL. * add 'Requires' as 'BuildRequires' also. This allows us to run cloud-init tools in the build environment, and also will allow us to run tests in the build process. * build for both systemd and upstart (centos 5) init systems. * Add 'centos' as a variant Adding the variant means we can use the 'centos' user as default on centos rather than a 'fedora' or 'rhel'. * drop argparse from the requirements. On any system other than python 2.6, having a 'requirements' that mentions argparse just causes problems. Instead we add that Requires to the spec directly. * list dependency on dmidecode (as redhat distro spec had) * remove duplicate line in files section ({_unitdir}/cloud-*) * Use rpm macros for init-system chunks and drop use of init_system variable template * Add el6 only build-req on python-argparse * python-cheetah is not required in the build environment as the the spec is already rendered. (We will soon move the spec to jinja). --- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/centos.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 cloudinit/distros/centos.py (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index f56c0cf7..1fd48a7b 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -32,7 +32,7 @@ from cloudinit.distros.parsers import hosts OSFAMILIES = { 'debian': ['debian', 'ubuntu'], - 'redhat': ['fedora', 'rhel'], + 'redhat': ['centos', 'fedora', 'rhel'], 'gentoo': ['gentoo'], 'freebsd': ['freebsd'], 'suse': ['sles'], diff --git a/cloudinit/distros/centos.py b/cloudinit/distros/centos.py new file mode 100644 index 00000000..4b803d2e --- /dev/null +++ b/cloudinit/distros/centos.py @@ -0,0 +1,12 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.distros import rhel +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + + +class Distro(rhel.Distro): + pass + +# vi: ts=4 expandtab -- cgit v1.2.3 From 67bab5bb804e2346673430868935f6bbcdb88f13 Mon Sep 17 00:00:00 2001 From: Ryan McCabe Date: Thu, 8 Jun 2017 13:24:23 -0400 Subject: net: Allow for NetworkManager configuration In cases where the config json specifies nameserver entries, if there are interfaces configured to use dhcp, NetworkManager, if enabled, will clobber the /etc/resolv.conf that cloud-init has produced, which can break dns. If there are no interfaces configured to use dhcp, NetworkManager could clobber /etc/resolv.conf with an empty file. This patch adds a mechanism for dropping additional configuration into /etc/NetworkManager/conf.d/ and disables management of /etc/resolv.conf by NetworkManager when nameserver information is provided in the config. LP: #1693251 Signed-off-by: Ryan McCabe --- cloudinit/distros/parsers/networkmanager_conf.py | 23 ++++++++++++++++++++++ cloudinit/net/sysconfig.py | 25 ++++++++++++++++++++++++ tests/unittests/test_net.py | 21 ++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 cloudinit/distros/parsers/networkmanager_conf.py (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/parsers/networkmanager_conf.py b/cloudinit/distros/parsers/networkmanager_conf.py new file mode 100644 index 00000000..ac51f122 --- /dev/null +++ b/cloudinit/distros/parsers/networkmanager_conf.py @@ -0,0 +1,23 @@ +# Copyright (C) 2017 Red Hat, Inc. +# +# Author: Ryan McCabe +# +# This file is part of cloud-init. See LICENSE file for license information. + +import configobj + +# This module is used to set additional NetworkManager configuration +# in /etc/NetworkManager/conf.d +# + + +class NetworkManagerConf(configobj.ConfigObj): + def __init__(self, contents): + configobj.ConfigObj.__init__(self, contents, + interpolation=False, + write_empty_values=False) + + def set_section_keypair(self, section_name, key, value): + if section_name not in self.sections: + self.main[section_name] = {} + self.main[section_name] = {key: value} diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 5d9b3d10..7ed11d1e 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -5,6 +5,7 @@ import re import six +from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf from cloudinit import util @@ -249,6 +250,9 @@ class Renderer(renderer.Renderer): self.netrules_path = config.get( 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') self.dns_path = config.get('dns_path', 'etc/resolv.conf') + nm_conf_path = 'etc/NetworkManager/conf.d/99-cloud-init.conf' + self.networkmanager_conf_path = config.get('networkmanager_conf_path', + nm_conf_path) @classmethod def _render_iface_shared(cls, iface, iface_cfg): @@ -438,6 +442,21 @@ class Renderer(renderer.Renderer): content.add_search_domain(searchdomain) return "\n".join([_make_header(';'), str(content)]) + @staticmethod + def _render_networkmanager_conf(network_state): + content = networkmanager_conf.NetworkManagerConf("") + + # If DNS server information is provided, configure + # NetworkManager to not manage dns, so that /etc/resolv.conf + # does not get clobbered. + if network_state.dns_nameservers: + content.set_section_keypair('main', 'dns', 'none') + + if len(content) == 0: + return None + out = "".join([_make_header(), "\n", "\n".join(content.write()), "\n"]) + return out + @classmethod def _render_bridge_interfaces(cls, network_state, iface_contents): bridge_filter = renderer.filter_by_type('bridge') @@ -498,6 +517,12 @@ class Renderer(renderer.Renderer): resolv_content = self._render_dns(network_state, existing_dns_path=dns_path) util.write_file(dns_path, resolv_content, file_mode) + if self.networkmanager_conf_path: + nm_conf_path = util.target_path(target, + self.networkmanager_conf_path) + nm_conf_content = self._render_networkmanager_conf(network_state) + if nm_conf_content: + util.write_file(nm_conf_path, nm_conf_content, file_mode) if self.netrules_path: netrules_content = self._render_persistent_net(network_state) netrules_path = util.target_path(target, self.netrules_path) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 91e5fb59..8edc0b89 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -155,6 +155,13 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', @@ -216,6 +223,13 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', @@ -299,6 +313,13 @@ USERCTL=no ; Created by cloud-init on instance boot automatically, do not edit. ; nameserver 172.19.0.12 +""".lstrip()), + ('etc/NetworkManager/conf.d/99-cloud-init.conf', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +[main] +dns = none """.lstrip()), ('etc/udev/rules.d/70-persistent-net.rules', "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', -- cgit v1.2.3 From 85c984c3c74b0a8751698e20915b89756580b09f Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Fri, 21 Jul 2017 11:56:50 -0700 Subject: archlinux: fix set hostname usage of write_file. cloud-init fails to set the hostname on Arch Linux because that _write_hostname passes conf instead of str(conf) to util.write_file. LP: #1705306 --- cloudinit/distros/arch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 75d46201..b4c0ba72 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -119,7 +119,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, conf, 0o644) + util.write_file(out_fn, str(conf), omode="w", mode=0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) -- cgit v1.2.3 From 0ef61b289472665f4e3059a24a8b9b91246f06ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 8 Jun 2017 15:42:12 -0500 Subject: locale: Do not re-run locale-gen if provided locale is system default. If the system configure default in /etc/default/locale is set to the same value that is provided for cloud-init's "locale" setting, then do not re-run locale-gen. This allows images built with a locale already generated to not re-run locale-gen (which can be very heavy). Also here is a fix to invoke update-locale correctly and remove the internal writing of /etc/default/locale. We were calling update-locale This ends up having no affect. The more correct invocation is: update-locale LANG= Also added some support here should we ever want to change setting LANG to setting LC_ALL (or any other key). Lastly, a test change to allow us to use assert_not_called from mock. Versions of mock in CentOS 6 do not have assert_not_called. --- cloudinit/distros/debian.py | 48 +++++++++++++---- tests/unittests/helpers.py | 12 +++++ tests/unittests/test_distros/test_debian.py | 82 +++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 tests/unittests/test_distros/test_debian.py (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index d06d46a6..abfb81f4 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -37,11 +37,11 @@ ENI_HEADER = """# This file is generated from information provided by """ NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" +LOCALE_CONF_FN = "/etc/default/locale" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" - locale_conf_fn = "/etc/default/locale" network_conf_fn = { "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", "netplan": "/etc/netplan/50-cloud-init.yaml" @@ -64,16 +64,8 @@ class Distro(distros.Distro): def apply_locale(self, locale, out_fn=None): if not out_fn: - out_fn = self.locale_conf_fn - util.subp(['locale-gen', locale], capture=False) - util.subp(['update-locale', locale], capture=False) - # "" provides trailing newline during join - lines = [ - util.make_header(), - 'LANG="%s"' % (locale), - "", - ] - util.write_file(out_fn, "\n".join(lines)) + out_fn = LOCALE_CONF_FN + apply_locale(locale, out_fn) def install_packages(self, pkglist): self.update_package_sources() @@ -225,4 +217,38 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): LOG.warning(msg) + +def apply_locale(locale, sys_path=LOCALE_CONF_FN, keyname='LANG'): + """Apply the locale. + + Run locale-gen for the provided locale and set the default + system variable `keyname` appropriately in the provided `sys_path`. + + If sys_path indicates that `keyname` is already set to `locale` + then no changes will be made and locale-gen not called. + This allows images built with a locale already generated to not re-run + locale-gen which can be very heavy. + """ + if not locale: + raise ValueError('Failed to provide locale value.') + + if not sys_path: + raise ValueError('Invalid path: %s' % sys_path) + + if os.path.exists(sys_path): + locale_content = util.load_file(sys_path) + # if LANG isn't present, regen + sys_defaults = util.load_shell_content(locale_content) + sys_val = sys_defaults.get(keyname, "") + if sys_val.lower() == locale.lower(): + LOG.debug( + "System has '%s=%s' requested '%s', skipping regeneration.", + keyname, sys_val, locale) + return + + util.subp(['locale-gen', locale], capture=False) + util.subp( + ['update-locale', '--locale-file=' + sys_path, + '%s=%s' % (keyname, locale)], capture=False) + # vi: ts=4 expandtab diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 6691cf82..08c5c469 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -376,4 +376,16 @@ except AttributeError: return wrapper return decorator + +# older versions of mock do not have the useful 'assert_not_called' +if not hasattr(mock.Mock, 'assert_not_called'): + def __mock_assert_not_called(mmock): + if mmock.call_count != 0: + msg = ("[citest] Expected '%s' to not have been called. " + "Called %s times." % + (mmock._mock_name or 'mock', mmock.call_count)) + raise AssertionError(msg) + mock.Mock.assert_not_called = __mock_assert_not_called + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_distros/test_debian.py b/tests/unittests/test_distros/test_debian.py new file mode 100644 index 00000000..2330ad52 --- /dev/null +++ b/tests/unittests/test_distros/test_debian.py @@ -0,0 +1,82 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from ..helpers import (CiTestCase, mock) + +from cloudinit.distros.debian import apply_locale +from cloudinit import util + + +@mock.patch("cloudinit.distros.debian.util.subp") +class TestDebianApplyLocale(CiTestCase): + def test_no_rerun(self, m_subp): + """If system has defined locale, no re-run is expected.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=%s\n' % locale, omode="w") + apply_locale(locale, sys_path=spath) + m_subp.assert_not_called() + + def test_rerun_if_different(self, m_subp): + """If system has different locale, locale-gen should be called.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=fr_FR.UTF-8', omode="w") + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_if_no_file(self, m_subp): + """If system has no locale file, locale-gen should be called.""" + spath = self.tmp_path("default-locale") + m_subp.return_value = (None, None) + locale = 'en_US.UTF-8' + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_on_unset_system_locale(self, m_subp): + """If system has unset locale, locale-gen should be called.""" + m_subp.return_value = (None, None) + spath = self.tmp_path("default-locale") + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=', omode="w") + apply_locale(locale, sys_path=spath) + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, 'LANG=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_rerun_on_mismatched_keys(self, m_subp): + """If key is LC_ALL and system has only LANG, rerun is expected.""" + m_subp.return_value = (None, None) + spath = self.tmp_path("default-locale") + locale = 'en_US.UTF-8' + util.write_file(spath, 'LANG=', omode="w") + apply_locale(locale, sys_path=spath, keyname='LC_ALL') + self.assertEqual( + [['locale-gen', locale], + ['update-locale', '--locale-file=' + spath, + 'LC_ALL=%s' % locale]], + [p[0][0] for p in m_subp.call_args_list]) + + def test_falseish_locale_raises_valueerror(self, m_subp): + """locale as None or "" is invalid and should raise ValueError.""" + + with self.assertRaises(ValueError) as ctext_m: + apply_locale(None) + m_subp.assert_not_called() + + self.assertEqual( + 'Failed to provide locale value.', str(ctext_m.exception)) + + with self.assertRaises(ValueError) as ctext_m: + apply_locale("") + m_subp.assert_not_called() + self.assertEqual( + 'Failed to provide locale value.', str(ctext_m.exception)) -- cgit v1.2.3