From 8c2caad47a958799476f705e42a1fad9ec636233 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 9 Nov 2017 16:10:47 -0500 Subject: Azure: don't generate network configuration for SRIOV devices Azure kernel now configures the SRIOV devices itself so cloud-init does not need to provide any SRIOV device configuration or udev naming rules. LP: #1721579 --- cloudinit/sources/DataSourceAzure.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 80c2bd12..8c3492d9 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -465,10 +465,8 @@ class DataSourceAzure(sources.DataSource): 1. Probe the drivers of the net-devices present and inject them in the network configuration under params: driver: value - 2. If the driver value is 'mlx4_core', the control mode should be - set to manual. The device will be later used to build a bond, - for now we want to ensure the device gets named but does not - break any network configuration + 2. Generate a fallback network config that does not include any of + the blacklisted devices. """ blacklist = ['mlx4_core'] if not self._network_config: @@ -477,25 +475,6 @@ class DataSourceAzure(sources.DataSource): netconfig = net.generate_fallback_config( blacklist_drivers=blacklist, config_driver=True) - # if we have any blacklisted devices, update the network_config to - # include the device, mac, and driver values, but with no ip - # config; this ensures udev rules are generated but won't affect - # ip configuration - bl_found = 0 - for bl_dev in [dev for dev in net.get_devicelist() - if net.device_driver(dev) in blacklist]: - bl_found += 1 - cfg = { - 'type': 'physical', - 'name': 'vf%d' % bl_found, - 'mac_address': net.get_interface_mac(bl_dev), - 'params': { - 'driver': net.device_driver(bl_dev), - 'device_id': net.device_devid(bl_dev), - }, - } - netconfig['config'].append(cfg) - self._network_config = netconfig return self._network_config -- cgit v1.2.3 From 420c3452f2009e08f545eaa9ef126ab922133678 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Wed, 8 Nov 2017 15:06:59 -0500 Subject: Improve warning message when a template is not found. At present the location for the template file look up upon failure includes the template file itself. However based on the wording of the message it should only contain the template directory issue LP: #1731035 --- cloudinit/cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index d8a9fc86..ba616781 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -56,8 +56,8 @@ class Cloud(object): def get_template_filename(self, name): fn = self.paths.template_tpl % (name) if not os.path.isfile(fn): - LOG.warning("No template found at %s for template named %s", - fn, name) + LOG.warning("No template found in %s for template named %s", + os.path.dirname(fn), name) return None return fn -- cgit v1.2.3 From 9bc4ce0596544ffa56d9d67245b00e07006a8662 Mon Sep 17 00:00:00 2001 From: Dave Mulford Date: Mon, 9 Oct 2017 15:28:15 -0500 Subject: rh_subscription: Perform null checks for enabled and disabled repos. The rh_subscription module doesn't perform null checks when attempting to iterate on the enabled and disable repos arrays. When only one is specified, cloud-init fails to run. --- cloudinit/config/cc_rh_subscription.py | 46 ++++++++++++++++++++------------- tests/unittests/test_rh_subscription.py | 15 +++++++++++ 2 files changed, 43 insertions(+), 18 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 7f36cf8f..a9d21e78 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -38,14 +38,16 @@ Subscription`` example config. server-hostname: """ +from cloudinit import log as logging from cloudinit import util +LOG = logging.getLogger(__name__) + distros = ['fedora', 'rhel'] def handle(name, cfg, _cloud, log, _args): - sm = SubscriptionManager(cfg) - sm.log = log + sm = SubscriptionManager(cfg, log=log) if not sm.is_configured(): log.debug("%s: module not configured.", name) return None @@ -86,10 +88,9 @@ def handle(name, cfg, _cloud, log, _args): if not return_stat: raise SubscriptionError("Unable to attach pools {0}" .format(sm.pools)) - if (sm.enable_repo is not None) or (sm.disable_repo is not None): - return_stat = sm.update_repos(sm.enable_repo, sm.disable_repo) - if not return_stat: - raise SubscriptionError("Unable to add or remove repos") + return_stat = sm.update_repos() + if not return_stat: + raise SubscriptionError("Unable to add or remove repos") sm.log_success("rh_subscription plugin completed successfully") except SubscriptionError as e: sm.log_warn(str(e)) @@ -108,7 +109,10 @@ class SubscriptionManager(object): 'rhsm-baseurl', 'server-hostname', 'auto-attach', 'service-level'] - def __init__(self, cfg): + def __init__(self, cfg, log=None): + if log is None: + log = LOG + self.log = log self.cfg = cfg self.rhel_cfg = self.cfg.get('rh_subscription', {}) self.rhsm_baseurl = self.rhel_cfg.get('rhsm-baseurl') @@ -130,7 +134,7 @@ class SubscriptionManager(object): def log_warn(self, msg): '''Simple wrapper for logging warning messages. Useful for unittests''' - self.log.warn(msg) + self.log.warning(msg) def _verify_keys(self): ''' @@ -245,7 +249,7 @@ class SubscriptionManager(object): return False reg_id = return_out.split("ID: ")[1].rstrip() - self.log.debug("Registered successfully with ID {0}".format(reg_id)) + self.log.debug("Registered successfully with ID %s", reg_id) return True def _set_service_level(self): @@ -347,7 +351,7 @@ class SubscriptionManager(object): try: self._sub_man_cli(cmd) self.log.debug("Attached the following pools to your " - "system: %s" % (", ".join(pool_list)) + "system: %s", (", ".join(pool_list)) .replace('--pool=', '')) return True except util.ProcessExecutionError as e: @@ -355,18 +359,24 @@ class SubscriptionManager(object): "due to {1}".format(pool, e)) return False - def update_repos(self, erepos, drepos): + def update_repos(self): ''' Takes a list of yum repo ids that need to be disabled or enabled; then it verifies if they are already enabled or disabled and finally executes the action to disable or enable ''' - if (erepos is not None) and (not isinstance(erepos, list)): + erepos = self.enable_repo + drepos = self.disable_repo + if erepos is None: + erepos = [] + if drepos is None: + drepos = [] + if not isinstance(erepos, list): self.log_warn("Repo IDs must in the format of a list.") return False - if (drepos is not None) and (not isinstance(drepos, list)): + if not isinstance(drepos, list): self.log_warn("Repo IDs must in the format of a list.") return False @@ -399,14 +409,14 @@ class SubscriptionManager(object): for fail in enable_list_fail: # Check if the repo exists or not if fail in active_repos: - self.log.debug("Repo {0} is already enabled".format(fail)) + self.log.debug("Repo %s is already enabled", fail) else: self.log_warn("Repo {0} does not appear to " "exist".format(fail)) if len(disable_list_fail) > 0: for fail in disable_list_fail: - self.log.debug("Repo {0} not disabled " - "because it is not enabled".format(fail)) + self.log.debug("Repo %s not disabled " + "because it is not enabled", fail) cmd = ['repos'] if len(disable_list) > 0: @@ -422,10 +432,10 @@ class SubscriptionManager(object): return False if len(enable_list) > 0: - self.log.debug("Enabled the following repos: %s" % + self.log.debug("Enabled the following repos: %s", (", ".join(enable_list)).replace('--enable=', '')) if len(disable_list) > 0: - self.log.debug("Disabled the following repos: %s" % + self.log.debug("Disabled the following repos: %s", (", ".join(disable_list)).replace('--disable=', '')) return True diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index e9d5702a..22718108 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -2,6 +2,7 @@ """Tests for registering RHEL subscription via rh_subscription.""" +import copy import logging from cloudinit.config import cc_rh_subscription @@ -68,6 +69,20 @@ class GoodTests(TestCase): self.assertEqual(self.SM.log_success.call_count, 1) self.assertEqual(self.SM._sub_man_cli.call_count, 2) + @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_getRepos") + @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_sub_man_cli") + def test_update_repos_disable_with_none(self, m_sub_man_cli, m_get_repos): + cfg = copy.deepcopy(self.config) + m_get_repos.return_value = ([], ['repo1']) + m_sub_man_cli.return_value = (b'', b'') + cfg['rh_subscription'].update( + {'enable-repo': ['repo1'], 'disable-repo': None}) + mysm = cc_rh_subscription.SubscriptionManager(cfg) + self.assertEqual(True, mysm.update_repos()) + m_get_repos.assert_called_with() + self.assertEqual(m_sub_man_cli.call_args_list, + [mock.call(['repos', '--enable=repo1'])]) + def test_full_registration(self): ''' Registration with auto-attach, service-level, adding pools, -- cgit v1.2.3 From 22a14a6a6d45ae55d2c2307d7b097eef9863bb0c Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Wed, 8 Nov 2017 15:45:53 -0500 Subject: hosts: Fix openSUSE and SLES setup for /etc/hosts and clarify docs. The etc/hosts file is was not properly setup for openSUSE or SLES when manage_etc_hosts is set in the config file. Improve the doc to address the fact that the 'localhost' ip is distribution dependent (not always 127.0.0.1). LP: #1731022 --- cloudinit/config/cc_update_etc_hosts.py | 4 +- templates/hosts.opensuse.tmpl | 26 -------- templates/hosts.suse.tmpl | 10 +++- .../test_handler/test_handler_etc_hosts.py | 69 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 30 deletions(-) delete mode 100644 templates/hosts.opensuse.tmpl create mode 100644 tests/unittests/test_handler/test_handler_etc_hosts.py (limited to 'cloudinit') diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index b3947849..c96eede1 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -23,8 +23,8 @@ using the template located in ``/etc/cloud/templates/hosts.tmpl``. In the If ``manage_etc_hosts`` is set to ``localhost``, then cloud-init will not rewrite ``/etc/hosts`` entirely, but rather will ensure that a entry for the -fqdn with ip ``127.0.1.1`` is present in ``/etc/hosts`` (i.e. -``ping `` will ping ``127.0.1.1``). +fqdn with a distribution dependent ip is present in ``/etc/hosts`` (i.e. +``ping `` will ping ``127.0.0.1`` or ``127.0.1.1`` or other ip). .. note:: if ``manage_etc_hosts`` is set ``true`` or ``template``, the contents diff --git a/templates/hosts.opensuse.tmpl b/templates/hosts.opensuse.tmpl deleted file mode 100644 index 655da3f7..00000000 --- a/templates/hosts.opensuse.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -* - This file /etc/cloud/templates/hosts.opensuse.tmpl is only utilized - if enabled in cloud-config. Specifically, in order to enable it - you need to add the following to config: - manage_etc_hosts: True -*# -# Your system has configured 'manage_etc_hosts' as True. -# As a result, if you wish for changes to this file to persist -# then you will need to either -# a.) make changes to the master file in -# /etc/cloud/templates/hosts.opensuse.tmpl -# b.) change or remove the value of 'manage_etc_hosts' in -# /etc/cloud/cloud.cfg or cloud-config from user-data -# -# The following lines are desirable for IPv4 capable hosts -127.0.0.1 localhost - -# The following lines are desirable for IPv6 capable hosts -::1 localhost ipv6-localhost ipv6-loopback -fe00::0 ipv6-localnet - -ff00::0 ipv6-mcastprefix -ff02::1 ipv6-allnodes -ff02::2 ipv6-allrouters -ff02::3 ipv6-allhosts - diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl index b6082692..8e664dbf 100644 --- a/templates/hosts.suse.tmpl +++ b/templates/hosts.suse.tmpl @@ -13,12 +13,18 @@ you need to add the following to config: # /etc/cloud/cloud.cfg or cloud-config from user-data # # The following lines are desirable for IPv4 capable hosts -127.0.0.1 localhost +127.0.0.1 {{fqdn}} {{hostname}} +127.0.0.1 localhost.localdomain localhost +127.0.0.1 localhost4.localdomain4 localhost4 # The following lines are desirable for IPv6 capable hosts +::1 {{fqdn}} {{hostname}} +::1 localhost.localdomain localhost +::1 localhost6.localdomain6 localhost6 ::1 localhost ipv6-localhost ipv6-loopback -fe00::0 ipv6-localnet + +fe00::0 ipv6-localnet ff00::0 ipv6-mcastprefix ff02::1 ipv6-allnodes ff02::2 ipv6-allrouters diff --git a/tests/unittests/test_handler/test_handler_etc_hosts.py b/tests/unittests/test_handler/test_handler_etc_hosts.py new file mode 100644 index 00000000..ced05a8d --- /dev/null +++ b/tests/unittests/test_handler/test_handler_etc_hosts.py @@ -0,0 +1,69 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +from cloudinit.config import cc_update_etc_hosts + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from cloudinit.tests import helpers as t_help + +import logging +import os +import shutil + +LOG = logging.getLogger(__name__) + + +class TestHostsFile(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestHostsFile, self).setUp() + self.tmp = self.tmp_dir() + + def _fetch_distro(self, kind): + cls = distros.fetch(kind) + paths = helpers.Paths({}) + return cls(kind, {}, paths) + + def test_write_etc_hosts_suse_localhost(self): + cfg = { + 'manage_etc_hosts': 'localhost', + 'hostname': 'cloud-init.test.us' + } + os.makedirs('%s/etc/' % self.tmp) + hosts_content = '192.168.1.1 blah.blah.us blah\n' + fout = open('%s/etc/hosts' % self.tmp, 'w') + fout.write(hosts_content) + fout.close() + distro = self._fetch_distro('sles') + distro.hosts_fn = '%s/etc/hosts' % self.tmp + paths = helpers.Paths({}) + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + cc_update_etc_hosts.handle('test', cfg, cc, LOG, []) + contents = util.load_file('%s/etc/hosts' % self.tmp) + if '127.0.0.1\tcloud-init.test.us\tcloud-init' not in contents: + self.assertIsNone('No entry for 127.0.0.1 in etc/hosts') + if '192.168.1.1\tblah.blah.us\tblah' not in contents: + self.assertIsNone('Default etc/hosts content modified') + + def test_write_etc_hosts_suse_template(self): + cfg = { + 'manage_etc_hosts': 'template', + 'hostname': 'cloud-init.test.us' + } + shutil.copytree('templates', '%s/etc/cloud/templates' % self.tmp) + distro = self._fetch_distro('sles') + paths = helpers.Paths({}) + paths.template_tpl = '%s' % self.tmp + '/etc/cloud/templates/%s.tmpl' + ds = None + cc = cloud.Cloud(ds, paths, {}, distro, None) + self.patchUtils(self.tmp) + cc_update_etc_hosts.handle('test', cfg, cc, LOG, []) + contents = util.load_file('%s/etc/hosts' % self.tmp) + if '127.0.0.1 cloud-init.test.us cloud-init' not in contents: + self.assertIsNone('No entry for 127.0.0.1 in etc/hosts') + if '::1 cloud-init.test.us cloud-init' not in contents: + self.assertIsNone('No entry for 127.0.0.1 in etc/hosts') -- cgit v1.2.3 From e10ad2d7854b87024b5d051db50166125fce2279 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Thu, 6 Mar 2014 13:26:05 -0800 Subject: Catch UrlError when #include'ing URLs Without this the entire stage can fail, which will leave an instance unaccessible. Reviewed-by: Tom Kirchner Reviewed-by: Matt Nierzwicki Reviewed-by: Ben Cressey --- cloudinit/user_data.py | 28 ++++++++++++++++--------- tests/unittests/test_data.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 10 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 88cb7f84..e163c722 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -19,6 +19,7 @@ import six from cloudinit import handlers from cloudinit import log as logging +from cloudinit.url_helper import UrlError from cloudinit import util LOG = logging.getLogger(__name__) @@ -222,16 +223,23 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = util.read_file_or_url(include_url, - ssl_details=self.ssl_details) - if include_once_on and resp.ok(): - util.write_file(include_once_fn, resp.contents, mode=0o600) - if resp.ok(): - content = resp.contents - else: - LOG.warning(("Fetching from %s resulted in" - " a invalid http code of %s"), - include_url, resp.code) + try: + resp = util.read_file_or_url(include_url, + ssl_details=self.ssl_details) + if include_once_on and resp.ok(): + util.write_file(include_once_fn, resp.contents, + mode=0o600) + if resp.ok(): + content = resp.contents + else: + LOG.warning(("Fetching from %s resulted in" + " a invalid http code of %s"), + include_url, resp.code) + except UrlError as urle: + LOG.warning(urle) + except IOError as ioe: + LOG.warning("Fetching from %s resulted in %s", + include_url, ioe) if content is not None: new_msg = convert_string(content) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 6d621d26..275b16d2 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -18,6 +18,8 @@ from email.mime.application import MIMEApplication from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart +import httpretty + from cloudinit import handlers from cloudinit import helpers as c_helpers from cloudinit import log @@ -522,6 +524,54 @@ c: 4 self.assertEqual(cfg.get('password'), 'gocubs') self.assertEqual(cfg.get('locale'), 'chicago') + @httpretty.activate + @mock.patch('cloudinit.url_helper.time.sleep') + def test_include(self, mock_sleep): + """Test #include.""" + included_url = 'http://hostname/path' + included_data = '#cloud-config\nincluded: true\n' + httpretty.register_uri(httpretty.GET, included_url, included_data) + + blob = '#include\n%s\n' % included_url + + self.reRoot() + ci = stages.Init() + ci.datasource = FakeDataSource(blob) + ci.fetch() + ci.consume_data() + cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) + cc = util.load_yaml(cc_contents) + self.assertTrue(cc.get('included')) + + @httpretty.activate + @mock.patch('cloudinit.url_helper.time.sleep') + def test_include_bad_url(self, mock_sleep): + """Test #include with a bad URL.""" + bad_url = 'http://bad/forbidden' + bad_data = '#cloud-config\nbad: true\n' + httpretty.register_uri(httpretty.GET, bad_url, bad_data, status=403) + + included_url = 'http://hostname/path' + included_data = '#cloud-config\nincluded: true\n' + httpretty.register_uri(httpretty.GET, included_url, included_data) + + blob = '#include\n%s\n%s' % (bad_url, included_url) + + self.reRoot() + ci = stages.Init() + ci.datasource = FakeDataSource(blob) + log_file = self.capture_log(logging.WARNING) + ci.fetch() + ci.consume_data() + + self.assertIn("403 Client Error: Forbidden for url: %s" % bad_url, + log_file.getvalue()) + + cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) + cc = util.load_yaml(cc_contents) + self.assertIsNone(cc.get('bad')) + self.assertTrue(cc.get('included')) + class TestUDProcess(helpers.ResourceUsingTestCase): -- cgit v1.2.3 From 6ad23fe9b11f07e4404c8a1f2f1e9cba2640dceb Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 16 Nov 2017 20:47:02 -0700 Subject: centos: Provide the failed #include url in error messages On python 2.7 and earlier (CentOS 6 & 7), UrlErrors raised by requests do not report the url which failed. In such cases, append the url if not present in the error message. This fixes nightly CI failures at https://jenkins.ubuntu.com/server/view/cloud-init/. --- cloudinit/user_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'cloudinit') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index e163c722..cc55daf8 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -236,7 +236,12 @@ class UserDataProcessor(object): " a invalid http code of %s"), include_url, resp.code) except UrlError as urle: - LOG.warning(urle) + message = str(urle) + # Older versions of requests.exceptions.HTTPError may not + # include the errant url. Append it for clarity in logs. + if include_url not in message: + message += ' for url: {0}'.format(include_url) + LOG.warning(message) except IOError as ioe: LOG.warning("Fetching from %s resulted in %s", include_url, ioe) -- cgit v1.2.3 From d90318b21d0379e24337bcb92a0a90ebfa359c35 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 16 Nov 2017 20:58:45 -0700 Subject: ntp: fix configuration template rendering for openSUSE and SLES Add opensuse distro support to cc_ntp module. LP: #1726572 --- cloudinit/config/cc_ntp.py | 9 ++- templates/ntp.conf.opensuse.tmpl | 88 ++++++++++++++++++++++++ templates/ntp.conf.sles.tmpl | 12 ---- tests/unittests/test_handler/test_handler_ntp.py | 26 +++++++ 4 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 templates/ntp.conf.opensuse.tmpl (limited to 'cloudinit') diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index d43d060c..f50bcb35 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -23,7 +23,7 @@ frequency = PER_INSTANCE NTP_CONF = '/etc/ntp.conf' TIMESYNCD_CONF = '/etc/systemd/timesyncd.conf.d/cloud-init.conf' NR_POOL_SERVERS = 4 -distros = ['centos', 'debian', 'fedora', 'opensuse', 'ubuntu'] +distros = ['centos', 'debian', 'fedora', 'opensuse', 'sles', 'ubuntu'] # The schema definition for each cloud-config module is a strict contract for @@ -174,8 +174,13 @@ def rename_ntp_conf(config=None): def generate_server_names(distro): names = [] + pool_distro = distro + # For legal reasons x.pool.sles.ntp.org does not exist, + # use the opensuse pool + if distro == 'sles': + pool_distro = 'opensuse' for x in range(0, NR_POOL_SERVERS): - name = "%d.%s.pool.ntp.org" % (x, distro) + name = "%d.%s.pool.ntp.org" % (x, pool_distro) names.append(name) return names diff --git a/templates/ntp.conf.opensuse.tmpl b/templates/ntp.conf.opensuse.tmpl new file mode 100644 index 00000000..f3ab565f --- /dev/null +++ b/templates/ntp.conf.opensuse.tmpl @@ -0,0 +1,88 @@ +## template:jinja + +## +## Radio and modem clocks by convention have addresses in the +## form 127.127.t.u, where t is the clock type and u is a unit +## number in the range 0-3. +## +## Most of these clocks require support in the form of a +## serial port or special bus peripheral. The particular +## device is normally specified by adding a soft link +## /dev/device-u to the particular hardware device involved, +## where u correspond to the unit number above. +## +## Generic DCF77 clock on serial port (Conrad DCF77) +## Address: 127.127.8.u +## Serial Port: /dev/refclock-u +## +## (create soft link /dev/refclock-0 to the particular ttyS?) +## +# server 127.127.8.0 mode 5 prefer + +## +## Undisciplined Local Clock. This is a fake driver intended for backup +## and when no outside source of synchronized time is available. +## +# server 127.127.1.0 # local clock (LCL) +# fudge 127.127.1.0 stratum 10 # LCL is unsynchronized + +## +## Add external Servers using +## # rcntpd addserver +## The servers will only be added to the currently running instance, not +## to /etc/ntp.conf. +## +{% if pools %}# pools +{% endif %} +{% for pool in pools -%} +pool {{pool}} iburst +{% endfor %} +{%- if servers %}# servers +{% endif %} +{% for server in servers -%} +server {{server}} iburst +{% endfor %} + +# Access control configuration; see /usr/share/doc/packages/ntp/html/accopt.html for +# details. The web page +# might also be helpful. +# +# Note that "restrict" applies to both servers and clients, so a configuration +# that might be intended to block requests from certain clients could also end +# up blocking replies from your own upstream servers. + +# By default, exchange time with everybody, but don't allow configuration. +restrict -4 default notrap nomodify nopeer noquery +restrict -6 default notrap nomodify nopeer noquery + +# Local users may interrogate the ntp server more closely. +restrict 127.0.0.1 +restrict ::1 + +# Clients from this (example!) subnet have unlimited access, but only if +# cryptographically authenticated. +#restrict 192.168.123.0 mask 255.255.255.0 notrust + +## +## Miscellaneous stuff +## + +driftfile /var/lib/ntp/drift/ntp.drift # path for drift file + +logfile /var/log/ntp # alternate log file +# logconfig =syncstatus + sysevents +# logconfig =all + +# statsdir /tmp/ # directory for statistics files +# filegen peerstats file peerstats type day enable +# filegen loopstats file loopstats type day enable +# filegen clockstats file clockstats type day enable + +# +# Authentication stuff +# +keys /etc/ntp.keys # path for keys file +trustedkey 1 # define trusted keys +requestkey 1 # key (7) for accessing server variables +controlkey 1 # key (6) for accessing server variables + diff --git a/templates/ntp.conf.sles.tmpl b/templates/ntp.conf.sles.tmpl index 5c5fc4db..f3ab565f 100644 --- a/templates/ntp.conf.sles.tmpl +++ b/templates/ntp.conf.sles.tmpl @@ -1,17 +1,5 @@ ## template:jinja -################################################################################ -## /etc/ntp.conf -## -## Sample NTP configuration file. -## See package 'ntp-doc' for documentation, Mini-HOWTO and FAQ. -## Copyright (c) 1998 S.u.S.E. GmbH Fuerth, Germany. -## -## Author: Michael Andres, -## Michael Skibbe, -## -################################################################################ - ## ## Radio and modem clocks by convention have addresses in the ## form 127.127.t.u, where t is the clock type and u is a unit diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 3abe5786..28a8455d 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -430,5 +430,31 @@ class TestNtp(FilesystemMockingTestCase): "[Time]\nNTP=192.168.2.1 192.168.2.2 0.mypool.org \n", content.decode()) + def test_write_ntp_config_template_defaults_pools_empty_lists_sles(self): + """write_ntp_config_template defaults pools servers upon empty config. + + When both pools and servers are empty, default NR_POOL_SERVERS get + configured. + """ + distro = 'sles' + mycloud = self._get_cloud(distro) + ntp_conf = self.tmp_path('ntp.conf', self.new_root) # Doesn't exist + # Create ntp.conf.tmpl + with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: + stream.write(NTP_TEMPLATE) + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): + cc_ntp.write_ntp_config_template({}, mycloud, ntp_conf) + content = util.read_file_or_url('file://' + ntp_conf).contents + default_pools = [ + "{0}.opensuse.pool.ntp.org".format(x) + for x in range(0, cc_ntp.NR_POOL_SERVERS)] + self.assertEqual( + "servers []\npools {0}\n".format(default_pools), + content.decode()) + self.assertIn( + "Adding distro default ntp pool servers: {0}".format( + ",".join(default_pools)), + self.logs.getvalue()) + # vi: ts=4 expandtab -- cgit v1.2.3 From d3a0958c09c73a78fda6e922b749a1b98036e984 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 19 Nov 2017 16:27:06 -0700 Subject: EC2: Kill dhclient process used in sandbox dhclient. dhclient runs, obtains a address and then backgrounds itself. cloud-init did not take care to kill it after it was done with it. After it has run and created the leases, we can kill it. LP: #1732964 --- cloudinit/net/dhcp.py | 9 ++++++++- cloudinit/net/tests/test_dhcp.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 0cba7032..f3a412a9 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -8,6 +8,7 @@ import configobj import logging import os import re +import signal from cloudinit.net import find_fallback_nic, get_devicelist from cloudinit import temp_utils @@ -119,7 +120,13 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] util.subp(cmd, capture=True) - return parse_dhcp_lease_file(lease_file) + pid = None + try: + pid = int(util.load_file(pid_file).strip()) + return parse_dhcp_lease_file(lease_file) + finally: + if pid: + os.kill(pid, signal.SIGKILL) def networkd_parse_lease(content): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 1c1f504a..3d8e15c0 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -2,6 +2,7 @@ import mock import os +import signal from textwrap import dedent from cloudinit.net.dhcp import ( @@ -114,8 +115,9 @@ class TestDHCPDiscoveryClean(CiTestCase): self.assertEqual('eth9', call[0][1]) self.assertIn('/var/tmp/cloud-init/cloud-init-dhcp-', call[0][2]) + @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.util.subp') - def test_dhcp_discovery_run_in_sandbox(self, m_subp): + def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill): """dhcp_discovery brings up the interface and runs dhclient. It also returns the parsed dhcp.leases file generated in the sandbox. @@ -134,6 +136,10 @@ class TestDHCPDiscoveryClean(CiTestCase): """) lease_file = os.path.join(tmpdir, 'dhcp.leases') write_file(lease_file, lease_content) + pid_file = os.path.join(tmpdir, 'dhclient.pid') + my_pid = 1 + write_file(pid_file, "%d\n" % my_pid) + self.assertItemsEqual( [{'interface': 'eth9', 'fixed-address': '192.168.2.74', 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], @@ -149,6 +155,7 @@ class TestDHCPDiscoveryClean(CiTestCase): [os.path.join(tmpdir, 'dhclient'), '-1', '-v', '-lf', lease_file, '-pf', os.path.join(tmpdir, 'dhclient.pid'), 'eth9', '-sf', '/bin/true'], capture=True)]) + m_kill.assert_has_calls([mock.call(my_pid, signal.SIGKILL)]) class TestSystemdParseLeases(CiTestCase): -- cgit v1.2.3 From 281a82181716183d526e76f4e0415e0a6f680cbe Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 20 Nov 2017 15:56:40 -0500 Subject: EC2: Fix bug using fallback_nic and metadata when restoring from cache. If user upgraded to new cloud-init and attempted to run 'cloud-init init' without rebooting, cloud-init restores the datasource object from pickle. The older version pickled datasource object had no value for _network_config or fallback_nic. This caused the Ec2 datasource to attempt to reconfigure networking with a None fallback_nic. The pickled object also cached an older version of ec2 metadata which didn't contain network information. This branch does two things: - Add a fallback_interface property to DatasourceEC2 to support reading the old .fallback_nic attribute if it was set. New versions will call net.find_fallback_nic() if there has not been one found. - Re-crawl metadata if we are on Ec2 and don't have a 'network' key in metadata LP: #1732917 --- cloudinit/net/dhcp.py | 3 +- cloudinit/sources/DataSourceEc2.py | 44 +++++++++++++++++++++-------- tests/unittests/test_datasource/test_ec2.py | 33 ++++++++++++++++++++++ 3 files changed, 67 insertions(+), 13 deletions(-) (limited to 'cloudinit') diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index f3a412a9..d8624d82 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -42,8 +42,7 @@ def maybe_perform_dhcp_discovery(nic=None): if nic is None: nic = find_fallback_nic() if nic is None: - LOG.debug( - 'Skip dhcp_discovery: Unable to find fallback nic.') + LOG.debug('Skip dhcp_discovery: Unable to find fallback nic.') return {} elif nic not in get_devicelist(): LOG.debug( diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0ef22174..7bbbfb63 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -65,7 +65,7 @@ class DataSourceEc2(sources.DataSource): get_network_metadata = False # Track the discovered fallback nic for use in configuration generation. - fallback_nic = None + _fallback_interface = None def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -92,18 +92,17 @@ class DataSourceEc2(sources.DataSource): elif self.cloud_platform == Platforms.NO_EC2_METADATA: return False - self.fallback_nic = net.find_fallback_nic() if self.get_network_metadata: # Setup networking in init-local stage. if util.is_FreeBSD(): LOG.debug("FreeBSD doesn't support running dhclient with -sf") return False - dhcp_leases = dhcp.maybe_perform_dhcp_discovery(self.fallback_nic) + dhcp_leases = dhcp.maybe_perform_dhcp_discovery( + self.fallback_interface) if not dhcp_leases: # DataSourceEc2Local failed in init-local stage. DataSourceEc2 # will still run in init-network stage. return False dhcp_opts = dhcp_leases[-1] - self.fallback_nic = dhcp_opts.get('interface') net_params = {'interface': dhcp_opts.get('interface'), 'ip': dhcp_opts.get('fixed-address'), 'prefix_or_mask': dhcp_opts.get('subnet-mask'), @@ -301,21 +300,44 @@ class DataSourceEc2(sources.DataSource): return None result = None - net_md = self.metadata.get('network') + no_network_metadata_on_aws = bool( + 'network' not in self.metadata and + self.cloud_platform == Platforms.AWS) + if no_network_metadata_on_aws: + LOG.debug("Metadata 'network' not present:" + " Refreshing stale metadata from prior to upgrade.") + util.log_time( + logfunc=LOG.debug, msg='Re-crawl of metadata service', + func=self._crawl_metadata) + # Limit network configuration to only the primary/fallback nic - macs_to_nics = { - net.get_interface_mac(self.fallback_nic): self.fallback_nic} + iface = self.fallback_interface + macs_to_nics = {net.get_interface_mac(iface): iface} + net_md = self.metadata.get('network') if isinstance(net_md, dict): result = convert_ec2_metadata_network_config( - net_md, macs_to_nics=macs_to_nics, - fallback_nic=self.fallback_nic) + net_md, macs_to_nics=macs_to_nics, fallback_nic=iface) else: - LOG.warning("unexpected metadata 'network' key not valid: %s", - net_md) + LOG.warning("Metadata 'network' key not valid: %s.", net_md) self._network_config = result return self._network_config + @property + def fallback_interface(self): + if self._fallback_interface is None: + # fallback_nic was used at one point, so restored objects may + # have an attribute there. respect that if found. + _legacy_fbnic = getattr(self, 'fallback_nic', None) + if _legacy_fbnic: + self._fallback_interface = _legacy_fbnic + self.fallback_nic = None + else: + self._fallback_interface = net.find_fallback_nic() + if self._fallback_interface is None: + LOG.warning("Did not find a fallback interface on EC2.") + return self._fallback_interface + def _crawl_metadata(self): """Crawl metadata service when available. diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py index 6af699a6..ba328ee9 100644 --- a/tests/unittests/test_datasource/test_ec2.py +++ b/tests/unittests/test_datasource/test_ec2.py @@ -305,6 +305,39 @@ class TestEc2(test_helpers.HttprettyTestCase): ds._network_config = {'cached': 'data'} self.assertEqual({'cached': 'data'}, ds.network_config) + @httpretty.activate + @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') + def test_network_config_cached_property_refreshed_on_upgrade(self, m_dhcp): + """Refresh the network_config Ec2 cache if network key is absent. + + This catches an upgrade issue where obj.pkl contained stale metadata + which lacked newly required network key. + """ + old_metadata = copy.deepcopy(DEFAULT_METADATA) + old_metadata.pop('network') + ds = self._setup_ds( + platform_data=self.valid_platform_data, + sys_cfg={'datasource': {'Ec2': {'strict_id': True}}}, + md=old_metadata) + self.assertTrue(ds.get_data()) + # Provide new revision of metadata that contains network data + register_mock_metaserver( + 'http://169.254.169.254/2009-04-04/meta-data/', DEFAULT_METADATA) + mac1 = '06:17:04:d7:26:09' # Defined in DEFAULT_METADATA + get_interface_mac_path = ( + 'cloudinit.sources.DataSourceEc2.net.get_interface_mac') + ds.fallback_nic = 'eth9' + with mock.patch(get_interface_mac_path) as m_get_interface_mac: + m_get_interface_mac.return_value = mac1 + ds.network_config # Will re-crawl network metadata + self.assertIn('Re-crawl of metadata service', self.logs.getvalue()) + expected = {'version': 1, 'config': [ + {'mac_address': '06:17:04:d7:26:09', + 'name': 'eth9', + 'subnets': [{'type': 'dhcp4'}, {'type': 'dhcp6'}], + 'type': 'physical'}]} + self.assertEqual(expected, ds.network_config) + @httpretty.activate @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery') def test_valid_platform_with_strict_true(self, m_dhcp): -- cgit v1.2.3