From bf52085a1fa3529329a5c48097a12a6e9b93eb22 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 15:19:51 -0400 Subject: NoCloud: the local portion of NoCloud incorrectly claimed datasources The intent has always been for the local datasource (NoCloud) to require the provider of metadata to provide 'dsmode=local'. If that wasn't found, then the default 'dsmode' would be 'net', and the NoCloudNet datasource would then find the data. The bug here was that the default 'net' wasn't being set when data was found on a local source. --- ChangeLog | 1 + 1 file changed, 1 insertion(+) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 32a4f5d6..70ba9ae3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ (LP: #1422388) - readurl, read_file_or_url returns bytes, user must convert as necessary - SmartOS: use v2 metadata service (LP: #1436417) [Daniel Watkins] + - NoCloud: fix local datasource claiming found without explicit dsmode 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) -- cgit v1.2.3 From f4789639af50ea631cf75af526ad013b05670caa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 9 Apr 2015 11:54:01 -0400 Subject: systemd: use network-online instead of network.target (LP: #1440180) don't declare a Wants/Requires on network.target; this is a passive target that should only be pulled in by implementors of the networking service. The requirement for network needs to be expressed as a dependency on network-online.target. LP: #1440180 --- ChangeLog | 2 ++ systemd/cloud-config.service | 4 ++-- systemd/cloud-final.service | 4 ++-- systemd/cloud-init.service | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 6651b8eb..ff525337 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,8 @@ - SmartOS: use v2 metadata service (LP: #1436417) [Daniel Watkins] - NoCloud: fix local datasource claiming found without explicit dsmode - Snappy: add support for installing snappy packages and configuring. + - systemd: use network-online instead of network.target (LP: #1440180) + [Steve Langasek] 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/systemd/cloud-config.service b/systemd/cloud-config.service index ac25c776..f9f1996e 100644 --- a/systemd/cloud-config.service +++ b/systemd/cloud-config.service @@ -1,7 +1,7 @@ [Unit] Description=Apply the settings specified in cloud-config -After=network.target cloud-config.target syslog.target -Wants=network.target cloud-config.target +After=network-online.target cloud-config.target syslog.target +Wants=network-online.target cloud-config.target [Service] Type=oneshot diff --git a/systemd/cloud-final.service b/systemd/cloud-final.service index bbcdf30b..c023ad94 100644 --- a/systemd/cloud-final.service +++ b/systemd/cloud-final.service @@ -1,7 +1,7 @@ [Unit] Description=Execute cloud user/final scripts -After=network.target cloud-config.service syslog.target rc-local.service -Wants=network.target cloud-config.service +After=network-online.target cloud-config.service syslog.target rc-local.service +Wants=network-online.target cloud-config.service [Service] Type=oneshot diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index 398b90ea..48920283 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -1,8 +1,8 @@ [Unit] Description=Initial cloud-init job (metadata service crawler) -After=local-fs.target network.target cloud-init-local.service +After=local-fs.target network-online.target cloud-init-local.service Before=sshd.service sshd-keygen.service systemd-user-sessions.service -Requires=network.target +Requires=network-online.target Wants=local-fs.target cloud-init-local.service sshd.service sshd-keygen.service [Service] -- cgit v1.2.3 From d55770f91a6b6a0808ba3869da1e54f86e1ab3a2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 21 Apr 2015 11:53:00 -0700 Subject: Update changelog with previously merged branches --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index ff525337..c5ad7c60 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,8 @@ - Snappy: add support for installing snappy packages and configuring. - systemd: use network-online instead of network.target (LP: #1440180) [Steve Langasek] + - Add functionality to fixate the uid of a newly added user. + - Don't overwrite the hostname if the user has changed it after we set it. 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) -- cgit v1.2.3 From 151ece4efcd6d8f5051e86dff2bcd7d218e50ca2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 19 May 2015 08:21:34 -0700 Subject: EC2: be aware of eu-central-1 availability zone eu-central-1 means that 'central' is a direction to update the regular expression to understand. LP: #1456684 --- ChangeLog | 1 + cloudinit/distros/__init__.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 0b4175b7..ff06e989 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,7 @@ - Fix exception when running with no arguments on Python 3. [Daniel Watkins] - Centos: detect/expect use of systemd on centos 7. [Brian Rak] - Azure: remove dependency on walinux-agent [Daniel Watkins] + - EC2: know about eu-central-1 availability-zone (LP: #1456684) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 05721922..e0cce670 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -556,8 +556,12 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, if not mirror_info: mirror_info = {} - ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % - "north|northeast|east|southeast|south|southwest|west|northwest") + # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) + # the region is us-east-1. so region = az[0:-1] + directions_re = '|'.join([ + 'central', 'east', 'north', 'northeast', 'northwest', + 'south', 'southeast', 'southwest', 'west']) + ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re) subst = {} if availability_zone: -- cgit v1.2.3 From e6d4bfd6d3b24c1106ecd38e51355243e6558983 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 10 Jun 2015 17:42:12 +0100 Subject: Add ChangeLog entry for last merge. --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 30885329..cc12cb6e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,8 @@ - Doc: include information on user-data in OpenStack [Daniel Watkins] - Systemd: check for systemd using sd_booted symantics (LP: #1461201) [Lars Kellogg-Stedman] + - Add an rh_subscription module to handle registration of Red Hat instances. + [Brent Baude] 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) -- cgit v1.2.3 From 66c13ab5aca67ca3aa3d1536154989f98b85107a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 15 Jun 2015 17:20:51 -0400 Subject: apt_configure: fix importing of apt gpg keys under in python3 LP: #1463373 --- ChangeLog | 1 + cloudinit/config/cc_apt_configure.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index cc12cb6e..6261147e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,6 +48,7 @@ [Lars Kellogg-Stedman] - Add an rh_subscription module to handle registration of Red Hat instances. [Brent Baude] + - cc_apt_configure: fix importing keys under python3 (LP: #1463373) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2c51d116..9e9e9e26 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -109,7 +109,7 @@ def handle(name, cfg, cloud, log, _args): # get gpg keyid from keyserver def getkeybyid(keyid, keyserver): - with util.ExtendedTemporaryFile(suffix='.sh') as fh: + with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh: fh.write(EXPORT_GPG_KEYID) fh.flush() cmd = ['/bin/sh', fh.name, keyid, keyserver] -- cgit v1.2.3 From 6e06afffed8614cb143e3a13bab5aa382ccbbce9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 16 Jun 2015 11:18:33 -0400 Subject: growpart: fix specification of 'devices' list. given config: {'growpart': {'devices': ["/"]}} the 'devices' was ignored, it was incorrectly read from the top level non-namespaced location. LP: #1465436 --- ChangeLog | 1 + cloudinit/config/cc_growpart.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 6261147e..47b8dec2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -49,6 +49,7 @@ - Add an rh_subscription module to handle registration of Red Hat instances. [Brent Baude] - cc_apt_configure: fix importing keys under python3 (LP: #1463373) + - cc_growpart: fix specification of 'devices' list (LP: #1465436) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index f52c41f0..859d69f1 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -276,7 +276,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("use ignore_growroot_disabled to ignore") return - devices = util.get_cfg_option_list(cfg, "devices", ["/"]) + devices = util.get_cfg_option_list(mycfg, "devices", ["/"]) if not len(devices): log.debug("growpart: empty device list") return -- cgit v1.2.3 From 3a59cf0077f01b82a8b464568bcdd94ce2112b1d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Jul 2015 20:42:56 -0400 Subject: _read_dmi_syspath: fix bad log message causing unintended exception --- ChangeLog | 1 + cloudinit/util.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 661e968b..bef5f77d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -56,6 +56,7 @@ - distro mirrors: provide datasource to mirror selection code to support GCE regional mirrors. (LP: #1470890) - add udev rules that identify ephemeral device on Azure (LP: #1411582) + - _read_dmi_syspath: fix bad log message causing unintended exception 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/util.py b/cloudinit/util.py index db4e02b8..02ba654a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2145,7 +2145,7 @@ def _read_dmi_syspath(key): return key_data.strip() except Exception as e: - logexc(LOG, "failed read of %s", dmi_key_path, e) + logexc(LOG, "failed read of %s", dmi_key_path) return None -- cgit v1.2.3 From f2f766015a0e875605a24703c5f0e18751650fe8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 30 Jul 2015 22:06:31 -0400 Subject: status_wrapper in main: fix use of print_exc when handling exception --- ChangeLog | 1 + bin/cloud-init | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 8a1c77bc..d885b9f9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -58,6 +58,7 @@ - add udev rules that identify ephemeral device on Azure (LP: #1411582) - _read_dmi_syspath: fix bad log message causing unintended exception - rsyslog: add additional configuration mode (LP: #1478103) + - status_wrapper in main: fix use of print_exc when handling exception 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/bin/cloud-init b/bin/cloud-init index 1d3e7ee3..d50ac694 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -506,7 +506,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): except Exception as e: util.logexc(LOG, "failed of stage %s", mode) - print_exc("failed run of stage %s", mode) + print_exc("failed run of stage %s" mode) v1[mode]['errors'] = [str(e)] v1[mode]['finished'] = time.time() -- cgit v1.2.3 From 3c39c3f7638245e9581a2e1f4faae2dc2680f0c7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 Sep 2015 14:26:30 -0400 Subject: NoCloud: fix consumption of vendor-data the content of vendordata was was being assigned to vendordata, rather than vendordata_raw. The result was that it is not processed for includes or part handlers or other things as it is in other datasources. LP: #1493453 --- ChangeLog | 1 + cloudinit/sources/DataSourceNoCloud.py | 2 +- tests/unittests/test_datasource/test_nocloud.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 7869ab7e..6fb70696 100644 --- a/ChangeLog +++ b/ChangeLog @@ -60,6 +60,7 @@ - rsyslog: add additional configuration mode (LP: #1478103) - status_wrapper in main: fix use of print_exc when handling exception - reporting: add reporting module for web hook or logging of events. + - NoCloud: fix consumption of vendordata (LP: #1493453) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 6a861af3..4dffe6e6 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -190,7 +190,7 @@ class DataSourceNoCloud(sources.DataSource): self.seed = ",".join(found) self.metadata = mydata['meta-data'] self.userdata_raw = mydata['user-data'] - self.vendordata = mydata['vendor-data'] + self.vendordata_raw = mydata['vendor-data'] return True LOG.debug("%s: not claiming datasource, dsmode=%s", self, diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 85b4c25a..2d5fc37c 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -121,7 +121,7 @@ class TestNoCloudDataSource(TestCase): ret = dsrc.get_data() self.assertEqual(dsrc.userdata_raw, ud) self.assertEqual(dsrc.metadata, md) - self.assertEqual(dsrc.vendordata, vd) + self.assertEqual(dsrc.vendordata_raw, vd) self.assertTrue(ret) def test_nocloud_no_vendordata(self): -- cgit v1.2.3 From ba3e59cbb5ae58a2267fcbcd23eecaaa26f2c396 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 Sep 2015 16:53:59 -0400 Subject: power_state: support 'condition' argument if 'condition' is provided to config in power_state, then consult it before powering off. This allows the user to shut down only if a condition is met, and leave the system in a debuggable state otherwise. An example is as simple as: power_state: mode: poweroff condition: ['sh', '-c', '[ -f /disable-poweroff ]'] --- ChangeLog | 1 + cloudinit/config/cc_power_state_change.py | 57 +++++++++++++++++++--- doc/examples/cloud-config-power-state.txt | 9 ++++ .../test_handler/test_handler_power_state.py | 48 ++++++++++++++++-- 4 files changed, 105 insertions(+), 10 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 6fb70696..bbb7e990 100644 --- a/ChangeLog +++ b/ChangeLog @@ -61,6 +61,7 @@ - status_wrapper in main: fix use of print_exc when handling exception - reporting: add reporting module for web hook or logging of events. - NoCloud: fix consumption of vendordata (LP: #1493453) + - power_state_change: support 'condition' to disable or enable poweroff 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 09d37371..7d9567e3 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,6 +22,7 @@ from cloudinit import util import errno import os import re +import six import subprocess import time @@ -48,10 +49,40 @@ def givecmdline(pid): return None +def check_condition(cond, log=None): + if isinstance(cond, bool): + if log: + log.debug("Static Condition: %s" % cond) + return cond + + pre = "check_condition command (%s): " % cond + try: + proc = subprocess.Popen(cond, shell=not isinstance(cond, list)) + proc.communicate() + ret = proc.returncode + if ret == 0: + if log: + log.debug(pre + "exited 0. condition met.") + return True + elif ret == 1: + if log: + log.debug(pre + "exited 1. condition not met.") + return False + else: + if log: + log.warn(pre + "unexpected exit %s. " % ret + + "do not apply change.") + return False + except Exception as e: + if log: + log.warn(pre + "Unexpected error: %s" % e) + return False + + def handle(_name, cfg, _cloud, log, _args): try: - (args, timeout) = load_power_state(cfg) + (args, timeout, condition) = load_power_state(cfg) if args is None: log.debug("no power_state provided. doing nothing") return @@ -59,6 +90,10 @@ def handle(_name, cfg, _cloud, log, _args): log.warn("%s Not performing power state change!" % str(e)) return + if condition is False: + log.debug("Condition was false. Will not perform state change.") + return + mypid = os.getpid() cmdline = givecmdline(mypid) @@ -70,8 +105,8 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, execmd, - [args, devnull_fp]) + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + condition, execmd, [args, devnull_fp]) def load_power_state(cfg): @@ -80,7 +115,7 @@ def load_power_state(cfg): pstate = cfg.get('power_state') if pstate is None: - return (None, None) + return (None, None, None) if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") @@ -115,7 +150,10 @@ def load_power_state(cfg): raise ValueError("failed to convert timeout '%s' to float." % pstate['timeout']) - return (args, timeout) + condition = pstate.get("condition", True) + if not isinstance(condition, six.string_types + (list, bool)): + raise TypeError("condition type %s invalid. must be list, bool, str") + return (args, timeout, condition) def doexit(sysexit): @@ -133,7 +171,7 @@ def execmd(exe_args, output=None, data_in=None): doexit(ret) -def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args): +def run_after_pid_gone(pid, pidcmdline, timeout, log, condition, func, args): # wait until pid, with /proc/pid/cmdline contents of pidcmdline # is no longer alive. After it is gone, or timeout has passed # execute func(args) @@ -175,4 +213,11 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args): if log: log.debug(msg) + + try: + if not check_condition(condition, log): + return + except Exception as e: + fatal("Unexpected Exception when checking condition: %s" % e) + func(*args) diff --git a/doc/examples/cloud-config-power-state.txt b/doc/examples/cloud-config-power-state.txt index 8df14366..b470153d 100644 --- a/doc/examples/cloud-config-power-state.txt +++ b/doc/examples/cloud-config-power-state.txt @@ -23,9 +23,18 @@ # message: provided as the message argument to 'shutdown'. default is none. # timeout: the amount of time to give the cloud-init process to finish # before executing shutdown. +# condition: apply state change only if condition is met. +# May be boolean True (always met), or False (never met), +# or a command string or list to be executed. +# command's exit code indicates: +# 0: condition met +# 1: condition not met +# other exit codes will result in 'not met', but are reserved +# for future use. # power_state: delay: "+30" mode: poweroff message: Bye Bye timeout: 30 + condition: True diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 2f86b8f8..5687b10d 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -1,6 +1,9 @@ +import sys + from cloudinit.config import cc_power_state_change as psc from .. import helpers as t_help +from ..helpers import mock class TestLoadPowerState(t_help.TestCase): @@ -9,12 +12,12 @@ class TestLoadPowerState(t_help.TestCase): def test_no_config(self): # completely empty config should mean do nothing - (cmd, _timeout) = psc.load_power_state({}) + (cmd, _timeout, _condition) = psc.load_power_state({}) self.assertEqual(cmd, None) def test_irrelevant_config(self): # no power_state field in config should return None for cmd - (cmd, _timeout) = psc.load_power_state({'foo': 'bar'}) + (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}) self.assertEqual(cmd, None) def test_invalid_mode(self): @@ -53,23 +56,60 @@ class TestLoadPowerState(t_help.TestCase): def test_no_message(self): # if message is not present, then no argument should be passed for it cfg = {'power_state': {'mode': 'poweroff'}} - (cmd, _timeout) = psc.load_power_state(cfg) + (cmd, _timeout, _condition) = psc.load_power_state(cfg) self.assertNotIn("", cmd) check_lps_ret(psc.load_power_state(cfg)) self.assertTrue(len(cmd) == 3) + def test_condition_null_raises(self): + cfg = {'power_state': {'mode': 'poweroff', 'condition': None}} + self.assertRaises(TypeError, psc.load_power_state, cfg) + + def test_condition_default_is_true(self): + cfg = {'power_state': {'mode': 'poweroff'}} + _cmd, _timeout, cond = psc.load_power_state(cfg) + self.assertEqual(cond, True) + + +class TestCheckCondition(t_help.TestCase): + def cmd_with_exit(self, rc): + return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc]) + + def test_true_is_true(self): + self.assertEqual(psc.check_condition(True), True) + + def test_false_is_false(self): + self.assertEqual(psc.check_condition(False), False) + + def test_cmd_exit_zero_true(self): + self.assertEqual(psc.check_condition(self.cmd_with_exit(0)), True) + + def test_cmd_exit_one_false(self): + self.assertEqual(psc.check_condition(self.cmd_with_exit(1)), False) + + def test_cmd_exit_nonzero_warns(self): + mocklog = mock.Mock() + self.assertEqual( + psc.check_condition(self.cmd_with_exit(2), mocklog), False) + self.assertEqual(mocklog.warn.call_count, 1) + + def check_lps_ret(psc_return, mode=None): - if len(psc_return) != 2: + if len(psc_return) != 3: raise TypeError("length returned = %d" % len(psc_return)) errs = [] cmd = psc_return[0] timeout = psc_return[1] + condition = psc_return[2] if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") + if 'condition' is None: + errs.append("condition was not returned") + if mode is not None: opt = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'}[mode] if opt not in psc_return[0]: -- cgit v1.2.3 From 86bd318e41b0bec10765d0498a125de062afe1f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Oct 2015 12:39:23 -0400 Subject: support configuring and installing the Ubuntu fan driver #cloud-config fan: config: | # fan 240 10.0.0.0/8 eth0/16 dhcp 10.0.0.0/8 eth1/16 dhcp off # fan 241 241.0.0.0/8 eth0/16 dhcp config_path: /etc/network/fan LP: #1504604 --- ChangeLog | 1 + cloudinit/config/cc_fan.py | 101 +++++++++++++++++++++++++++++++++++++++++++++ config/cloud.cfg | 1 + 3 files changed, 103 insertions(+) create mode 100644 cloudinit/config/cc_fan.py (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index bbb7e990..b7a66aa1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -62,6 +62,7 @@ - reporting: add reporting module for web hook or logging of events. - NoCloud: fix consumption of vendordata (LP: #1493453) - power_state_change: support 'condition' to disable or enable poweroff + - ubuntu fan: support for config and installing of ubuntu fan (LP: #1504604) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py new file mode 100644 index 00000000..39e3850e --- /dev/null +++ b/cloudinit/config/cc_fan.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +fan module allows configuration of Ubuntu Fan + https://wiki.ubuntu.com/FanNetworking + +Example config: + #cloud-config + fan: + config: | + # fan 240 + 10.0.0.0/8 eth0/16 dhcp + 10.0.0.0/8 eth1/16 dhcp off + # fan 241 + 241.0.0.0/8 eth0/16 dhcp + config_path: /etc/network/fan + +If cloud-init sees a 'fan' entry in cloud-config it will + a.) write 'config_path' with the contents + b.) install the package 'ubuntu-fan' if it is not installed + c.) ensure the service is started (or restarted if was previously running) +""" + +from cloudinit import log as logging +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE + +BUILTIN_CFG = { + 'config': None, + 'config_path': '/etc/network/fan', +} + + +def stop_update_start(service, config_file, content, systemd=False): + if systemd: + cmds = {'stop': ['systemctl', 'stop', service], + 'start': ['systemctl', 'start', service], + 'enable': ['systemctl', 'enable', service]} + else: + cmds = {'stop': ['service', 'stop'], + 'start': ['service', 'start']} + + def run(cmd, msg): + try: + return util.subp(cmd, capture=True) + except util.ProcessExecutionError as e: + LOG.warn("failed: %s (%s): %s", service, cmd, e) + return False + + stop_failed = not run(cmds['stop'], msg='stop %s' % service) + if not content.endswith('\n'): + content += '\n' + util.write_file(config_file, content, omode="w") + + ret = run(cmds['start'], msg='start %s' % service) + if ret and stop_failed: + LOG.warn("success: %s started", service) + + if 'enable' in cmds: + ret = run(cmds['enable'], msg='enable %s' % service) + + return ret + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('fan') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) + + if not mycfg.get('config'): + LOG.debug("%s: no 'fan' config entry. disabling", name) + return + + util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") + distro = cloud.distro + if not util.which('fanctl'): + distro.install_packages(['ubuntu-fan']) + + stop_update_start( + service='ubuntu-fan', config_file=mycfg.get('config_path'), + content=mycfg.get('config'), systemd=distro.uses_systemd()) diff --git a/config/cloud.cfg b/config/cloud.cfg index 2b27f379..74794ab0 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -53,6 +53,7 @@ cloud_config_modules: - apt-pipelining - apt-configure - package-update-upgrade-install + - fan - landscape - timezone - puppet -- cgit v1.2.3 From ee2e4800d6519f7022f5f1d9f38820395b005be2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 3 Nov 2015 20:48:19 -0500 Subject: Ubuntu templates: make sources.list consistent with ISO installs. This modifies a designed decision to have trimmer sources.list than one would find on a server iso install. Specifically, here we now enable: restricted and multiverse and backports. LP: #1177432 --- ChangeLog | 2 ++ templates/sources.list.ubuntu.tmpl | 43 +++++++++++++++----------------------- 2 files changed, 19 insertions(+), 26 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 5fbfaccc..ea25cb69 100644 --- a/ChangeLog +++ b/ChangeLog @@ -65,6 +65,8 @@ - ubuntu fan: support for config and installing of ubuntu fan (LP: #1504604) - Azure: support extracting SSH key values from ovf-env.xml (LP: #1506244) - AltCloud: fix call to udevadm settle (LP: #1507526) + - Ubuntu templates: modify sources.list template to provide same sources + as install from server or desktop ISO. (LP: #1177432) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/templates/sources.list.ubuntu.tmpl b/templates/sources.list.ubuntu.tmpl index 3fec51d3..59ae089b 100644 --- a/templates/sources.list.ubuntu.tmpl +++ b/templates/sources.list.ubuntu.tmpl @@ -9,13 +9,13 @@ # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. -deb {{mirror}} {{codename}} main -deb-src {{mirror}} {{codename}} main +deb {{mirror}} {{codename}} main restricted +deb-src {{mirror}} {{codename}} main restricted ## Major bug fix updates produced after the final release of the ## distribution. -deb {{mirror}} {{codename}}-updates main -deb-src {{mirror}} {{codename}}-updates main +deb {{mirror}} {{codename}}-updates main restricted +deb-src {{mirror}} {{codename}}-updates main restricted ## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu ## team. Also, please note that software in universe WILL NOT receive any @@ -26,31 +26,29 @@ deb {{mirror}} {{codename}}-updates universe deb-src {{mirror}} {{codename}}-updates universe ## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -## team, and may not be under a free licence. Please satisfy yourself as to +## team, and may not be under a free licence. Please satisfy yourself as to ## your rights to use the software. Also, please note that software in ## multiverse WILL NOT receive any review or updates from the Ubuntu ## security team. -# deb {{mirror}} {{codename}} multiverse -# deb-src {{mirror}} {{codename}} multiverse -# deb {{mirror}} {{codename}}-updates multiverse -# deb-src {{mirror}} {{codename}}-updates multiverse +deb {{mirror}} {{codename}} multiverse +deb-src {{mirror}} {{codename}} multiverse +deb {{mirror}} {{codename}}-updates multiverse +deb-src {{mirror}} {{codename}}-updates multiverse ## N.B. software from this repository may not have been tested as ## extensively as that contained in the main release, although it includes ## newer versions of some applications which may provide useful features. ## Also, please note that software in backports WILL NOT receive any review ## or updates from the Ubuntu security team. -deb {{mirror}} {{codename}}-backports main -deb-src {{mirror}} {{codename}}-backports main -deb {{mirror}} {{codename}}-backports universe -deb-src {{mirror}} {{codename}}-backports universe +#deb {{mirror}} {{codename}}-backports main restricted universe multiverse +#deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse -## Uncomment the following two to add software from the 'backports' -## multiverse and restricted repositories. -# deb {{mirror}} {{codename}}-backports multiverse -# deb-src {{mirror}} {{codename}}-backports multiverse -# deb {{mirror}} {{codename}}-backports restricted -# deb-src {{mirror}} {{codename}}-backports restricted +deb {{security}} {{codename}}-security main restricted +deb-src {{security}} {{codename}}-security main restricted +deb {{security}} {{codename}}-security universe +deb-src {{security}} {{codename}}-security universe +deb {{security}} {{codename}}-security multiverse +deb-src {{security}} {{codename}}-security multiverse ## Uncomment the following two lines to add software from Canonical's ## 'partner' repository. @@ -58,10 +56,3 @@ deb-src {{mirror}} {{codename}}-backports universe ## respective vendors as a service to Ubuntu users. # deb http://archive.canonical.com/ubuntu {{codename}} partner # deb-src http://archive.canonical.com/ubuntu {{codename}} partner - -deb {{security}} {{codename}}-security main -deb-src {{security}} {{codename}}-security main -deb {{security}} {{codename}}-security universe -deb-src {{security}} {{codename}}-security universe -# deb {{security}} {{codename}}-security multiverse -# deb-src {{security}} {{codename}}-security multiverse -- cgit v1.2.3 From f512c0126aac5c8708065c6e8aa5510f83574657 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 30 Nov 2015 15:33:28 -0500 Subject: systemd/power_state: fix power_state when cloud-final exited failure if a runcmd exited failure, then power_state would not work. This was because systemd was killing off subprocesses, and cloud-init implemented power off with a subprocess that waited for the parent to exit. LP: #1449318 --- ChangeLog | 2 ++ systemd/cloud-final.service | 1 + 2 files changed, 3 insertions(+) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index f41412fa..0ba16492 100644 --- a/ChangeLog +++ b/ChangeLog @@ -69,6 +69,8 @@ as install from server or desktop ISO. (LP: #1177432) - cc_mounts: use 'nofail' if system uses systemd. (LP: #1514485) - Azure: get instance id from dmi instead of SharedConfig (LP: #1506187) + - systemd/power_state: fix power_state to work even if cloud-final + exited non-zero (LP: #1449318) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/systemd/cloud-final.service b/systemd/cloud-final.service index c023ad94..bcbdd36f 100644 --- a/systemd/cloud-final.service +++ b/systemd/cloud-final.service @@ -8,6 +8,7 @@ Type=oneshot ExecStart=/usr/bin/cloud-init modules --mode=final RemainAfterExit=yes TimeoutSec=0 +KillMode=process # Output needs to appear in instance console output StandardOutput=journal+console -- cgit v1.2.3 From ee40614b0a34a110265493c176c64db823aa34b3 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Wed, 3 Feb 2016 22:21:40 -0600 Subject: lxd: add support for setting up lxd using 'lxd init' If lxd key is present in cfg, then run 'lxd init' with values from the 'init' entry in lxd configuration as flags. --- ChangeLog | 1 + cloudinit/config/cc_lxd.py | 50 +++++++++++++++++++ config/cloud.cfg | 1 + tests/unittests/test_handler/test_handler_lxd.py | 62 ++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 cloudinit/config/cc_lxd.py create mode 100644 tests/unittests/test_handler/test_handler_lxd.py (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 0ba16492..9fbc920d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ - Azure: get instance id from dmi instead of SharedConfig (LP: #1506187) - systemd/power_state: fix power_state to work even if cloud-final exited non-zero (LP: #1449318) + - lxd: add support for setting up lxd using 'lxd init' 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py new file mode 100644 index 00000000..0db8356b --- /dev/null +++ b/cloudinit/config/cc_lxd.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +This module initializes lxd using 'lxd init' + +Example config: + #cloud-config + lxd: + init: + network_address: + network_port: + storage_backend: + storage_create_device: + storage_create_loop: + storage_pool: + trust_password: +""" + +from cloudinit import util + + +def handle(name, cfg, cloud, log, args): + if not cfg.get('lxd') and cfg['lxd'].get('init'): + log.debug("Skipping module named %s, not present or disabled by cfg") + return + lxd_conf = cfg['lxd']['init'] + keys = ('network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', 'storage_pool', + 'trust_password') + cmd = ['lxd', 'init', '--auto'] + for k in keys: + if lxd_conf.get(k): + cmd.extend(["--%s" % k.replace('_', '-'), lxd_conf[k]]) + util.subp(cmd) diff --git a/config/cloud.cfg b/config/cloud.cfg index 74794ab0..795df19f 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -56,6 +56,7 @@ cloud_config_modules: - fan - landscape - timezone + - lxd - puppet - chef - salt-minion diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py new file mode 100644 index 00000000..89863d52 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -0,0 +1,62 @@ +from cloudinit.config import cc_lxd +from cloudinit import (util, distros, helpers, cloud) +from cloudinit.sources import DataSourceNoCloud +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestLxd(t_help.TestCase): + def setUp(self): + super(TestLxd, self).setUp() + self.unapply = [] + apply_patches([(util, 'subp', self._mock_subp)]) + self.subp_called = [] + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + + def _mock_subp(self, *args, **kwargs): + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + return + + def _get_cloud(self, distro): + cls = distros.fetch(distro) + paths = helpers.Paths({}) + d = cls(distro, {}, paths) + ds = DataSourceNoCloud.DataSourceNoCloud({}, d, paths) + cc = cloud.Cloud(ds, paths, {}, d, None) + return cc + + def test_lxd_init(self): + cfg = { + 'lxd': { + 'init': { + 'network_address': '0.0.0.0', + 'storage_backend': 'zfs', + 'storage_pool': 'poolname', + } + } + } + cc = self._get_cloud('ubuntu') + cc_lxd.handle('cc_lxd', cfg, cc, LOG, []) + + self.assertEqual( + self.subp_called[0].get('args'), + ['lxd', 'init', '--auto', '--network-address', '0.0.0.0', + '--storage-backend', 'zfs', '--storage-pool', 'poolname']) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret -- cgit v1.2.3 From 290afe72d53b5e38c3781934e23a676a3c1986e5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 1 Mar 2016 12:30:31 -0500 Subject: timezone: use a symlink when updating /etc/localtime Unless /etc/localtime is an existing file and not a symlink, then we will symlink instead of copying the tz_file to /etc/localtime. The copy was due to an old bug in Ubuntu, symlink should be preferred. LP: #1543025 --- ChangeLog | 2 ++ cloudinit/distros/__init__.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index b31148ac..2f1f9f87 100644 --- a/ChangeLog +++ b/ChangeLog @@ -81,6 +81,8 @@ - lxd: add support for setting up lxd using 'lxd init' (LP: #1522879) - Add Image Customization Parser for VMware vSphere Hypervisor Support. [Sankar Tanguturi] + - timezone: use a symlink rather than copy for /etc/localtime + unless it is already a file (LP: #1543025). 0.7.6: - open 0.7.6 diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 71884b32..8167c594 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -897,5 +897,9 @@ def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", util.write_file(tz_conf, str(tz).rstrip() + "\n") # This ensures that the correct tz will be used for the system if tz_local and tz_file: - util.copy(tz_file, tz_local) + # use a symlink if there exists a symlink or tz_local is not present + if os.path.islink(tz_local) or not os.path.exists(tz_local): + os.symlink(tz_file, tz_local) + else: + util.copy(tz_file, tz_local) return -- cgit v1.2.3 From 6e31038b9cccbcb4a33693060b96fc4f71d86789 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 7 Mar 2016 21:31:25 -0500 Subject: No longer run pollinate by default in seed_random The user can still choose to run pollinate here to seed their random data. And in an environment with network datasource, that would be expected to work. However, we do not want to run it any more from cloud-init because a.) pollinate's own init system jobs should get it ran before ssh, which is the primary purpose of wanting cloud-init to run it. b.) with a local datasource, there is no network guarantee when init_modules run, so pollinate -q would often cause issues then. c.) cloud-init would run pollinate and log the failure causing many cloud-init specific failures that it could do nothing about. LP: #1554152 --- ChangeLog | 1 + cloudinit/config/cc_seed_random.py | 2 +- tests/unittests/test_handler/test_handler_seed_random.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index a80a5d5f..6da276b5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -85,6 +85,7 @@ unless it is already a file (LP: #1543025). - Enable password changing via a hashed string [Alex Sirbu] - Added BigStep datasource [Alex Sirbu] + - No longer run pollinate in seed_random (LP: #1554152) 0.7.6: - open 0.7.6 diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 3288a853..1b011216 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -83,7 +83,7 @@ def handle(name, cfg, cloud, log, _args): len(seed_data), seed_path) util.append_file(seed_path, seed_data) - command = mycfg.get('command', ['pollinate', '-q']) + command = mycfg.get('command', None) req = mycfg.get('command_required', False) try: env = os.environ.copy() diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 34d11f21..98bc9b81 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -170,28 +170,30 @@ class TestRandomSeed(t_help.TestCase): contents = util.load_file(self._seed_file) self.assertEquals('tiny-tim-was-here-so-was-josh', contents) - def test_seed_command_not_provided_pollinate_available(self): + def test_seed_command_provided_and_available(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {'pollinate': '/usr/bin/pollinate'} - cc_seed_random.handle('test', {}, c, LOG, []) + cfg = {'random_seed': {'command': ['pollinate', '-q']}} + cc_seed_random.handle('test', cfg, c, LOG, []) subp_args = [f['args'] for f in self.subp_called] self.assertIn(['pollinate', '-q'], subp_args) - def test_seed_command_not_provided_pollinate_not_available(self): + def test_seed_command_not_provided(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {} cc_seed_random.handle('test', {}, c, LOG, []) # subp should not have been called as which would say not available - self.assertEquals(self.subp_called, list()) + self.assertFalse(self.subp_called) def test_unavailable_seed_command_and_required_raises_error(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {} + cfg = {'random_seed': {'command': ['THIS_NO_COMMAND'], + 'command_required': True}} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, - c, LOG, []) + 'test', cfg, c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) -- cgit v1.2.3 From b839ad32b9bf4541583ecbe68a0bd5dd9f12345a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 10 Mar 2016 12:32:46 -0500 Subject: dmi data: fix failure of reading dmi data for unset dmi values it is not uncommon to find dmi data in /sys full of 'ff'. utf-8 decoding of those would fail, causing warning and stacktrace. Return '.' instead of \xff. This maps to what dmidecode would return $ dmidecode --string system-product-name ................................. --- ChangeLog | 1 + cloudinit/util.py | 13 ++++++++++--- tests/unittests/test_util.py | 9 +++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index da1ca9ee..ebaacf6a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -88,6 +88,7 @@ - No longer run pollinate in seed_random (LP: #1554152) - groups: add defalt user to 'lxd' group. Create groups listed for a user if they do not exist. (LP: #1539317) + - dmi data: fix failure of reading dmi data for unset dmi values 0.7.6: - open 0.7.6 diff --git a/cloudinit/util.py b/cloudinit/util.py index e7407ea4..1d50edc9 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2140,13 +2140,20 @@ def _read_dmi_syspath(key): LOG.debug("did not find %s", dmi_key_path) return None - key_data = load_file(dmi_key_path) + key_data = load_file(dmi_key_path, decode=False) if not key_data: LOG.debug("%s did not return any data", dmi_key_path) return None - LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) - return key_data.strip() + # in the event that this is all \xff and a carriage return + # then return '.' in its place. + if key_data == b'\xff' * (len(key_data) - 1) + b'\n': + key_data = b'.' * (len(key_data) - 1) + b'\n' + + str_data = key_data.decode('utf8').strip() + + LOG.debug("dmi data %s returned %s", dmi_key_path, str_data) + return str_data except Exception: logexc(LOG, "failed read of %s", dmi_key_path) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 95990165..542e4075 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -385,6 +385,15 @@ class TestReadDMIData(helpers.FilesystemMockingTestCase): self.patch_mapping({}) self.assertEqual(None, util.read_dmi_data('expect-fail')) + def test_dots_returned_instead_of_foxfox(self): + my_len = 32 + dmi_value = b'\xff' * my_len + b'\n' + expected = '.' * my_len + dmi_key = 'system-product-name' + sysfs_key = 'product_name' + self._create_sysfs_file(sysfs_key, dmi_value) + self.assertEqual(expected, util.read_dmi_data(dmi_key)) + class TestMultiLog(helpers.FilesystemMockingTestCase): -- cgit v1.2.3 From 0964b42e5117cce640a8ba9102a76fa54a698898 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 21 Mar 2016 21:47:24 -0400 Subject: quickly check to see if the previous instance id is still valid This adds a check in cloud-init to see if the existing (cached) datasource is still valid. It relies on support from the Datasource to implement 'check_instance_id'. That method should quickly determine (if possible) if the instance id found in the datasource is still valid. This means that we can still notice new instance ids without depending on a network datasource on every boot. I've also implemented check_instance_id for the superclass and for 3 classes: DataSourceAzure (check dmi data) DataSourceOpenstack (check dmi data) DataSourceNocloud (check the seeded data or kernel command line) LP: #1553815 --- ChangeLog | 2 ++ bin/cloud-init | 19 ++++++++--------- cloudinit/sources/DataSourceAzure.py | 4 ++++ cloudinit/sources/DataSourceNoCloud.py | 35 ++++++++++++++++++++++++++++++++ cloudinit/sources/DataSourceOpenStack.py | 4 ++++ cloudinit/sources/__init__.py | 16 +++++++++++++++ cloudinit/stages.py | 24 ++++++++++++++-------- 7 files changed, 85 insertions(+), 19 deletions(-) (limited to 'ChangeLog') diff --git a/ChangeLog b/ChangeLog index 0ec4f49e..b08665b0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -92,6 +92,8 @@ - doc: mention label for nocloud datasource must be 'cidata' [Peter Hurley] - ssh_pwauth: fix module to support 'unchanged' and match behavior described in documentation [Chris Cosby] + - quickly check to see if the previous instance id is still valid to + avoid dependency on network metadata service on every boot (LP: #1553815) 0.7.6: - open 0.7.6 diff --git a/bin/cloud-init b/bin/cloud-init index 7f665e7e..11cc0237 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -212,6 +212,7 @@ def main_init(name, args): # Stage 4 path_helper = init.paths if not args.local: + existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug(("Checking to see if files that we need already" " exist from a previous run that would allow us" @@ -236,21 +237,17 @@ def main_init(name, args): LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") else: - # The cache is not instance specific, so it has to be purged - # but we want 'start' to benefit from a cache if - # a previous start-local populated one... - manual_clean = util.get_cfg_option_bool(init.cfg, - 'manual_cache_clean', False) - if manual_clean: - LOG.debug("Not purging instance link, manual cleaning enabled") - init.purge_cache(False) - else: - init.purge_cache() + existing = "check" + if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): + existing = "trust" + + init.purge_cache() # Delete the non-net file as well util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) + # Stage 5 try: - init.fetch() + init.fetch(existing=existing) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 2af0ad9b..832b3063 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -254,6 +254,10 @@ class DataSourceAzureNet(sources.DataSource): def get_config_obj(self): return self.cfg + def check_instance_id(self): + # quickly (local check only) if self.instance_id is still valid + return sources.instance_id_matches_system_uuid(self.get_instance_id()) + def count_files(mp): return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*')) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 4cad6877..d07e6f84 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -197,6 +197,41 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data']['dsmode']) return False + def check_instance_id(self): + # quickly (local check only) if self.instance_id is still valid + # we check kernel command line or files. + current = self.get_instance_id() + if not current: + return None + + quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, + dirs=[self.seed_dir]) + if not quick_id: + return None + return quick_id == current + + +def _quick_read_instance_id(cmdline_id, dirs=None): + if dirs is None: + dirs = [] + + iid_key = 'instance-id' + if cmdline_id is None: + fill = {} + if parse_cmdline_data(cmdline_id, fill) and iid_key in fill: + return fill[iid_key] + + for d in dirs: + try: + data = util.pathprefix2dict(d, required=['meta-data']) + md = util.load_yaml(data['meta-data']) + if iid_key in md: + return md[iid_key] + except ValueError: + pass + + return None + # Returns true or false indicating if cmdline indicated # that this module should be used diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 469c2e2a..79bb9d63 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -150,6 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return True + def check_instance_id(self): + # quickly (local check only) if self.instance_id is still valid + return sources.instance_id_matches_system_uuid(self.get_instance_id()) + def read_metadata_service(base_url, ssl_details=None): reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index d3cfa560..28540a7b 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -217,6 +217,10 @@ class DataSource(object): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) + def check_instance_id(self): + # quickly (local check only) if self.instance_id is still + return False + def normalize_pubkey_data(pubkey_data): keys = [] @@ -299,6 +303,18 @@ def list_sources(cfg_list, depends, pkg_list): return src_list +def instance_id_matches_system_uuid(instance_id, field='system-uuid'): + # quickly (local check only) if self.instance_id is still valid + # we check kernel command line or files. + if not instance_id: + return False + + dmi_value = util.read_dmi_data(field) + if not dmi_value: + return False + return instance_id.lower() == dmi_value.lower() + + # 'depends' is a list of dependencies (DEP_FILESYSTEM) # ds_list is a list of 2 item lists # ds_list = [ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index dbcf3d55..edad6450 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -140,7 +140,7 @@ class Init(object): ] return initial_dirs - def purge_cache(self, rm_instance_lnk=True): + def purge_cache(self, rm_instance_lnk=False): rm_list = [] rm_list.append(self.paths.boot_finished) if rm_instance_lnk: @@ -238,21 +238,29 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) - def _get_data_source(self): + def _get_data_source(self, existing): if self.datasource is not NULL_DATA_SOURCE: return self.datasource with events.ReportEventStack( name="check-cache", - description="attempting to read from cache", + description="attempting to read from cache [%s]" % existing, parent=self.reporter) as myrep: ds = self._restore_from_cache() - if ds: - LOG.debug("Restored from cache, datasource: %s", ds) - myrep.description = "restored from cache" + if ds and existing == "trust": + myrep.description = "restored from cache: %s" % ds + elif ds and existing == "check": + if hasattr(ds, 'check_instance_id') and ds.check_instance_id(): + myrep.description = "restored from checked cache: %s" % ds + else: + myrep.description = "cache invalid in datasource: %s" % ds + ds = None else: myrep.description = "no cache found" + LOG.debug(myrep.description) + if not ds: + util.del_file(self.paths.instance_link) (cfg_list, pkg_list) = self._get_datasources() # Deep copy so that user-data handlers can not modify # (which will affect user-data handlers down the line...) @@ -332,8 +340,8 @@ class Init(object): self._reset() return iid - def fetch(self): - return self._get_data_source() + def fetch(self, existing="check"): + return self._get_data_source(existing=existing) def instancify(self): return self._reflect_cur_instance() -- cgit v1.2.3