From 30b4d15764a1a9644379cf95770e8b2480856882 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 5 Dec 2017 16:25:11 -0700 Subject: cli: Add clean and status subcommands The 'cloud-init clean' command allows a user or script to clear cloud-init artifacts from the system so that cloud-init sees the system as unconfigured upon reboot. Optional parameters can be provided to remove cloud-init logs and reboot after clean. The 'cloud-init status' command allows the user or script to check whether cloud-init has finished all configuration stages and whether errors occurred. An optional --wait argument will poll on a 0.25 second interval until cloud-init configuration is complete. The benefit here is scripts can block on cloud-init completion before performing post-config tasks. --- cloudinit/distros/__init__.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index d5becd12..99e60e7a 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -102,11 +102,8 @@ class Distro(object): self._apply_hostname(writeable_hostname) def uses_systemd(self): - try: - res = os.lstat('/run/systemd/system') - return stat.S_ISDIR(res.st_mode) - except Exception: - return False + """Wrapper to report whether this distro uses systemd or sysvinit.""" + return uses_systemd() @abc.abstractmethod def package_command(self, cmd, args=None, pkgs=None): @@ -761,4 +758,13 @@ def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", util.copy(tz_file, tz_local) return + +def uses_systemd(): + try: + res = os.lstat('/run/systemd/system') + return stat.S_ISDIR(res.st_mode) + except Exception: + return False + + # vi: ts=4 expandtab -- cgit v1.2.3 From 703241a3c50f2cfec21e7c8e90616c3378ebbea2 Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Mon, 27 Nov 2017 21:54:09 +0000 Subject: ec2: Use instance-identity doc for region and instance-id The instance identity document is a better source for region information, partly because region isn't actually in meta-data at all, only availability-zone, which happens to be named similarly. Reviewed-by: Ethan Faust Reviewed-by: Cyle Riggs Reviewed-by: Tom Kirchner Reviewed-by: Matt Nierzwicki [ajorgens@amazon.com: rebase onto 0.7.9] [ajorgens@amazon.com: changes per merge proposal discussions] --- cloudinit/distros/__init__.py | 15 +++++----- cloudinit/ec2_utils.py | 39 ++++++++++++++++++++------ cloudinit/sources/DataSourceEc2.py | 30 ++++++++++++++++---- tests/unittests/test_datasource/test_aliyun.py | 16 +++++++++++ 4 files changed, 78 insertions(+), 22 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 99e60e7a..55260eae 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -45,6 +45,10 @@ OSFAMILIES = { LOG = logging.getLogger(__name__) +# This is a best guess regex, based on current EC2 AZs on 2017-12-11. +# It could break when Amazon adds new regions and new AZs. +_EC2_AZ_RE = re.compile('^[a-z][a-z]-(?:[a-z]+-)+[0-9][a-z]$') + @six.add_metaclass(abc.ABCMeta) class Distro(object): @@ -683,18 +687,13 @@ def _get_package_mirror_info(mirror_info, data_source=None, if not mirror_info: mirror_info = {} - # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) - # the region is us-east-1. so region = az[0:-1] - directions_re = '|'.join([ - 'central', 'east', 'north', 'northeast', 'northwest', - 'south', 'southeast', 'southwest', 'west']) - ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % directions_re) - subst = {} if data_source and data_source.availability_zone: subst['availability_zone'] = data_source.availability_zone - if re.match(ec2_az_re, data_source.availability_zone): + # ec2 availability zones are named cc-direction-[0-9][a-d] (us-east-1b) + # the region is us-east-1. so region = az[0:-1] + if _EC2_AZ_RE.match(data_source.availability_zone): subst['ec2_region'] = "%s" % data_source.availability_zone[0:-1] if data_source and data_source.region: diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 723d6bd6..d6c61e4c 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -1,6 +1,8 @@ # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2014 Amazon.com, Inc. or its affiliates. # # Author: Joshua Harlow +# Author: Andrew Jorgensen # # This file is part of cloud-init. See LICENSE file for license information. @@ -164,14 +166,11 @@ def get_instance_userdata(api_version='latest', return user_data -def get_instance_metadata(api_version='latest', - metadata_address='http://169.254.169.254', - ssl_details=None, timeout=5, retries=5, - leaf_decoder=None): - md_url = url_helper.combine_url(metadata_address, api_version) - # Note, 'meta-data' explicitly has trailing /. - # this is required for CloudStack (LP: #1356855) - md_url = url_helper.combine_url(md_url, 'meta-data/') +def _get_instance_metadata(tree, api_version='latest', + metadata_address='http://169.254.169.254', + ssl_details=None, timeout=5, retries=5, + leaf_decoder=None): + md_url = url_helper.combine_url(metadata_address, api_version, tree) caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) @@ -189,7 +188,29 @@ def get_instance_metadata(api_version='latest', md = {} return md except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", md_url) + util.logexc(LOG, "Failed fetching %s from url %s", tree, md_url) return {} + +def get_instance_metadata(api_version='latest', + metadata_address='http://169.254.169.254', + ssl_details=None, timeout=5, retries=5, + leaf_decoder=None): + # Note, 'meta-data' explicitly has trailing /. + # this is required for CloudStack (LP: #1356855) + return _get_instance_metadata(tree='meta-data/', api_version=api_version, + metadata_address=metadata_address, + ssl_details=ssl_details, timeout=timeout, + retries=retries, leaf_decoder=leaf_decoder) + + +def get_instance_identity(api_version='latest', + metadata_address='http://169.254.169.254', + ssl_details=None, timeout=5, retries=5, + leaf_decoder=None): + return _get_instance_metadata(tree='dynamic/instance-identity', + api_version=api_version, + metadata_address=metadata_address, + ssl_details=ssl_details, timeout=timeout, + retries=retries, leaf_decoder=leaf_decoder) # vi: ts=4 expandtab diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index e5c88334..0f89f34d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -154,7 +154,12 @@ class DataSourceEc2(sources.DataSource): return self.min_metadata_version def get_instance_id(self): - return self.metadata['instance-id'] + if self.cloud_platform == Platforms.AWS: + # Prefer the ID from the instance identity document, but fall back + return self.identity.get( + 'instanceId', self.metadata['instance-id']) + else: + return self.metadata['instance-id'] def _get_url_settings(self): mcfg = self.ds_cfg @@ -268,15 +273,27 @@ class DataSourceEc2(sources.DataSource): @property def availability_zone(self): try: - return self.metadata['placement']['availability-zone'] + if self.cloud_platform == Platforms.AWS: + return self.identity.get( + 'availabilityZone', + self.metadata['placement']['availability-zone']) + else: + return self.metadata['placement']['availability-zone'] except KeyError: return None @property def region(self): - az = self.availability_zone - if az is not None: - return az[:-1] + if self.cloud_platform == Platforms.AWS: + region = self.identity.get('region') + # Fallback to trimming the availability zone if region is missing + if self.availability_zone and not region: + region = self.availability_zone[:-1] + return region + else: + az = self.availability_zone + if az is not None: + return az[:-1] return None @property @@ -357,6 +374,9 @@ class DataSourceEc2(sources.DataSource): api_version, self.metadata_address) self.metadata = ec2.get_instance_metadata( api_version, self.metadata_address) + if self.cloud_platform == Platforms.AWS: + self.identity = ec2.get_instance_identity( + api_version, self.metadata_address).get('document', {}) except Exception: util.logexc( LOG, "Failed reading from metadata address %s", diff --git a/tests/unittests/test_datasource/test_aliyun.py b/tests/unittests/test_datasource/test_aliyun.py index 714f5dac..4fa9616b 100644 --- a/tests/unittests/test_datasource/test_aliyun.py +++ b/tests/unittests/test_datasource/test_aliyun.py @@ -47,6 +47,9 @@ def register_mock_metaserver(base_url, data): elif isinstance(body, list): register(base_url.rstrip('/'), '\n'.join(body) + '\n') elif isinstance(body, dict): + if not body: + register(base_url.rstrip('/') + '/', 'not found', + status_code=404) vals = [] for k, v in body.items(): if isinstance(v, (str, list)): @@ -91,9 +94,22 @@ class TestAliYunDatasource(test_helpers.HttprettyTestCase): self.metadata_address, self.ds.min_metadata_version, 'user-data') + # EC2 provides an instance-identity document which must return 404 here + # for this test to pass. + @property + def default_identity(self): + return {} + + @property + def identity_url(self): + return os.path.join(self.metadata_address, + self.ds.min_metadata_version, + 'dynamic', 'instance-identity') + def regist_default_server(self): register_mock_metaserver(self.metadata_url, self.default_metadata) register_mock_metaserver(self.userdata_url, self.default_userdata) + register_mock_metaserver(self.identity_url, self.default_identity) def _test_get_data(self): self.assertEqual(self.ds.metadata, self.default_metadata) -- cgit v1.2.3 From 1d8c327139a8c291eeb244ee1a6a8badd83e9e72 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Fri, 26 Jan 2018 13:36:30 -0700 Subject: Fix potential cases of uninitialized variables. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While addressing undeclared variable in 'cloud-init status', I also fixed the errors raised by automated code reviews against cloud-init master at https://lgtm.com/projects/g/cloud-init/cloud-init/alerts The following items are addressed:  * Fix 'cloud-init status':     * Only report 'running' state when any stage in /run/cloud-init/status.json has a start time but no finished time. Default start time to 0 if null.     * undeclared variable 'reason' now reports 'Cloud-init enabled by systemd cloud-init-generator' when systemd enables cloud-init  * cc_rh_subscription.py util.subp return values aren't set during if an exception is raised, use ProcessExecution as e instead.  * distros/freebsd.py:    * Drop repetitive looping over ipv4 and ipv6 nic lists.    * Initialize bsddev to 'NOTFOUND' in the event that no devs are discovered    * declare nics_with_addresses = set() in broader scope outside check_downable conditional  * cloudinit/util.py: Raise TypeError if mtype parameter isn't string, iterable or None. LP: #1744796 --- cloudinit/cmd/status.py | 7 +++++-- cloudinit/cmd/tests/test_status.py | 21 ++++++++++++++++++--- cloudinit/config/cc_power_state_change.py | 1 + cloudinit/config/cc_rh_subscription.py | 5 ++--- cloudinit/distros/freebsd.py | 11 +++-------- cloudinit/util.py | 4 ++++ 6 files changed, 33 insertions(+), 16 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/cmd/status.py b/cloudinit/cmd/status.py index 3e5d0d07..d7aaee9d 100644 --- a/cloudinit/cmd/status.py +++ b/cloudinit/cmd/status.py @@ -93,6 +93,8 @@ def _is_cloudinit_disabled(disable_file, paths): elif not os.path.exists(os.path.join(paths.run_dir, 'enabled')): is_disabled = True reason = 'Cloud-init disabled by cloud-init-generator' + else: + reason = 'Cloud-init enabled by systemd cloud-init-generator' return (is_disabled, reason) @@ -127,10 +129,11 @@ def _get_status_details(paths): status_detail = value elif isinstance(value, dict): errors.extend(value.get('errors', [])) + start = value.get('start') or 0 finished = value.get('finished') or 0 - if finished == 0: + if finished == 0 and start != 0: status = STATUS_RUNNING - event_time = max(value.get('start', 0), finished) + event_time = max(start, finished) if event_time > latest_event: latest_event = event_time if errors: diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py index 6d4a11e8..a7c0a91a 100644 --- a/cloudinit/cmd/tests/test_status.py +++ b/cloudinit/cmd/tests/test_status.py @@ -93,6 +93,19 @@ class TestStatus(CiTestCase): self.assertTrue(is_disabled, 'expected disabled cloud-init') self.assertEqual('Cloud-init disabled by cloud-init-generator', reason) + def test__is_cloudinit_disabled_false_when_enabled_in_systemd(self): + '''Report enabled when systemd generator creates the enabled file.''' + enabled_file = os.path.join(self.paths.run_dir, 'enabled') + write_file(enabled_file, '') + (is_disabled, reason) = wrap_and_call( + 'cloudinit.cmd.status', + {'uses_systemd': True, + 'get_cmdline': 'something ignored'}, + status._is_cloudinit_disabled, self.disable_file, self.paths) + self.assertFalse(is_disabled, 'expected enabled cloud-init') + self.assertEqual( + 'Cloud-init enabled by systemd cloud-init-generator', reason) + def test_status_returns_not_run(self): '''When status.json does not exist yet, return 'not run'.''' self.assertFalse( @@ -137,8 +150,9 @@ class TestStatus(CiTestCase): self.assertEqual(expected, m_stdout.getvalue()) def test_status_returns_running(self): - '''Report running when status file exists but isn't finished.''' - write_json(self.status_file, {'v1': {'init': {'finished': None}}}) + '''Report running when status exists with an unfinished stage.''' + write_json(self.status_file, + {'v1': {'init': {'start': 1, 'finished': None}}}) cmdargs = myargs(long=False, wait=False) with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: retcode = wrap_and_call( @@ -338,7 +352,8 @@ class TestStatus(CiTestCase): def test_status_main(self): '''status.main can be run as a standalone script.''' - write_json(self.status_file, {'v1': {'init': {'finished': None}}}) + write_json(self.status_file, + {'v1': {'init': {'start': 1, 'finished': None}}}) with self.assertRaises(SystemExit) as context_manager: with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: wrap_and_call( diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index eba58b02..4da3a588 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -194,6 +194,7 @@ def doexit(sysexit): def execmd(exe_args, output=None, data_in=None): + ret = 1 try: proc = subprocess.Popen(exe_args, stdin=subprocess.PIPE, stdout=output, stderr=subprocess.STDOUT) diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index a9d21e78..530808ce 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -276,9 +276,8 @@ class SubscriptionManager(object): cmd = ['attach', '--auto'] try: return_out, return_err = self._sub_man_cli(cmd) - except util.ProcessExecutionError: - self.log_warn("Auto-attach failed with: " - "{0}]".format(return_err.strip())) + except util.ProcessExecutionError as e: + self.log_warn("Auto-attach failed with: {0}".format(e)) return False for line in return_out.split("\n"): if line is not "": diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index bad112fe..aa468bca 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -116,6 +116,7 @@ class Distro(distros.Distro): (out, err) = util.subp(['ifconfig', '-a']) ifconfigoutput = [x for x in (out.strip()).splitlines() if len(x.split()) > 0] + bsddev = 'NOT_FOUND' for line in ifconfigoutput: m = re.match('^\w+', line) if m: @@ -347,15 +348,9 @@ class Distro(distros.Distro): bymac[Distro.get_interface_mac(n)] = { 'name': n, 'up': self.is_up(n), 'downable': None} + nics_with_addresses = set() if check_downable: - nics_with_addresses = set() - ipv6 = self.get_ipv6() - ipv4 = self.get_ipv4() - for bytes_out in (ipv6, ipv4): - for i in ipv6: - nics_with_addresses.update(i) - for i in ipv4: - nics_with_addresses.update(i) + nics_with_addresses = set(self.get_ipv4() + self.get_ipv6()) for d in bymac.values(): d['downable'] = (d['up'] is False or diff --git a/cloudinit/util.py b/cloudinit/util.py index 9976400f..338fb971 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1587,6 +1587,10 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): mtypes = list(mtype) elif mtype is None: mtypes = None + else: + raise TypeError( + 'Unsupported type provided for mtype parameter: {_type}'.format( + _type=type(mtype))) # clean up 'mtype' input a bit based on platform. platsys = platform.system().lower() -- cgit v1.2.3