From cf7ba5b30597660b7ab4fbdd92f0762e0011c1d6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Jun 2016 14:40:05 -0700 Subject: Fix the broken import and 'parse_net_config_data' function usage --- cloudinit/distros/debian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index e71aaa97..603b0b61 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,8 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import net from cloudinit.net import eni +from cloudinit.net.network_state import parse_net_config_data from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -81,7 +81,7 @@ class Distro(distros.Distro): return ['all'] def _write_network_config(self, netconfig): - ns = net.parse_net_config_data(netconfig) + ns = parse_net_config_data(netconfig) self._net_renderer.render_network_state( target="/", network_state=ns, eni=self.network_conf_fn, links_prefix=self.links_prefix, -- cgit v1.2.3 From df583f83ac9d4869e0b3df5a904d13890d2f0986 Mon Sep 17 00:00:00 2001 From: Phil Roche Date: Mon, 13 Jun 2016 09:52:52 +0100 Subject: Removes trailing dot in metadata.google.internal GCE metadata lookup. A bug was reported (lp:1581200) where if there is no DNS server configured or it is not running then the metadata lookup on GCE will fail as it contains a trailing dot 'metadata.google.internal.'. As there is no DNS configured or running it will use the /etc/hosts file but the hosts file does not contain an entry with the trailing dot. One solution is to add an entry to the /etc/hosts file with the trailing dot but according to the manpage, /etc/hosts entries must end with an alphanumeric character and cannot end with a dot. The trailing dot was added to avoid MIM by dns search but we should probably assume the instance being started has no DNS and as such when querying metadata should use a URL that will resolve using /etc/hosts. LP: #1581200 --- cloudinit/sources/DataSourceGCE.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 9234d1f8..c660a350 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,7 +25,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') -- cgit v1.2.3 From 780f88f4dd8ddd36c54e217ea5af3b18b7339758 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 14 Jun 2016 15:08:24 -0400 Subject: [Revert] Remove trailing dot from GCE metadata URL This change broke tox tests. --- ChangeLog | 1 - cloudinit/sources/DataSourceGCE.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 66539792..e2869522 100644 --- a/ChangeLog +++ b/ChangeLog @@ -122,7 +122,6 @@ - ConfigDrive: improved support for networking information from a network_data.json or older interfaces formated network_config. - Change missing Cheetah log warning to debug [Andrew Jorgensen] - - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] 0.7.6: - open 0.7.6 diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c660a350..9234d1f8 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,7 +25,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') -- cgit v1.2.3 From 632d56a23a947a23e18fa234d675e34a1a119593 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 14 Jun 2016 15:18:53 -0400 Subject: fix pep8 failure introduced in recent commit. The commit 1232 (Refactor a large part of the networking code) broke pep8. --- cloudinit/net/eni.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index a695f5ed..5a90eb32 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -391,8 +391,8 @@ class Renderer(object): iface['control'] = subnet.get('control', 'auto') if iface['mode'].endswith('6'): iface['inet'] += '6' - elif (iface['mode'] == 'static' - and ":" in subnet['address']): + elif (iface['mode'] == 'static' and + ":" in subnet['address']): iface['inet'] += '6' if iface['mode'].startswith('dhcp'): iface['mode'] = 'dhcp' -- cgit v1.2.3 From 93afde09d89e60c29dfd20790e30a06f031c82e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 14:56:51 -0700 Subject: Make the bin/cloud-init an actual console entrypoint This allows for the test_cli test to be more sane. --- bin/cloud-init | 689 -------------------------------------------- cloudinit/cmd/__init__.py | 21 ++ cloudinit/cmd/main.py | 682 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 8 +- tests/unittests/test_cli.py | 37 +-- 5 files changed, 721 insertions(+), 716 deletions(-) delete mode 100755 bin/cloud-init create mode 100644 cloudinit/cmd/__init__.py create mode 100755 cloudinit/cmd/main.py diff --git a/bin/cloud-init b/bin/cloud-init deleted file mode 100755 index 21c3a684..00000000 --- a/bin/cloud-init +++ /dev/null @@ -1,689 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -import json -import os -import sys -import time -import tempfile -import traceback - -# This is more just for running from the bin folder so that -# cloud-init binary can find the cloudinit module -possible_topdir = os.path.normpath(os.path.join(os.path.abspath( - sys.argv[0]), os.pardir, os.pardir)) -if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")): - sys.path.insert(0, possible_topdir) - -from cloudinit import patcher -patcher.patch() - -from cloudinit import log as logging -from cloudinit import netinfo -from cloudinit import signal_handler -from cloudinit import sources -from cloudinit import stages -from cloudinit import templater -from cloudinit import util -from cloudinit import reporting -from cloudinit.reporting import events -from cloudinit import version - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, - CLOUD_CONFIG) - - -# Pretty little cheetah formatted welcome message template -WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " - "${timestamp}. Up ${uptime} seconds.") - -# Module section template -MOD_SECTION_TPL = "cloud_%s_modules" - -# Things u can query on -QUERY_DATA_TYPES = [ - 'data', - 'data_raw', - 'instance_id', -] - -# Frequency shortname to full name -# (so users don't have to remember the full name...) -FREQ_SHORT_NAMES = { - 'instance': PER_INSTANCE, - 'always': PER_ALWAYS, - 'once': PER_ONCE, -} - -LOG = logging.getLogger() - - -# Used for when a logger may not be active -# and we still want to print exceptions... -def print_exc(msg=''): - if msg: - sys.stderr.write("%s\n" % (msg)) - sys.stderr.write('-' * 60) - sys.stderr.write("\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.write('-' * 60) - sys.stderr.write("\n") - - -def welcome(action, msg=None): - if not msg: - msg = welcome_format(action) - util.multi_log("%s\n" % (msg), - console=False, stderr=True, log=LOG) - return msg - - -def welcome_format(action): - tpl_params = { - 'version': version.version_string(), - 'uptime': util.uptime(), - 'timestamp': util.time_rfc2822(), - 'action': action, - } - return templater.render_string(WELCOME_MSG_TPL, tpl_params) - - -def extract_fns(args): - # Files are already opened so lets just pass that along - # since it would of broke if it couldn't have - # read that file already... - fn_cfgs = [] - if args.files: - for fh in args.files: - # The realpath is more useful in logging - # so lets resolve to that... - fn_cfgs.append(os.path.realpath(fh.name)) - return fn_cfgs - - -def run_module_section(mods, action_name, section): - full_section_name = MOD_SECTION_TPL % (section) - (which_ran, failures) = mods.run_section(full_section_name) - total_attempted = len(which_ran) + len(failures) - if total_attempted == 0: - msg = ("No '%s' modules to run" - " under section '%s'") % (action_name, full_section_name) - sys.stderr.write("%s\n" % (msg)) - LOG.debug(msg) - return [] - else: - LOG.debug("Ran %s modules with %s failures", - len(which_ran), len(failures)) - return failures - - -def apply_reporting_cfg(cfg): - if cfg.get('reporting'): - reporting.update_configuration(cfg.get('reporting')) - - -def main_init(name, args): - deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] - if args.local: - deps = [sources.DEP_FILESYSTEM] - - if not args.local: - # See doc/kernel-cmdline.txt - # - # This is used in maas datasource, in "ephemeral" (read-only root) - # environment where the instance netboots to iscsi ro root. - # and the entity that controls the pxe config has to configure - # the maas datasource. - # - # Could be used elsewhere, only works on network based (not local). - root_name = "%s.d" % (CLOUD_CONFIG) - target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") - util.read_write_cmdline_url(target_fn) - - # Cloud-init 'init' stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Setup logging/output redirections with resultant config (if any) - # 3. Initialize the cloud-init filesystem - # 4. Check if we can stop early by looking for various files - # 5. Fetch the datasource - # 6. Connect to the current instance location + update the cache - # 7. Consume the userdata (handlers get activated here) - # 8. Construct the modules object - # 9. Adjust any subsequent logging/output redirections using the modules - # objects config as it may be different from init object - # 10. Run the modules for the 'init' stage - # 11. Done! - if not args.local: - w_msg = welcome_format(name) - else: - w_msg = welcome_format("%s-local" % (name)) - init = stages.Init(ds_deps=deps, reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - outfmt = None - errfmt = None - try: - LOG.debug("Closing stdin") - util.close_stdin() - (outfmt, errfmt) = util.fixup_output(init.cfg, name) - except: - util.logexc(LOG, "Failed to setup output redirection!") - print_exc("Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(init.cfg) - apply_reporting_cfg(init.cfg) - - # Any log usage prior to setupLogging above did not have local user log - # config applied. We send the welcome message now, as stderr/out have - # been redirected and log now configured. - welcome(name, msg=w_msg) - - # Stage 3 - try: - init.initialize() - except Exception: - util.logexc(LOG, "Failed to initialize, likely bad things to come!") - # Stage 4 - path_helper = init.paths - mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK - - if mode == sources.DSMODE_NETWORK: - existing = "trust" - sys.stderr.write("%s\n" % (netinfo.debug_info())) - LOG.debug(("Checking to see if files that we need already" - " exist from a previous run that would allow us" - " to stop early.")) - # no-net is written by upstart cloud-init-nonet when network failed - # to come up - stop_files = [ - os.path.join(path_helper.get_cpath("data"), "no-net"), - ] - existing_files = [] - for fn in stop_files: - if os.path.isfile(fn): - existing_files.append(fn) - - if existing_files: - LOG.debug("[%s] Exiting. stop file %s existed", - mode, existing_files) - return (None, []) - else: - LOG.debug("Execution continuing, no previous run detected that" - " would allow us to stop early.") - else: - existing = "check" - if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): - existing = "trust" - - init.purge_cache() - # Delete the non-net file as well - util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) - - # Stage 5 - try: - init.fetch(existing=existing) - # if in network mode, and the datasource is local - # then work was done at that stage. - if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: - LOG.debug("[%s] Exiting. datasource %s in local mode", - mode, init.datasource) - return (None, []) - except sources.DataSourceNotFoundException: - # In the case of 'cloud-init init' without '--local' it is a bit - # more likely that the user would consider it failure if nothing was - # found. When using upstart it will also mentions job failure - # in console log if exit code is != 0. - if mode == sources.DSMODE_LOCAL: - LOG.debug("No local datasource found") - else: - util.logexc(LOG, ("No instance datasource found!" - " Likely bad things to come!")) - if not args.force: - init.apply_network_config(bring_up=not args.local) - LOG.debug("[%s] Exiting without datasource in local mode", mode) - if mode == sources.DSMODE_LOCAL: - return (None, []) - else: - return (None, ["No instance datasource found."]) - else: - LOG.debug("[%s] barreling on in force mode without datasource", - mode) - - # Stage 6 - iid = init.instancify() - LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", - mode, name, iid, init.is_new_instance()) - - init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) - - if mode == sources.DSMODE_LOCAL: - if init.datasource.dsmode != mode: - LOG.debug("[%s] Exiting. datasource %s not in local mode.", - mode, init.datasource) - return (init.datasource, []) - else: - LOG.debug("[%s] %s is in local mode, will apply init modules now.", - mode, init.datasource) - - # update fully realizes user-data (pulling in #include if necessary) - init.update() - # Stage 7 - try: - # Attempt to consume the data per instance. - # This may run user-data handlers and/or perform - # url downloads and such as needed. - (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) - if not ran: - # Just consume anything that is set to run per-always - # if nothing ran in the per-instance code - # - # See: https://bugs.launchpad.net/bugs/819507 for a little - # reason behind this... - init.consume_data(PER_ALWAYS) - except Exception: - util.logexc(LOG, "Consuming user data failed!") - return (init.datasource, ["Consuming user data failed!"]) - - apply_reporting_cfg(init.cfg) - - # Stage 8 - re-read and apply relevant cloud-config to include user-data - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 9 - try: - outfmt_orig = outfmt - errfmt_orig = errfmt - (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) - if outfmt_orig != outfmt or errfmt_orig != errfmt: - LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) - (outfmt, errfmt) = util.fixup_output(mods.cfg, name) - except: - util.logexc(LOG, "Failed to re-adjust output redirection!") - logging.setupLogging(mods.cfg) - - # Stage 10 - return (init.datasource, run_module_section(mods, name, name)) - - -def main_modules(action_name, args): - name = args.mode - # Cloud-init 'modules' stages are broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Get the datasource from the init object, if it does - # not exist then that means the main_init stage never - # worked, and thus this stage can not run. - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the modules for the given stage name - # 6. Done! - w_msg = welcome_format("%s:%s" % (action_name, name)) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, theres nothing to do - msg = ('Can not apply stage %s, no datasource found! Likely bad ' - 'things to come!' % name) - util.logexc(LOG, msg) - print_exc(msg) - if not args.force: - return [(msg)] - # Stage 3 - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, name) - except: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - return run_module_section(mods, name, name) - - -def main_query(name, _args): - raise NotImplementedError(("Action '%s' is not" - " currently implemented") % (name)) - - -def main_single(name, args): - # Cloud-init single stage is broken up into the following sub-stages - # 1. Ensure that the init object fetches its config without errors - # 2. Attempt to fetch the datasource (warn if it doesn't work) - # 3. Construct the modules object - # 4. Adjust any subsequent logging/output redirections using - # the modules objects configuration - # 5. Run the single module - # 6. Done! - mod_name = args.name - w_msg = welcome_format(name) - init = stages.Init(ds_deps=[], reporter=args.reporter) - # Stage 1 - init.read_cfg(extract_fns(args)) - # Stage 2 - try: - init.fetch(existing="trust") - except sources.DataSourceNotFoundException: - # There was no datasource found, - # that might be bad (or ok) depending on - # the module being ran (so continue on) - util.logexc(LOG, ("Failed to fetch your datasource," - " likely bad things to come!")) - print_exc(("Failed to fetch your datasource," - " likely bad things to come!")) - if not args.force: - return 1 - # Stage 3 - mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) - mod_args = args.module_args - if mod_args: - LOG.debug("Using passed in arguments %s", mod_args) - mod_freq = args.frequency - if mod_freq: - LOG.debug("Using passed in frequency %s", mod_freq) - mod_freq = FREQ_SHORT_NAMES.get(mod_freq) - # Stage 4 - try: - LOG.debug("Closing stdin") - util.close_stdin() - util.fixup_output(mods.cfg, None) - except: - util.logexc(LOG, "Failed to setup output redirection!") - if args.debug: - # Reset so that all the debug handlers are closed out - LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) - logging.resetLogging() - logging.setupLogging(mods.cfg) - apply_reporting_cfg(init.cfg) - - # now that logging is setup and stdout redirected, send welcome - welcome(name, msg=w_msg) - - # Stage 5 - (which_ran, failures) = mods.run_single(mod_name, - mod_args, - mod_freq) - if failures: - LOG.warn("Ran %s but it failed!", mod_name) - return 1 - elif not which_ran: - LOG.warn("Did not run %s, does it exist?", mod_name) - return 1 - else: - # Guess it worked - return 0 - - -def atomic_write_file(path, content, mode='w'): - tf = None - try: - tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), - delete=False, mode=mode) - tf.write(content) - tf.close() - os.rename(tf.name, path) - except Exception as e: - if tf is not None: - os.unlink(tf.name) - raise e - - -def atomic_write_json(path, data): - return atomic_write_file(path, json.dumps(data, indent=1) + "\n") - - -def status_wrapper(name, args, data_d=None, link_d=None): - if data_d is None: - data_d = os.path.normpath("/var/lib/cloud/data") - if link_d is None: - link_d = os.path.normpath("/run/cloud-init") - - status_path = os.path.join(data_d, "status.json") - status_link = os.path.join(link_d, "status.json") - result_path = os.path.join(data_d, "result.json") - result_link = os.path.join(link_d, "result.json") - - util.ensure_dirs((data_d, link_d,)) - - (_name, functor) = args.action - - if name == "init": - if args.local: - mode = "init-local" - else: - mode = "init" - elif name == "modules": - mode = "modules-%s" % args.mode - else: - raise ValueError("unknown name: %s" % name) - - modes = ('init', 'init-local', 'modules-config', 'modules-final') - - status = None - if mode == 'init-local': - for f in (status_link, result_link, status_path, result_path): - util.del_file(f) - else: - try: - status = json.loads(util.load_file(status_path)) - except: - pass - - if status is None: - nullstatus = { - 'errors': [], - 'start': None, - 'finished': None, - } - status = {'v1': {}} - for m in modes: - status['v1'][m] = nullstatus.copy() - status['v1']['datasource'] = None - - v1 = status['v1'] - v1['stage'] = mode - v1[mode]['start'] = time.time() - - atomic_write_json(status_path, status) - util.sym_link(os.path.relpath(status_path, link_d), status_link, - force=True) - - try: - ret = functor(name, args) - if mode in ('init', 'init-local'): - (datasource, errors) = ret - if datasource is not None: - v1['datasource'] = str(datasource) - else: - errors = ret - - v1[mode]['errors'] = [str(e) for e in errors] - - except Exception as e: - util.logexc(LOG, "failed stage %s", mode) - print_exc("failed run of stage %s" % mode) - v1[mode]['errors'] = [str(e)] - - v1[mode]['finished'] = time.time() - v1['stage'] = None - - atomic_write_json(status_path, status) - - if mode == "modules-final": - # write the 'finished' file - errors = [] - for m in modes: - if v1[m]['errors']: - errors.extend(v1[m].get('errors', [])) - - atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], - 'errors': errors}}) - util.sym_link(os.path.relpath(result_path, link_d), result_link, - force=True) - - return len(v1[mode]['errors']) - - -def main(): - parser = argparse.ArgumentParser() - - # Top level args - parser.add_argument('--version', '-v', action='version', - version='%(prog)s ' + (version.version_string())) - parser.add_argument('--file', '-f', action='append', - dest='files', - help=('additional yaml configuration' - ' files to use'), - type=argparse.FileType('rb')) - parser.add_argument('--debug', '-d', action='store_true', - help=('show additional pre-action' - ' logging (default: %(default)s)'), - default=False) - parser.add_argument('--force', action='store_true', - help=('force running even if no datasource is' - ' found (use at your own risk)'), - dest='force', - default=False) - - parser.set_defaults(reporter=None) - subparsers = parser.add_subparsers() - - # Each action and its sub-options (if any) - parser_init = subparsers.add_parser('init', - help=('initializes cloud-init and' - ' performs initial modules')) - parser_init.add_argument("--local", '-l', action='store_true', - help="start in local mode (default: %(default)s)", - default=False) - # This is used so that we can know which action is selected + - # the functor to use to run this subcommand - parser_init.set_defaults(action=('init', main_init)) - - # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser('modules', - help=('activates modules using ' - 'a given configuration key')) - parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) - parser_mod.set_defaults(action=('modules', main_modules)) - - # These settings are used when you want to query information - # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', - help=('query information stored ' - 'in cloud-init')) - parser_query.add_argument("--name", '-n', action="store", - help="item name to query on", - required=True, - choices=QUERY_DATA_TYPES) - parser_query.set_defaults(action=('query', main_query)) - - # This subcommand allows you to run a single module - parser_single = subparsers.add_parser('single', - help=('run a single module ')) - parser_single.set_defaults(action=('single', main_single)) - parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) - parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) - parser_single.add_argument("--report", action="store_true", - help="enable reporting", - required=False) - parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) - parser_single.set_defaults(action=('single', main_single)) - - args = parser.parse_args() - - # Setup basic logging to start (until reinitialized) - # iff in debug mode... - if args.debug: - logging.setupBasicLogging() - - # Setup signal handlers before running - signal_handler.attach_handlers() - - if not hasattr(args, 'action'): - parser.error('too few arguments') - (name, functor) = args.action - if name in ("modules", "init"): - functor = status_wrapper - - report_on = True - if name == "init": - if args.local: - rname, rdesc = ("init-local", "searching for local datasources") - else: - rname, rdesc = ("init-network", - "searching for network datasources") - elif name == "modules": - rname, rdesc = ("modules-%s" % args.mode, - "running modules for %s" % args.mode) - elif name == "single": - rname, rdesc = ("single/%s" % args.name, - "running single module %s" % args.name) - report_on = args.report - - args.reporter = events.ReportEventStack( - rname, rdesc, reporting_enabled=report_on) - with args.reporter: - return util.log_time( - logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, - get_uptime=True, func=functor, args=(name, args)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/cloudinit/cmd/__init__.py b/cloudinit/cmd/__init__.py new file mode 100644 index 00000000..da124641 --- /dev/null +++ b/cloudinit/cmd/__init__.py @@ -0,0 +1,21 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py new file mode 100755 index 00000000..27b7b56d --- /dev/null +++ b/cloudinit/cmd/main.py @@ -0,0 +1,682 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import json +import os +import sys +import tempfile +import time +import traceback + +from cloudinit import patcher +patcher.patch() + +from cloudinit import log as logging +from cloudinit import netinfo +from cloudinit import signal_handler +from cloudinit import sources +from cloudinit import stages +from cloudinit import templater +from cloudinit import util +from cloudinit import version + +from cloudinit import reporting +from cloudinit.reporting import events + +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, + CLOUD_CONFIG) + + +# Pretty little cheetah formatted welcome message template +WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " + "${timestamp}. Up ${uptime} seconds.") + +# Module section template +MOD_SECTION_TPL = "cloud_%s_modules" + +# Things u can query on +QUERY_DATA_TYPES = [ + 'data', + 'data_raw', + 'instance_id', +] + +# Frequency shortname to full name +# (so users don't have to remember the full name...) +FREQ_SHORT_NAMES = { + 'instance': PER_INSTANCE, + 'always': PER_ALWAYS, + 'once': PER_ONCE, +} + +LOG = logging.getLogger() + + +# Used for when a logger may not be active +# and we still want to print exceptions... +def print_exc(msg=''): + if msg: + sys.stderr.write("%s\n" % (msg)) + sys.stderr.write('-' * 60) + sys.stderr.write("\n") + traceback.print_exc(file=sys.stderr) + sys.stderr.write('-' * 60) + sys.stderr.write("\n") + + +def welcome(action, msg=None): + if not msg: + msg = welcome_format(action) + util.multi_log("%s\n" % (msg), + console=False, stderr=True, log=LOG) + return msg + + +def welcome_format(action): + tpl_params = { + 'version': version.version_string(), + 'uptime': util.uptime(), + 'timestamp': util.time_rfc2822(), + 'action': action, + } + return templater.render_string(WELCOME_MSG_TPL, tpl_params) + + +def extract_fns(args): + # Files are already opened so lets just pass that along + # since it would of broke if it couldn't have + # read that file already... + fn_cfgs = [] + if args.files: + for fh in args.files: + # The realpath is more useful in logging + # so lets resolve to that... + fn_cfgs.append(os.path.realpath(fh.name)) + return fn_cfgs + + +def run_module_section(mods, action_name, section): + full_section_name = MOD_SECTION_TPL % (section) + (which_ran, failures) = mods.run_section(full_section_name) + total_attempted = len(which_ran) + len(failures) + if total_attempted == 0: + msg = ("No '%s' modules to run" + " under section '%s'") % (action_name, full_section_name) + sys.stderr.write("%s\n" % (msg)) + LOG.debug(msg) + return [] + else: + LOG.debug("Ran %s modules with %s failures", + len(which_ran), len(failures)) + return failures + + +def apply_reporting_cfg(cfg): + if cfg.get('reporting'): + reporting.update_configuration(cfg.get('reporting')) + + +def main_init(name, args): + deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] + if args.local: + deps = [sources.DEP_FILESYSTEM] + + if not args.local: + # See doc/kernel-cmdline.txt + # + # This is used in maas datasource, in "ephemeral" (read-only root) + # environment where the instance netboots to iscsi ro root. + # and the entity that controls the pxe config has to configure + # the maas datasource. + # + # Could be used elsewhere, only works on network based (not local). + root_name = "%s.d" % (CLOUD_CONFIG) + target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") + util.read_write_cmdline_url(target_fn) + + # Cloud-init 'init' stage is broken up into the following sub-stages + # 1. Ensure that the init object fetches its config without errors + # 2. Setup logging/output redirections with resultant config (if any) + # 3. Initialize the cloud-init filesystem + # 4. Check if we can stop early by looking for various files + # 5. Fetch the datasource + # 6. Connect to the current instance location + update the cache + # 7. Consume the userdata (handlers get activated here) + # 8. Construct the modules object + # 9. Adjust any subsequent logging/output redirections using the modules + # objects config as it may be different from init object + # 10. Run the modules for the 'init' stage + # 11. Done! + if not args.local: + w_msg = welcome_format(name) + else: + w_msg = welcome_format("%s-local" % (name)) + init = stages.Init(ds_deps=deps, reporter=args.reporter) + # Stage 1 + init.read_cfg(extract_fns(args)) + # Stage 2 + outfmt = None + errfmt = None + try: + LOG.debug("Closing stdin") + util.close_stdin() + (outfmt, errfmt) = util.fixup_output(init.cfg, name) + except Exception: + util.logexc(LOG, "Failed to setup output redirection!") + print_exc("Failed to setup output redirection!") + if args.debug: + # Reset so that all the debug handlers are closed out + LOG.debug(("Logging being reset, this logger may no" + " longer be active shortly")) + logging.resetLogging() + logging.setupLogging(init.cfg) + apply_reporting_cfg(init.cfg) + + # Any log usage prior to setupLogging above did not have local user log + # config applied. We send the welcome message now, as stderr/out have + # been redirected and log now configured. + welcome(name, msg=w_msg) + + # Stage 3 + try: + init.initialize() + except Exception: + util.logexc(LOG, "Failed to initialize, likely bad things to come!") + # Stage 4 + path_helper = init.paths + mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK + + if mode == sources.DSMODE_NETWORK: + existing = "trust" + sys.stderr.write("%s\n" % (netinfo.debug_info())) + LOG.debug(("Checking to see if files that we need already" + " exist from a previous run that would allow us" + " to stop early.")) + # no-net is written by upstart cloud-init-nonet when network failed + # to come up + stop_files = [ + os.path.join(path_helper.get_cpath("data"), "no-net"), + ] + existing_files = [] + for fn in stop_files: + if os.path.isfile(fn): + existing_files.append(fn) + + if existing_files: + LOG.debug("[%s] Exiting. stop file %s existed", + mode, existing_files) + return (None, []) + else: + LOG.debug("Execution continuing, no previous run detected that" + " would allow us to stop early.") + else: + existing = "check" + if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): + existing = "trust" + + init.purge_cache() + # Delete the non-net file as well + util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net")) + + # Stage 5 + try: + init.fetch(existing=existing) + # if in network mode, and the datasource is local + # then work was done at that stage. + if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s in local mode", + mode, init.datasource) + return (None, []) + except sources.DataSourceNotFoundException: + # In the case of 'cloud-init init' without '--local' it is a bit + # more likely that the user would consider it failure if nothing was + # found. When using upstart it will also mentions job failure + # in console log if exit code is != 0. + if mode == sources.DSMODE_LOCAL: + LOG.debug("No local datasource found") + else: + util.logexc(LOG, ("No instance datasource found!" + " Likely bad things to come!")) + if not args.force: + init.apply_network_config(bring_up=not args.local) + LOG.debug("[%s] Exiting without datasource in local mode", mode) + if mode == sources.DSMODE_LOCAL: + return (None, []) + else: + return (None, ["No instance datasource found."]) + else: + LOG.debug("[%s] barreling on in force mode without datasource", + mode) + + # Stage 6 + iid = init.instancify() + LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", + mode, name, iid, init.is_new_instance()) + + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) + + if mode == sources.DSMODE_LOCAL: + if init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s not in local mode.", + mode, init.datasource) + return (init.datasource, []) + else: + LOG.debug("[%s] %s is in local mode, will apply init modules now.", + mode, init.datasource) + + # update fully realizes user-data (pulling in #include if necessary) + init.update() + # Stage 7 + try: + # Attempt to consume the data per instance. + # This may run user-data handlers and/or perform + # url downloads and such as needed. + (ran, _results) = init.cloudify().run('consume_data', + init.consume_data, + args=[PER_INSTANCE], + freq=PER_INSTANCE) + if not ran: + # Just consume anything that is set to run per-always + # if nothing ran in the per-instance code + # + # See: https://bugs.launchpad.net/bugs/819507 for a little + # reason behind this... + init.consume_data(PER_ALWAYS) + except Exception: + util.logexc(LOG, "Consuming user data failed!") + return (init.datasource, ["Consuming user data failed!"]) + + apply_reporting_cfg(init.cfg) + + # Stage 8 - re-read and apply relevant cloud-config to include user-data + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) + # Stage 9 + try: + outfmt_orig = outfmt + errfmt_orig = errfmt + (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name) + if outfmt_orig != outfmt or errfmt_orig != errfmt: + LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) + (outfmt, errfmt) = util.fixup_output(mods.cfg, name) + except Exception: + util.logexc(LOG, "Failed to re-adjust output redirection!") + logging.setupLogging(mods.cfg) + + # Stage 10 + return (init.datasource, run_module_section(mods, name, name)) + + +def main_modules(action_name, args): + name = args.mode + # Cloud-init 'modules' stages are broken up into the following sub-stages + # 1. Ensure that the init object fetches its config without errors + # 2. Get the datasource from the init object, if it does + # not exist then that means the main_init stage never + # worked, and thus this stage can not run. + # 3. Construct the modules object + # 4. Adjust any subsequent logging/output redirections using + # the modules objects configuration + # 5. Run the modules for the given stage name + # 6. Done! + w_msg = welcome_format("%s:%s" % (action_name, name)) + init = stages.Init(ds_deps=[], reporter=args.reporter) + # Stage 1 + init.read_cfg(extract_fns(args)) + # Stage 2 + try: + init.fetch(existing="trust") + except sources.DataSourceNotFoundException: + # There was no datasource found, theres nothing to do + msg = ('Can not apply stage %s, no datasource found! Likely bad ' + 'things to come!' % name) + util.logexc(LOG, msg) + print_exc(msg) + if not args.force: + return [(msg)] + # Stage 3 + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) + # Stage 4 + try: + LOG.debug("Closing stdin") + util.close_stdin() + util.fixup_output(mods.cfg, name) + except Exception: + util.logexc(LOG, "Failed to setup output redirection!") + if args.debug: + # Reset so that all the debug handlers are closed out + LOG.debug(("Logging being reset, this logger may no" + " longer be active shortly")) + logging.resetLogging() + logging.setupLogging(mods.cfg) + apply_reporting_cfg(init.cfg) + + # now that logging is setup and stdout redirected, send welcome + welcome(name, msg=w_msg) + + # Stage 5 + return run_module_section(mods, name, name) + + +def main_query(name, _args): + raise NotImplementedError(("Action '%s' is not" + " currently implemented") % (name)) + + +def main_single(name, args): + # Cloud-init single stage is broken up into the following sub-stages + # 1. Ensure that the init object fetches its config without errors + # 2. Attempt to fetch the datasource (warn if it doesn't work) + # 3. Construct the modules object + # 4. Adjust any subsequent logging/output redirections using + # the modules objects configuration + # 5. Run the single module + # 6. Done! + mod_name = args.name + w_msg = welcome_format(name) + init = stages.Init(ds_deps=[], reporter=args.reporter) + # Stage 1 + init.read_cfg(extract_fns(args)) + # Stage 2 + try: + init.fetch(existing="trust") + except sources.DataSourceNotFoundException: + # There was no datasource found, + # that might be bad (or ok) depending on + # the module being ran (so continue on) + util.logexc(LOG, ("Failed to fetch your datasource," + " likely bad things to come!")) + print_exc(("Failed to fetch your datasource," + " likely bad things to come!")) + if not args.force: + return 1 + # Stage 3 + mods = stages.Modules(init, extract_fns(args), reporter=args.reporter) + mod_args = args.module_args + if mod_args: + LOG.debug("Using passed in arguments %s", mod_args) + mod_freq = args.frequency + if mod_freq: + LOG.debug("Using passed in frequency %s", mod_freq) + mod_freq = FREQ_SHORT_NAMES.get(mod_freq) + # Stage 4 + try: + LOG.debug("Closing stdin") + util.close_stdin() + util.fixup_output(mods.cfg, None) + except Exception: + util.logexc(LOG, "Failed to setup output redirection!") + if args.debug: + # Reset so that all the debug handlers are closed out + LOG.debug(("Logging being reset, this logger may no" + " longer be active shortly")) + logging.resetLogging() + logging.setupLogging(mods.cfg) + apply_reporting_cfg(init.cfg) + + # now that logging is setup and stdout redirected, send welcome + welcome(name, msg=w_msg) + + # Stage 5 + (which_ran, failures) = mods.run_single(mod_name, + mod_args, + mod_freq) + if failures: + LOG.warn("Ran %s but it failed!", mod_name) + return 1 + elif not which_ran: + LOG.warn("Did not run %s, does it exist?", mod_name) + return 1 + else: + # Guess it worked + return 0 + + +def atomic_write_file(path, content, mode='w'): + tf = None + try: + tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), + delete=False, mode=mode) + tf.write(content) + tf.close() + os.rename(tf.name, path) + except Exception as e: + if tf is not None: + os.unlink(tf.name) + raise e + + +def atomic_write_json(path, data): + return atomic_write_file(path, json.dumps(data, indent=1) + "\n") + + +def status_wrapper(name, args, data_d=None, link_d=None): + if data_d is None: + data_d = os.path.normpath("/var/lib/cloud/data") + if link_d is None: + link_d = os.path.normpath("/run/cloud-init") + + status_path = os.path.join(data_d, "status.json") + status_link = os.path.join(link_d, "status.json") + result_path = os.path.join(data_d, "result.json") + result_link = os.path.join(link_d, "result.json") + + util.ensure_dirs((data_d, link_d,)) + + (_name, functor) = args.action + + if name == "init": + if args.local: + mode = "init-local" + else: + mode = "init" + elif name == "modules": + mode = "modules-%s" % args.mode + else: + raise ValueError("unknown name: %s" % name) + + modes = ('init', 'init-local', 'modules-config', 'modules-final') + + status = None + if mode == 'init-local': + for f in (status_link, result_link, status_path, result_path): + util.del_file(f) + else: + try: + status = json.loads(util.load_file(status_path)) + except Exception: + pass + + if status is None: + nullstatus = { + 'errors': [], + 'start': None, + 'finished': None, + } + status = {'v1': {}} + for m in modes: + status['v1'][m] = nullstatus.copy() + status['v1']['datasource'] = None + + v1 = status['v1'] + v1['stage'] = mode + v1[mode]['start'] = time.time() + + atomic_write_json(status_path, status) + util.sym_link(os.path.relpath(status_path, link_d), status_link, + force=True) + + try: + ret = functor(name, args) + if mode in ('init', 'init-local'): + (datasource, errors) = ret + if datasource is not None: + v1['datasource'] = str(datasource) + else: + errors = ret + + v1[mode]['errors'] = [str(e) for e in errors] + + except Exception as e: + util.logexc(LOG, "failed stage %s", mode) + print_exc("failed run of stage %s" % mode) + v1[mode]['errors'] = [str(e)] + + v1[mode]['finished'] = time.time() + v1['stage'] = None + + atomic_write_json(status_path, status) + + if mode == "modules-final": + # write the 'finished' file + errors = [] + for m in modes: + if v1[m]['errors']: + errors.extend(v1[m].get('errors', [])) + + atomic_write_json(result_path, + {'v1': {'datasource': v1['datasource'], + 'errors': errors}}) + util.sym_link(os.path.relpath(result_path, link_d), result_link, + force=True) + + return len(v1[mode]['errors']) + + +def main(sysv_args=None): + if sysv_args is not None: + parser = argparse.ArgumentParser(prog=sysv_args[0]) + sysv_args = sysv_args[1:] + else: + parser = argparse.ArgumentParser() + + # Top level args + parser.add_argument('--version', '-v', action='version', + version='%(prog)s ' + (version.version_string())) + parser.add_argument('--file', '-f', action='append', + dest='files', + help=('additional yaml configuration' + ' files to use'), + type=argparse.FileType('rb')) + parser.add_argument('--debug', '-d', action='store_true', + help=('show additional pre-action' + ' logging (default: %(default)s)'), + default=False) + parser.add_argument('--force', action='store_true', + help=('force running even if no datasource is' + ' found (use at your own risk)'), + dest='force', + default=False) + + parser.set_defaults(reporter=None) + subparsers = parser.add_subparsers() + + # Each action and its sub-options (if any) + parser_init = subparsers.add_parser('init', + help=('initializes cloud-init and' + ' performs initial modules')) + parser_init.add_argument("--local", '-l', action='store_true', + help="start in local mode (default: %(default)s)", + default=False) + # This is used so that we can know which action is selected + + # the functor to use to run this subcommand + parser_init.set_defaults(action=('init', main_init)) + + # These settings are used for the 'config' and 'final' stages + parser_mod = subparsers.add_parser('modules', + help=('activates modules using ' + 'a given configuration key')) + parser_mod.add_argument("--mode", '-m', action='store', + help=("module configuration name " + "to use (default: %(default)s)"), + default='config', + choices=('init', 'config', 'final')) + parser_mod.set_defaults(action=('modules', main_modules)) + + # These settings are used when you want to query information + # stored in the cloud-init data objects/directories/files + parser_query = subparsers.add_parser('query', + help=('query information stored ' + 'in cloud-init')) + parser_query.add_argument("--name", '-n', action="store", + help="item name to query on", + required=True, + choices=QUERY_DATA_TYPES) + parser_query.set_defaults(action=('query', main_query)) + + # This subcommand allows you to run a single module + parser_single = subparsers.add_parser('single', + help=('run a single module ')) + parser_single.set_defaults(action=('single', main_single)) + parser_single.add_argument("--name", '-n', action="store", + help="module name to run", + required=True) + parser_single.add_argument("--frequency", action="store", + help=("frequency of the module"), + required=False, + choices=list(FREQ_SHORT_NAMES.keys())) + parser_single.add_argument("--report", action="store_true", + help="enable reporting", + required=False) + parser_single.add_argument("module_args", nargs="*", + metavar='argument', + help=('any additional arguments to' + ' pass to this module')) + parser_single.set_defaults(action=('single', main_single)) + + args = parser.parse_args(args=sysv_args) + + # Setup basic logging to start (until reinitialized) + # iff in debug mode... + if args.debug: + logging.setupBasicLogging() + + # Setup signal handlers before running + signal_handler.attach_handlers() + + (name, functor) = args.action + + if name in ("modules", "init"): + functor = status_wrapper + + report_on = True + if name == "init": + if args.local: + rname, rdesc = ("init-local", "searching for local datasources") + else: + rname, rdesc = ("init-network", + "searching for network datasources") + elif name == "modules": + rname, rdesc = ("modules-%s" % args.mode, + "running modules for %s" % args.mode) + elif name == "single": + rname, rdesc = ("single/%s" % args.name, + "running single module %s" % args.name) + report_on = args.report + + args.reporter = events.ReportEventStack( + rname, rdesc, reporting_enabled=report_on) + with args.reporter: + return util.log_time( + logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, + get_uptime=True, func=functor, args=(name, args)) diff --git a/setup.py b/setup.py index 8c8f2402..0af576a9 100755 --- a/setup.py +++ b/setup.py @@ -204,10 +204,14 @@ setuptools.setup( author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', packages=setuptools.find_packages(exclude=['tests']), - scripts=['bin/cloud-init', - 'tools/cloud-init-per'], + scripts=['tools/cloud-init-per'], license='GPLv3', data_files=data_files, install_requires=requirements, cmdclass=cmdclass, + entry_points={ + 'console_scripts': [ + 'cloud-init = cloudinit.cmd.main:main' + ], + } ) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 163ce64c..8268cd5e 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -6,7 +6,7 @@ import sys from . import helpers as test_helpers mock = test_helpers.mock -BIN_CLOUDINIT = "bin/cloud-init" +from cloudinit.cmd import main as cli class TestCLI(test_helpers.FilesystemMockingTestCase): @@ -15,35 +15,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): super(TestCLI, self).setUp() self.stderr = six.StringIO() self.patchStdoutAndStderr(stderr=self.stderr) - self.sys_exit = mock.MagicMock() - self.patched_funcs.enter_context( - mock.patch.object(sys, 'exit', self.sys_exit)) - - def _call_main(self): - self.patched_funcs.enter_context( - mock.patch.object(sys, 'argv', ['cloud-init'])) - cli = imp.load_module( - 'cli', open(BIN_CLOUDINIT), '', ('', 'r', imp.PY_SOURCE)) + + def _call_main(self, sysv_args=None): + if not sysv_args: + sysv_args = ['cloud-init'] try: - return cli.main() - except Exception: - pass + return cli.main(sysv_args=sysv_args) + except SystemExit as e: + return e.code - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") def test_no_arguments_shows_usage(self): - self._call_main() - self.assertIn('usage: cloud-init', self.stderr.getvalue()) - - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") - def test_no_arguments_exits_2(self): exit_code = self._call_main() - if self.sys_exit.call_count: - self.assertEqual(mock.call(2), self.sys_exit.call_args) - else: - self.assertEqual(2, exit_code) + self.assertIn('usage: cloud-init', self.stderr.getvalue()) + self.assertEqual(2, exit_code) - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") def test_no_arguments_shows_error_message(self): - self._call_main() + exit_code = self._call_main() self.assertIn('cloud-init: error: too few arguments', self.stderr.getvalue()) + self.assertEqual(2, exit_code) -- cgit v1.2.3 From 628eaedba94dae953f6e0c0469881d9eb417cafe Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 15:00:05 -0700 Subject: Retain the prior attribute missing handling --- cloudinit/cmd/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 27b7b56d..c8916e03 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -654,7 +654,10 @@ def main(sysv_args=None): # Setup signal handlers before running signal_handler.attach_handlers() - (name, functor) = args.action + try: + (name, functor) = args.action + except AttributeError: + parser.error('too few arguments') if name in ("modules", "init"): functor = status_wrapper -- cgit v1.2.3 From 4dbfc55a6c90aee390ab981b0d4775d8b94e4803 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 14 Jun 2016 15:00:40 -0700 Subject: Don't continue running with no action --- cloudinit/cmd/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index c8916e03..2bd75328 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -646,6 +646,11 @@ def main(sysv_args=None): args = parser.parse_args(args=sysv_args) + try: + (name, functor) = args.action + except AttributeError: + parser.error('too few arguments') + # Setup basic logging to start (until reinitialized) # iff in debug mode... if args.debug: @@ -654,11 +659,6 @@ def main(sysv_args=None): # Setup signal handlers before running signal_handler.attach_handlers() - try: - (name, functor) = args.action - except AttributeError: - parser.error('too few arguments') - if name in ("modules", "init"): functor = status_wrapper -- cgit v1.2.3 From 946b0f5542ffb43149f60b5dd50b2fb450d23713 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 15 Jun 2016 10:31:44 +0100 Subject: Re-apply "Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche]" This commit includes the content of that commit, plus a fix for the tests (provided by Phil). --- ChangeLog | 1 + cloudinit/sources/DataSourceGCE.py | 2 +- tests/unittests/test_datasource/test_gce.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index e2869522..66539792 100644 --- a/ChangeLog +++ b/ChangeLog @@ -122,6 +122,7 @@ - ConfigDrive: improved support for networking information from a network_data.json or older interfaces formated network_config. - Change missing Cheetah log warning to debug [Andrew Jorgensen] + - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] 0.7.6: - open 0.7.6 diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 9234d1f8..c660a350 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,7 +25,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 1f7eb99e..6e62a4d2 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -52,7 +52,7 @@ GCE_META_ENCODING = { HEADERS = {'X-Google-Metadata-Request': 'True'} MD_URL_RE = re.compile( - r'http://metadata.google.internal./computeMetadata/v1/.*') + r'http://metadata.google.internal/computeMetadata/v1/.*') def _set_mock_metadata(gce_meta=None): -- cgit v1.2.3 From 2c5c2d7a500ccb88eee245a1d5f1ca1f9a2156e6 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 12:14:27 -0700 Subject: Silence pep8 warnings due to patcher activation --- cloudinit/cmd/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 2bd75328..63621c1d 100755 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -30,7 +30,7 @@ import time import traceback from cloudinit import patcher -patcher.patch() +patcher.patch() # noqa from cloudinit import log as logging from cloudinit import netinfo -- cgit v1.2.3 From 417fda64958f04c705860b737787fc921b965ade Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 12:17:05 -0700 Subject: Fix a few tools and tests for newer pep8 --- tests/unittests/test_cli.py | 3 ++- tools/run-pep8 | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 8268cd5e..a53e7116 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -4,10 +4,11 @@ import six import sys from . import helpers as test_helpers -mock = test_helpers.mock from cloudinit.cmd import main as cli +mock = test_helpers.mock + class TestCLI(test_helpers.FilesystemMockingTestCase): diff --git a/tools/run-pep8 b/tools/run-pep8 index 086400fc..4bd0bbfb 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,8 +1,7 @@ #!/bin/bash -pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) -# FIXME: cloud-init modifies sys module path, pep8 does not like -# bin_files=( "bin/cloud-init" ) +pycheck_dirs=( "cloudinit/" "tests/" "tools/" ) + CR=" " [ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose="" -- cgit v1.2.3 From 3d4509659b0408a8508a7be6eac8a77f10dc4c75 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 12:17:42 -0700 Subject: Remove /bin from run-pyflakes --- tools/run-pyflakes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run-pyflakes b/tools/run-pyflakes index 4bea17f4..b3759a94 100755 --- a/tools/run-pyflakes +++ b/tools/run-pyflakes @@ -3,7 +3,7 @@ PYTHON_VERSION=${PYTHON_VERSION:-2} CR=" " -pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) +pycheck_dirs=( "cloudinit/" "tests/" "tools/" ) set -f if [ $# -eq 0 ]; then -- cgit v1.2.3 From b77f11b35bf0978247fc88d0f4239c5bd4f65d66 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Jun 2016 12:19:12 -0700 Subject: Remove some unused imports --- tests/unittests/test_cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index a53e7116..5fa252f7 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -1,7 +1,4 @@ -import imp -import os import six -import sys from . import helpers as test_helpers -- cgit v1.2.3 From 4f989cc595775ab6b06fac604218d77910f3e4f4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 21:08:04 -0400 Subject: fis some Datasourcenocloud issues LP: #1592505 --- cloudinit/sources/DataSourceNoCloud.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 7e30118c..19d63950 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,7 +24,7 @@ import errno import os from cloudinit import log as logging -from cloudinit import net +from cloudinit.net import eni from cloudinit import sources from cloudinit import util @@ -194,8 +194,7 @@ class DataSourceNoCloud(sources.DataSource): # LP: #1568150 need getattr in the case that an old class object # has been loaded from a pickled file and now executing new source. dirs = getattr(self, 'seed_dirs', [self.seed_dir]) - quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, - dirs=dirs) + quick_id = _quick_read_instance_id(dirs=dirs) if not quick_id: return None return quick_id == current @@ -203,20 +202,19 @@ class DataSourceNoCloud(sources.DataSource): @property def network_config(self): if self._network_config is None: - if self.network_eni is not None: - self._network_config = net.convert_eni_data(self.network_eni) + if self._network_eni is not None: + self._network_config = eni.convert_eni_data(self._network_eni) return self._network_config -def _quick_read_instance_id(cmdline_id, dirs=None): +def _quick_read_instance_id(dirs=None): if dirs is None: dirs = [] iid_key = 'instance-id' - if cmdline_id is None: - fill = {} - if parse_cmdline_data(cmdline_id, fill) and iid_key in fill: - return fill[iid_key] + fill = {} + if load_cmdline_data(fill) and iid_key in fill: + return fill[iid_key] for d in dirs: if d is None: -- cgit v1.2.3 From d4b587ebf500ddc2259fffc94a3c69c199c9a427 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 22:50:12 -0400 Subject: fix some errors reported by pylint pylint --errors-only found several errors. Some of the changes here represent real errors, others just code that pylint did not like. --- cloudinit/config/cc_apt_configure.py | 3 ++- cloudinit/config/cc_growpart.py | 10 +++++----- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/arch.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 10 +++++----- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/sources/helpers/openstack.py | 2 +- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 96c4a43d..05ad4b03 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -208,8 +208,9 @@ def add_apt_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - def aa_repo_match(x): + def _aa_repo_match(x): return False + aa_repo_match = _aa_repo_match errorlist = [] srcdict = convert_to_new_format(srclist) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 859d69f1..40560f11 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -36,13 +36,13 @@ DEFAULT_CONFIG = { } -def enum(**enums): - return type('Enum', (), enums) +class RESIZE(object): + SKIPPED = "SKIPPED" + CHANGED = "CHANGED" + NOCHANGE = "NOCHANGE" + FAILED = "FAILED" -RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE", - FAILED="FAILED") - LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5c29c804..14b500f8 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -454,7 +454,7 @@ class Distro(object): keys = kwargs['ssh_authorized_keys'] if isinstance(keys, six.string_types): keys = [keys] - if isinstance(keys, dict): + elif isinstance(keys, dict): keys = list(keys.values()) if keys is not None: if not isinstance(keys, (tuple, list, set)): diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 93a2e008..66209f22 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -196,6 +196,6 @@ def convert_resolv_conf(settings): """Returns a settings string formatted for resolv.conf.""" result = '' if isinstance(settings, list): - for ns in list: + for ns in settings: result = result + 'nameserver %s\n' % ns - return result + return result diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index f2bb9366..43347cfb 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -37,16 +37,16 @@ from cloudinit.sources.helpers.vmware.imc.config_file \ import ConfigFile from cloudinit.sources.helpers.vmware.imc.config_nic \ import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum from cloudinit.sources.helpers.vmware.imc.guestcust_event \ import GuestCustEventEnum from cloudinit.sources.helpers.vmware.imc.guestcust_state \ import GuestCustStateEnum -from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ - import GuestCustErrorEnum -from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( - set_customization_status, +from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( + enable_nics, get_nics_to_enable, - enable_nics + set_customization_status ) LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 8f85b115..7b3a76b9 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -104,7 +104,7 @@ class DataSourceOpenNebula(sources.DataSource): def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == sources.DSMODE_NET: + if self.dsmode == sources.DSMODE_NETWORK: resolve_ip = True else: resolve_ip = False diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 494335b3..d52cb56a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -157,7 +157,7 @@ class BaseReader(object): pass @abc.abstractmethod - def _path_read(self, path): + def _path_read(self, path, decode=False): pass @abc.abstractmethod -- cgit v1.2.3 From b2fd5b65dc89c3fd5dbc1414968505367fc7126a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 23:03:11 -0400 Subject: fix usage of OSError.message that will not work in python3 python3's OSError does not have a .message attribute. --- cloudinit/sources/DataSourceAltCloud.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index cd61df31..a3529609 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -205,8 +205,7 @@ class DataSourceAltCloud(sources.DataSource): _err.message) return False except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False floppy_dev = '/dev/fd0' -- cgit v1.2.3 From 8e5f63cb2f11ce08f7de1df23b8937305b91b707 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 23:16:35 -0400 Subject: remove declaration of dsmode as local in DataSourceNoCloud this would cause the datasource to operate in local mode by default. --- cloudinit/sources/DataSourceNoCloud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 19d63950..cdc9eef5 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -34,7 +34,6 @@ LOG = logging.getLogger(__name__) class DataSourceNoCloud(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] -- cgit v1.2.3