From a1b185d0cce5064e9b36b4db7b55564e2ab1d7a8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 20 Oct 2016 22:53:17 -0400 Subject: Get early logging logged, including failures of cmdline url. Failures to load the kernel command line's url (cloud-config-url=) would previously get swallowed. This should make it much more obvious when that happens. With logging going to expected places at sane levels (WARN will go to stderr by default). --- cloudinit/cmd/main.py | 118 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 15 deletions(-) (limited to 'cloudinit/cmd') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index c83496c5..65b15edc 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -26,6 +26,7 @@ from cloudinit import signal_handler from cloudinit import sources from cloudinit import stages from cloudinit import templater +from cloudinit import url_helper from cloudinit import util from cloudinit import version @@ -129,23 +130,104 @@ def apply_reporting_cfg(cfg): reporting.update_configuration(cfg.get('reporting')) +def parse_cmdline_url(cmdline, names=('cloud-config-url', 'url')): + data = util.keyval_str_to_dict(cmdline) + for key in names: + if key in data: + return key, data[key] + raise KeyError("No keys (%s) found in string '%s'" % + (cmdline, names)) + + +def attempt_cmdline_url(path, network=True, cmdline=None): + """Write data from url referenced in command line to path. + + path: a file to write content to if downloaded. + network: should network access be assumed. + cmdline: the cmdline to parse for cloud-config-url. + + This is used in MAAS datasource, in "ephemeral" (read-only root) + environment where the instance netboots to iscsi ro root. + and the entity that controls the pxe config has to configure + the maas datasource. + + An attempt is made on network urls even in local datasource + for case of network set up in initramfs. + + Return value is a tuple of a logger function (logging.DEBUG) + and a message indicating what happened. + """ + + if cmdline is None: + cmdline = util.get_cmdline() + + try: + cmdline_name, url = parse_cmdline_url(cmdline) + except KeyError: + return (logging.DEBUG, "No kernel command line url found.") + + path_is_local = url.startswith("file://") or url.startswith("/") + + if path_is_local and os.path.exists(path): + if network: + m = ("file '%s' existed, possibly from local stage download" + " of command line url '%s'. Not re-writing." % (path, url)) + level = logging.INFO + if path_is_local: + level = logging.DEBUG + else: + m = ("file '%s' existed, possibly from previous boot download" + " of command line url '%s'. Not re-writing." % (path, url)) + level = logging.WARN + + return (level, m) + + kwargs = {'url': url, 'timeout': 10, 'retries': 2} + if network or path_is_local: + level = logging.WARN + kwargs['sec_between'] = 1 + else: + level = logging.DEBUG + kwargs['sec_between'] = .1 + + data = None + header = b'#cloud-config' + try: + resp = util.read_file_or_url(**kwargs) + if resp.ok(): + data = resp.contents + if not resp.contents.startswith(header): + if cmdline_name == 'cloud-config-url': + level = logging.WARN + else: + level = logging.INFO + return ( + level, + "contents of '%s' did not start with %s" % (url, header)) + else: + return (level, + "url '%s' returned code %s. Ignoring." % (url, resp.code)) + + except url_helper.UrlError as e: + return (level, "retrieving url '%s' failed: %s" % (url, e)) + + util.write_file(path, data, mode=0o600) + return (logging.INFO, + "wrote cloud-config data from %s='%s' to %s" % + (cmdline_name, url, path)) + + def main_init(name, args): deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] if args.local: deps = [sources.DEP_FILESYSTEM] - if not args.local: - # See doc/kernel-cmdline.txt - # - # This is used in maas datasource, in "ephemeral" (read-only root) - # environment where the instance netboots to iscsi ro root. - # and the entity that controls the pxe config has to configure - # the maas datasource. - # - # Could be used elsewhere, only works on network based (not local). - root_name = "%s.d" % (CLOUD_CONFIG) - target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") - util.read_write_cmdline_url(target_fn) + early_logs = [] + early_logs.append( + attempt_cmdline_url( + path=os.path.join("%s.d" % CLOUD_CONFIG, + "91_kernel_cmdline_url.cfg"), + network=not args.local)) # Cloud-init 'init' stage is broken up into the following sub-stages # 1. Ensure that the init object fetches its config without errors @@ -171,12 +253,14 @@ def main_init(name, args): outfmt = None errfmt = None try: - LOG.debug("Closing stdin") + early_logs.append((logging.DEBUG, "Closing stdin.")) util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) except Exception: - util.logexc(LOG, "Failed to setup output redirection!") - print_exc("Failed to setup output redirection!") + msg = "Failed to setup output redirection!" + util.logexc(LOG, msg) + print_exc(msg) + early_logs.append((logging.WARN, msg)) if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" @@ -190,6 +274,10 @@ def main_init(name, args): # been redirected and log now configured. welcome(name, msg=w_msg) + # re-play early log messages before logging was setup + for lvl, msg in early_logs: + LOG.log(lvl, msg) + # Stage 3 try: init.initialize() -- cgit v1.2.3 From e98709225510ee99ee0269c558c82b3e693e38e5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 4 Feb 2017 02:25:19 +0000 Subject: manual_cache_clean: When manually cleaning touch a file in instance dir. When manual_cache_clean is enabled, write a file to /var/lib/cloud/instance/manual-clean. That file can then be read by ds-identify or another tool to indicate that manual cleaning is in place. --- cloudinit/cmd/main.py | 9 ++++++++- cloudinit/helpers.py | 1 + cloudinit/stages.py | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) (limited to 'cloudinit/cmd') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 65b15edc..7c652574 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -312,8 +312,15 @@ def main_init(name, args): " would allow us to stop early.") else: existing = "check" - if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): + mcfg = util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False) + if mcfg: + LOG.debug("manual cache clean set from config") existing = "trust" + else: + mfile = path_helper.get_ipath_cur("manual_clean_marker") + if os.path.exists(mfile): + LOG.debug("manual cache clean found from marker: %s", mfile) + existing = "trust" init.purge_cache() # Delete the non-net file as well diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 4528fb01..38f5f899 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -339,6 +339,7 @@ class Paths(object): "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", "instance_id": ".instance-id", + "manual_clean_marker": "manual-clean", } # Set when a datasource becomes active self.datasource = ds diff --git a/cloudinit/stages.py b/cloudinit/stages.py index b0552dde..21763810 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -188,6 +188,12 @@ class Init(object): def _write_to_cache(self): if self.datasource is NULL_DATA_SOURCE: return False + if util.get_cfg_option_bool(self.cfg, 'manual_cache_clean', False): + # The empty file in instance/ dir indicates manual cleaning, + # and can be read by ds-identify. + util.write_file( + self.paths.get_ipath_cur("manual_clean_marker"), + omode="w", content="") return _pkl_store(self.datasource, self.paths.get_ipath_cur("obj_pkl")) def _get_datasources(self): -- cgit v1.2.3 From 79db2e2436d91510aceb8c036c4a945362c85a52 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Mar 2017 15:50:40 -0500 Subject: Support warning if the used datasource is not in ds-identify's list. If ds-identify is in report mode, and the datasource that is found is not in the list, then warn the user of this situation. --- cloudinit/cmd/main.py | 39 +++++++++++++++++++++++++++++++++++++++ cloudinit/warnings.py | 24 ++++++++++++++++++++++++ 2 files changed, 63 insertions(+) (limited to 'cloudinit/cmd') diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 7c652574..6ff4e1c0 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -29,6 +29,7 @@ from cloudinit import templater from cloudinit import url_helper from cloudinit import util from cloudinit import version +from cloudinit import warnings from cloudinit import reporting from cloudinit.reporting import events @@ -413,10 +414,48 @@ def main_init(name, args): # give the activated datasource a chance to adjust init.activate_datasource() + di_report_warn(datasource=init.datasource, cfg=init.cfg) + # Stage 10 return (init.datasource, run_module_section(mods, name, name)) +def di_report_warn(datasource, cfg): + if 'di_report' not in cfg: + LOG.debug("no di_report found in config.") + return + + dicfg = cfg.get('di_report', {}) + if not isinstance(dicfg, dict): + LOG.warn("di_report config not a dictionary: %s", dicfg) + return + + dslist = dicfg.get('datasource_list') + if dslist is None: + LOG.warn("no 'datasource_list' found in di_report.") + return + elif not isinstance(dslist, list): + LOG.warn("di_report/datasource_list not a list: %s", dslist) + return + + # ds.__module__ is like cloudinit.sources.DataSourceName + # where Name is the thing that shows up in datasource_list. + modname = datasource.__module__.rpartition(".")[2] + if modname.startswith(sources.DS_PREFIX): + modname = modname[len(sources.DS_PREFIX):] + else: + LOG.warn("Datasource '%s' came from unexpected module '%s'.", + datasource, modname) + + if modname in dslist: + LOG.debug("used datasource '%s' from '%s' was in di_report's list: %s", + datasource, modname, dslist) + return + + warnings.show_warning('dsid_missing_source', cfg, + source=modname, dslist=str(dslist)) + + def main_modules(action_name, args): name = args.mode # Cloud-init 'modules' stages are broken up into the following sub-stages diff --git a/cloudinit/warnings.py b/cloudinit/warnings.py index 77c092f9..3206d4e9 100644 --- a/cloudinit/warnings.py +++ b/cloudinit/warnings.py @@ -35,6 +35,30 @@ putting that content into datasource: Ec2: strict_id: false""", + 'dsid_missing_source': """ +A new feature in cloud-init identified possible datasources for +this system as: + {dslist} +However, the datasource used was: {source} + +In the future, cloud-init will only attempt to use datasources that +are identified or specifically configured. +For more information see + https://bugs.launchpad.net/bugs/1669675 + +If you are seeing this message, please file a bug against +cloud-init at + https://bugs.launchpad.net/cloud-init/+filebug?field.tags=dsid +Make sure to include the cloud provider your instance is +running on. + +After you have filed a bug, you can disable this warning by launching +your instance with the cloud-config below, or putting that content +into /etc/cloud/cloud.cfg.d/99-warnings.cfg + +#cloud-config +warnings: + dsid_missing_source: off""", } -- cgit v1.2.3