diff options
149 files changed, 5089 insertions, 2836 deletions
@@ -105,6 +105,27 @@ - centos: Ensure that resolve conf object is written as a str (LP: #1479988) - chef: straighten out validation_cert and validation_key (LP: #1568940) - phone_home: allow usage of fqdn (LP: #1566824) [Ollie Armstrong] + - cloudstack: Only use DHCPv4 lease files as a datasource (LP: #1576273) + [Wido den Hollander] + - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) + [Robert Jennings] + - Ec2: do not retry requests for user-data path on 404. + - settings on the kernel command line (cc:) override all local settings + rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) + - Improve merging documentation [Daniel Watkins] + - apt sources: support inserting key/key-id only, custom sources.list, + long gpg key fingerprints with spaces, and dictionary format (LP: #1574113) + - SmartOS: datasource improvements and support for metadata service + providing networking information. + - Datasources: centrally handle 'dsmode' and no longer require datasources + to "pass" if modules_init should be executed with network access. + - ConfigDrive: improved support for networking information from + a network_data.json or older interfaces formated network_config. + - Change missing Cheetah log warning to debug [Andrew Jorgensen] + - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] + - support network rendering to sysconfig (for centos and RHEL) + - write_files: if no permissions are given, just use default without warn. + - user_data: fix error when user-data is not utf-8 decodable (LP: #1532072) 0.7.6: - open 0.7.6 diff --git a/cloudinit/cmd/__init__.py b/cloudinit/cmd/__init__.py new file mode 100644 index 00000000..da124641 --- /dev/null +++ b/cloudinit/cmd/__init__.py @@ -0,0 +1,21 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/bin/cloud-init b/cloudinit/cmd/main.py index 5857af32..63621c1d 100755..100644 --- a/bin/cloud-init +++ b/cloudinit/cmd/main.py @@ -25,19 +25,12 @@ import argparse import json import os import sys -import time import tempfile +import time import traceback -# This is more just for running from the bin folder so that -# cloud-init binary can find the cloudinit module -possible_topdir = os.path.normpath(os.path.join(os.path.abspath( - sys.argv[0]), os.pardir, os.pardir)) -if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")): - sys.path.insert(0, possible_topdir) - from cloudinit import patcher -patcher.patch() +patcher.patch() # noqa from cloudinit import log as logging from cloudinit import netinfo @@ -46,9 +39,10 @@ from cloudinit import sources from cloudinit import stages from cloudinit import templater from cloudinit import util +from cloudinit import version + from cloudinit import reporting from cloudinit.reporting import events -from cloudinit import version from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, CLOUD_CONFIG) @@ -188,7 +182,7 @@ def main_init(name, args): LOG.debug("Closing stdin") util.close_stdin() (outfmt, errfmt) = util.fixup_output(init.cfg, name) - except: + except Exception: util.logexc(LOG, "Failed to setup output redirection!") print_exc("Failed to setup output redirection!") if args.debug: @@ -211,27 +205,27 @@ def main_init(name, args): util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths - if not args.local: + mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK + + if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug(("Checking to see if files that we need already" " exist from a previous run that would allow us" " to stop early.")) + # no-net is written by upstart cloud-init-nonet when network failed + # to come up stop_files = [ os.path.join(path_helper.get_cpath("data"), "no-net"), - path_helper.get_ipath_cur("obj_pkl"), ] existing_files = [] for fn in stop_files: - try: - c = util.load_file(fn) - if len(c): - existing_files.append((fn, len(c))) - except Exception: - pass + if os.path.isfile(fn): + existing_files.append(fn) + if existing_files: - LOG.debug("Exiting early due to the existence of %s files", - existing_files) + LOG.debug("[%s] Exiting. stop file %s existed", + mode, existing_files) return (None, []) else: LOG.debug("Execution continuing, no previous run detected that" @@ -248,34 +242,50 @@ def main_init(name, args): # Stage 5 try: init.fetch(existing=existing) + # if in network mode, and the datasource is local + # then work was done at that stage. + if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s in local mode", + mode, init.datasource) + return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. When using upstart it will also mentions job failure # in console log if exit code is != 0. - if args.local: + if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: - init.apply_network_config() - if args.local: + init.apply_network_config(bring_up=not args.local) + LOG.debug("[%s] Exiting without datasource in local mode", mode) + if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) - - if args.local: - if not init.ds_restored: - # if local mode and the datasource was not restored from cache - # (this is not first boot) then apply networking. - init.apply_network_config() else: - LOG.debug("skipping networking config from restored datasource.") + LOG.debug("[%s] barreling on in force mode without datasource", + mode) # Stage 6 iid = init.instancify() - LOG.debug("%s will now be targeting instance id: %s", name, iid) + LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", + mode, name, iid, init.is_new_instance()) + + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) + + if mode == sources.DSMODE_LOCAL: + if init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s not in local mode.", + mode, init.datasource) + return (init.datasource, []) + else: + LOG.debug("[%s] %s is in local mode, will apply init modules now.", + mode, init.datasource) + + # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: @@ -309,7 +319,7 @@ def main_init(name, args): if outfmt_orig != outfmt or errfmt_orig != errfmt: LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt) (outfmt, errfmt) = util.fixup_output(mods.cfg, name) - except: + except Exception: util.logexc(LOG, "Failed to re-adjust output redirection!") logging.setupLogging(mods.cfg) @@ -351,7 +361,7 @@ def main_modules(action_name, args): LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, name) - except: + except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out @@ -414,7 +424,7 @@ def main_single(name, args): LOG.debug("Closing stdin") util.close_stdin() util.fixup_output(mods.cfg, None) - except: + except Exception: util.logexc(LOG, "Failed to setup output redirection!") if args.debug: # Reset so that all the debug handlers are closed out @@ -494,7 +504,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): else: try: status = json.loads(util.load_file(status_path)) - except: + except Exception: pass if status is None: @@ -528,7 +538,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): v1[mode]['errors'] = [str(e) for e in errors] except Exception as e: - util.logexc(LOG, "failed of stage %s", mode) + util.logexc(LOG, "failed stage %s", mode) print_exc("failed run of stage %s" % mode) v1[mode]['errors'] = [str(e)] @@ -553,8 +563,12 @@ def status_wrapper(name, args, data_d=None, link_d=None): return len(v1[mode]['errors']) -def main(): - parser = argparse.ArgumentParser() +def main(sysv_args=None): + if sysv_args is not None: + parser = argparse.ArgumentParser(prog=sysv_args[0]) + sysv_args = sysv_args[1:] + else: + parser = argparse.ArgumentParser() # Top level args parser.add_argument('--version', '-v', action='version', @@ -630,7 +644,12 @@ def main(): ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) - args = parser.parse_args() + args = parser.parse_args(args=sysv_args) + + try: + (name, functor) = args.action + except AttributeError: + parser.error('too few arguments') # Setup basic logging to start (until reinitialized) # iff in debug mode... @@ -640,9 +659,6 @@ def main(): # Setup signal handlers before running signal_handler.attach_handlers() - if not hasattr(args, 'action'): - parser.error('too few arguments') - (name, functor) = args.action if name in ("modules", "init"): functor = status_wrapper @@ -667,7 +683,3 @@ def main(): return util.log_time( logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, get_uptime=True, func=functor, args=(name, args)) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..05ad4b03 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -22,6 +22,7 @@ import glob import os import re +from cloudinit import gpg from cloudinit import templater from cloudinit import util @@ -34,21 +35,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" -# A temporary shell program to get a given gpg key -# from a given keyserver -EXPORT_GPG_KEYID = """ - k=${1} ks=${2}; - exec 2>/dev/null - [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") - if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && - armour=$(gpg --export --armour "${k}") && - gpg --batch --yes --delete-keys "${k}" - fi - [ -n "${armour}" ] && echo "${armour}" -""" - def handle(name, cfg, cloud, log, _args): if util.is_false(cfg.get('apt_configure_enabled', True)): @@ -70,7 +56,7 @@ def handle(name, cfg, cloud, log, _args): if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirrors, cloud, log) + generate_sources_list(cfg, release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', {"primary": "archive.ubuntu.com/ubuntu", "security": "security.ubuntu.com/ubuntu"}) @@ -94,8 +80,8 @@ def handle(name, cfg, cloud, log, _args): def matcher(x): return False - errors = add_sources(cfg['apt_sources'], params, - aa_repo_match=matcher) + errors = add_apt_sources(cfg['apt_sources'], params, + aa_repo_match=matcher) for e in errors: log.warn("Add source error: %s", ':'.join(e)) @@ -108,17 +94,7 @@ def handle(name, cfg, cloud, log, _args): util.logexc(log, "Failed to run debconf-set-selections") -# get gpg keyid from keyserver -def getkeybyid(keyid, keyserver): - with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh: - fh.write(EXPORT_GPG_KEYID) - fh.flush() - cmd = ['/bin/sh', fh.name, keyid, keyserver] - (stdout, _stderr) = util.subp(cmd) - return stdout.strip() - - -def mirror2lists_fileprefix(mirror): +def mirrorurl_to_apt_fileprefix(mirror): string = mirror # take off http:// or ftp:// if string.endswith("/"): @@ -135,8 +111,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): nmirror = new_mirrors.get(name) if not nmirror: continue - oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror)) + oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror)) + nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror)) if oprefix == nprefix: continue olen = len(oprefix) @@ -149,7 +125,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirrors, cloud, log): +def generate_sources_list(cfg, codename, mirrors, cloud, log): + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + + custtmpl = cfg.get('apt_custom_sources_list', None) + if custtmpl is not None: + templater.render_string_to_file(custtmpl, + '/etc/apt/sources.list', params) + return + template_fn = cloud.get_template_filename('sources.list.%s' % (cloud.distro.name)) if not template_fn: @@ -158,13 +144,61 @@ def generate_sources_list(codename, mirrors, cloud, log): log.warn("No template found, not rendering /etc/apt/sources.list") return - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_sources(srclist, template_params=None, aa_repo_match=None): +def add_apt_key_raw(key): + """ + actual adding of a key as defined in key argument + to the system + """ + try: + util.subp(('apt-key', 'add', '-'), key) + except util.ProcessExecutionError: + raise ValueError('failed to add apt GPG Key to apt keyring') + + +def add_apt_key(ent): + """ + add key to the system as defined in ent (if any) + supports raw keys or keyid's + The latter will as a first step fetch the raw key from a keyserver + """ + if 'keyid' in ent and 'key' not in ent: + keyserver = "keyserver.ubuntu.com" + if 'keyserver' in ent: + keyserver = ent['keyserver'] + ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver) + + if 'key' in ent: + add_apt_key_raw(ent['key']) + + +def convert_to_new_format(srclist): + """convert_to_new_format + convert the old list based format to the new dict based one + """ + srcdict = {} + if isinstance(srclist, list): + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + elif isinstance(srclist, dict): + srcdict = srclist + else: + raise ValueError("unknown apt_sources format") + + return srcdict + + +def add_apt_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated sources.list entry in 'srclist'. When rendering template, also @@ -174,18 +208,34 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - def aa_repo_match(x): + def _aa_repo_match(x): return False + aa_repo_match = _aa_repo_match errorlist = [] - for ent in srclist: + srcdict = convert_to_new_format(srclist) + + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent['filename'] = filename + + # keys can be added without specifying a source + try: + add_apt_key(ent) + except ValueError as detail: + errorlist.append([ent, detail]) + if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith(os.path.sep): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -194,33 +244,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], ks) - except: - errorlist.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - errorlist.append([source, "failed add key"]) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") - except: + except Exception: errorlist.append([source, "failed write to file %s" % ent['filename']]) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index a295cc4e..b763a3c3 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -38,7 +38,7 @@ def handle(name, cfg, cloud, log, _args): content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except: + except Exception: util.logexc(log, "Failed to shellify bootcmd") raise @@ -49,6 +49,6 @@ def handle(name, cfg, cloud, log, _args): env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) - except: + except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index bbaf9646..b642f1f8 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -198,7 +198,7 @@ def is_device_valid(name, partition=False): d_type = "" try: d_type = device_type(name) - except: + except Exception: LOG.warn("Query against device %s failed" % name) return False diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 86ae97ab..98828b9e 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -20,8 +20,8 @@ import os -from cloudinit.settings import PER_ALWAYS from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS from cloudinit import util frequency = PER_ALWAYS @@ -56,7 +56,7 @@ def handle(name, _cfg, cloud, log, args): event_names = ['cloud-config'] if not is_upstart_system(): - log.debug("not upstart system, '%s' disabled") + log.debug("not upstart system, '%s' disabled", name) return cfgpath = cloud.paths.get_ipath_cur("cloud_config") diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 39e3850e..545fee22 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -37,8 +37,8 @@ If cloud-init sees a 'fan' entry in cloud-config it will """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 4a51476f..c9021eb1 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -66,7 +66,7 @@ def handle(_name, cfg, cloud, log, args): try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) util.write_file(boot_fin_fn, contents) - except: + except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) if cloud.datasource.is_disconnected: diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 859d69f1..40560f11 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -36,13 +36,13 @@ DEFAULT_CONFIG = { } -def enum(**enums): - return type('Enum', (), enums) +class RESIZE(object): + SKIPPED = "SKIPPED" + CHANGED = "CHANGED" + NOCHANGE = "NOCHANGE" + FAILED = "FAILED" -RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE", - FAILED="FAILED") - LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 3c2d9985..156722d9 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -69,5 +69,5 @@ def handle(name, cfg, _cloud, log, _args): try: util.subp(['debconf-set-selections'], dconf_sel) - except: + except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index aa844ee9..9a02f056 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -57,6 +57,6 @@ def handle(name, cfg, cloud, log, _args): (stdout, _stderr) = util.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) - except: + except Exception: log.warn("Writing keys to the system console failed!") raise diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index bf735648..70d4e7c3 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -52,7 +52,8 @@ def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') if not lxd_cfg: - log.debug("Skipping module named %s, not present or disabled by cfg") + log.debug("Skipping module named %s, not present or disabled by cfg", + name) return if not isinstance(lxd_cfg, dict): log.warn("lxd config must be a dictionary. found a '%s'", @@ -111,7 +112,7 @@ def handle(name, cfg, cloud, log, args): data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" util.subp(['debconf-communicate'], data) - except: + except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) # Remove the existing configuration file (forces re-generation) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 4fe3ee21..2b981935 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -244,7 +244,7 @@ def handle_swapcfg(swapcfg): LOG.debug("swap file %s already in use.", fname) return fname LOG.debug("swap file %s existed, but not in /proc/swaps", fname) - except: + except Exception: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) return fname @@ -379,7 +379,7 @@ def handle(_name, cfg, cloud, log, _args): toks = WS.split(line) if toks[3].find(comment) != -1: continue - except: + except Exception: pass fstab_lines.append(line) @@ -390,16 +390,16 @@ def handle(_name, cfg, cloud, log, _args): if needswap: try: util.subp(("swapon", "-a")) - except: + except Exception: util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: try: util.ensure_dir(d) - except: + except Exception: util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) - except: + except Exception: util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 3dcc9459..72176d42 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, args): tries = ph_cfg.get('tries') try: tries = int(tries) - except: + except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) @@ -87,7 +87,7 @@ def handle(name, cfg, cloud, log, args): for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) - except: + except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) @@ -117,6 +117,6 @@ def handle(name, cfg, cloud, log, args): util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) - except: + except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 0ecf3a4d..8118fac4 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -52,7 +52,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() - except: + except Exception: log.debug("Failed to get raw userdata in module %s", name) return @@ -63,7 +63,7 @@ def handle(name, _cfg, cloud, log, _args): "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) return - except: + except Exception: util.logexc(log, "Failed to parse query string %s into a dictionary", ud) raise diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 66dc3363..bc09d38c 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -34,5 +34,5 @@ def handle(name, cfg, cloud, log, _args): try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) - except: + except Exception: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 42b987eb..ee3b6c9f 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index b5d71c13..c0d62b12 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index d77d36d5..ecb527f6 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 5c53014f..699857d1 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -36,7 +36,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index 0c9e504e..80bf10ff 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -37,7 +37,7 @@ def handle(name, cfg, cloud, log, _args): try: util.runparts(runparts_path, exe_prefix=prefix) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 1b011216..5085c23a 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -24,8 +24,8 @@ import os from six import BytesIO -from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 58e1b713..5c8c23b8 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -155,7 +155,7 @@ def handle(_name, cfg, cloud, log, args): cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") - except: + except Exception: util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index fa9d54a0..1a485ee6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,12 +47,12 @@ Example config: """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util import glob -import tempfile import os +import tempfile LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index d24e43c0..cb9b70aa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -57,7 +57,7 @@ def handle(_name, cfg, cloud, log, _args): for f in glob.glob(key_pth): try: util.del_file(f) - except: + except Exception: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) - except: + except Exception: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: @@ -122,7 +122,7 @@ def handle(_name, cfg, cloud, log, _args): keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) - except: + except Exception: util.logexc(log, "Applying ssh credentials failed!") diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 2d480d7e..28c4585b 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -52,14 +52,14 @@ def handle(_name, cfg, cloud, log, args): else: try: import_ids = user_cfg['ssh_import_id'] - except: + except Exception: log.debug("User %s is not configured for ssh_import_id", user) continue try: import_ids = util.uniq_merge(import_ids) import_ids = [str(i) for i in import_ids] - except: + except Exception: log.debug("User %s is not correctly configured for ssh_import_id", user) continue diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 7e88ed85..884d79f1 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -40,10 +40,10 @@ It can be configured with the following option structure:: mechanism you've used to switch the init system. """ -from cloudinit.settings import PER_INSTANCE +from cloudinit.distros import ubuntu from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util -from cloudinit.distros import ubuntu import os import time diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 351cfc8c..b1096b9b 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -79,6 +79,8 @@ def write_files(name, files, log): def decode_perms(perm, default, log): + if perm is None: + return default try: if isinstance(perm, six.integer_types + (float,)): # Just 'downcast' it (if a float) diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py index 83ac1a0e..412431f2 100644 --- a/cloudinit/cs_utils.py +++ b/cloudinit/cs_utils.py @@ -33,7 +33,8 @@ API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html import json import platform -import serial +from cloudinit import serial + # these high timeouts are necessary as read may read a lot of data. READ_TIMEOUT = 60 diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5879dabf..14b500f8 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -31,6 +31,7 @@ import stat from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -50,8 +51,8 @@ OSFAMILIES = { LOG = logging.getLogger(__name__) +@six.add_metaclass(abc.ABCMeta) class Distro(object): - __metaclass__ = abc.ABCMeta usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -97,7 +98,7 @@ class Distro(object): try: res = os.lstat('/run/systemd/system') return stat.S_ISDIR(res.st_mode) - except: + except Exception: return False @abc.abstractmethod @@ -128,6 +129,8 @@ class Distro(object): mirror_info=arch_info) def apply_network(self, settings, bring_up=True): + # this applies network where 'settings' is interfaces(5) style + # it is obsolete compared to apply_network_config # Write it out dev_names = self._write_network(settings) # Now try to bring them up @@ -143,6 +146,9 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_network_config_names(self, netconfig): + net.apply_network_config_names(netconfig) + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() @@ -448,7 +454,7 @@ class Distro(object): keys = kwargs['ssh_authorized_keys'] if isinstance(keys, six.string_types): keys = [keys] - if isinstance(keys, dict): + elif isinstance(keys, dict): keys = list(keys.values()) if keys is not None: if not isinstance(keys, (tuple, list, set)): diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 93a2e008..66209f22 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -196,6 +196,6 @@ def convert_resolv_conf(settings): """Returns a settings string formatted for resolv.conf.""" result = '' if isinstance(settings, list): - for ns in list: + for ns in settings: result = result + 'nameserver %s\n' % ns - return result + return result diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 75ab340f..5ae9a509 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,9 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit.net import eni +from cloudinit.net.network_state import parse_net_config_data from cloudinit import util -from cloudinit import net from cloudinit.distros.parsers.hostname import HostnameConf @@ -42,6 +43,13 @@ APT_GET_WRAPPER = { 'enabled': 'auto', } +ENI_HEADER = """# This file is generated from information provided by +# the datasource. Changes to it will not persist across an instance. +# To disable cloud-init's network configuration capabilities, write a file +# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: +# network: {config: disabled} +""" + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" @@ -56,6 +64,12 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'debian' + self._net_renderer = eni.Renderer({ + 'eni_path': self.network_conf_fn, + 'eni_header': ENI_HEADER, + 'links_prefix_path': None, + 'netrules_path': None, + }) def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -79,13 +93,9 @@ class Distro(distros.Distro): return ['all'] def _write_network_config(self, netconfig): - ns = net.parse_net_config_data(netconfig) - net.render_network_state(target="/", network_state=ns, - eni=self.network_conf_fn, - links_prefix=self.links_prefix, - netrules=None) + ns = parse_net_config_data(netconfig) + self._net_renderer.render_network_state("/", ns) _maybe_remove_legacy_eth0() - return [] def _bring_up_interfaces(self, device_names): @@ -221,7 +231,7 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): msg = "removed %s with known contents" % path else: msg = (bmsg + " '%s' exists with user configured content." % path) - except: + except Exception: msg = bmsg + " %s exists, but could not be read." % path LOG.warn(msg) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 812e7002..1aa42d75 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -23,6 +23,8 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging +from cloudinit.net.network_state import parse_net_config_data +from cloudinit.net import sysconfig from cloudinit import util from cloudinit.distros import net_util @@ -59,10 +61,16 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'redhat' + self._net_renderer = sysconfig.Renderer() def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) + def _write_network_config(self, netconfig): + ns = parse_net_config_data(netconfig) + self._net_renderer.render_network_state("/", ns) + return [] + def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format entries = net_util.translate_network(settings) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 37b92a83..76dda042 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,7 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - if cause.code in status_codes: - return False - return True + return cause.code in status_codes def get_instance_userdata(api_version='latest', diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py new file mode 100644 index 00000000..6a76d785 --- /dev/null +++ b/cloudinit/gpg.py @@ -0,0 +1,74 @@ +"""gpg.py - Collection of gpg key related functions""" +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Christian Ehrhardt <christian.ehrhardt@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from cloudinit import log as logging +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +def export_armour(key): + """Export gpg key, armoured key gets returned""" + try: + (armour, _) = util.subp(["gpg", "--export", "--armour", key], + capture=True) + except util.ProcessExecutionError as error: + # debug, since it happens for any key not on the system initially + LOG.debug('Failed to export armoured key "%s": %s', key, error) + armour = None + return armour + + +def receive_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + LOG.debug('Receive gpg key "%s"', key) + try: + util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key], + capture=True) + except util.ProcessExecutionError as error: + raise ValueError(('Failed to import key "%s" ' + 'from server "%s" - error %s') % + (key, keyserver, error)) + + +def delete_key(key): + """Delete the specified key from the local gpg ring""" + try: + util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + capture=True) + except util.ProcessExecutionError as error: + LOG.warn('Failed delete key "%s": %s', key, error) + + +def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"): + """get gpg keyid from keyserver""" + armour = export_armour(keyid) + if not armour: + try: + receive_key(keyid, keyserver=keyserver) + armour = export_armour(keyid) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + finally: + # delete just imported key to leave environment as it was before + delete_key(keyid) + + return armour diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 53d5604a..b6c43ce8 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -71,8 +71,8 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), key=(lambda e: 0 - len(e))) +@six.add_metaclass(abc.ABCMeta) class Handler(object): - __metaclass__ = abc.ABCMeta def __init__(self, frequency, version=2): self.handler_version = version @@ -118,7 +118,7 @@ def run_part(mod, data, filename, payload, frequency, headers): mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) - except: + except Exception: util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " "frequency %s", mod, content_type, filename, mod_ver, frequency) @@ -157,7 +157,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): # register if it fails starting. handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 - except: + except Exception: util.logexc(LOG, "Failed at registering python file: %s (part " "handler %s)", modfname, curcount) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 07b6d0e0..cad4dc0f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -158,6 +158,6 @@ class CloudConfigPartHandler(handlers.Handler): for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) - except: + except Exception: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index c5bea711..ab381e00 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -80,7 +80,7 @@ def _has_suitable_upstart(): return False try: (version_out, _err) = util.subp(["initctl", "version"]) - except: + except Exception: util.logexc(LOG, "initctl version failed") return False diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 0cf982f3..fb95babc 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -86,7 +86,7 @@ class FileSemaphores(object): name = canon_sem_name(name) try: yield self._acquire(name, freq) - except: + except Exception: if clear_on_fail: self.clear(name, freq) raise @@ -219,7 +219,7 @@ class ConfigMerger(object): ds_cfg = self._ds.get_config_obj() if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) - except: + except Exception: util.logexc(LOG, "Failed loading of datasource config object " "from %s", self._ds) return d_cfgs @@ -230,7 +230,7 @@ class ConfigMerger(object): e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of env. config from %s', e_fn) return e_cfgs @@ -251,7 +251,7 @@ class ConfigMerger(object): if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs @@ -268,7 +268,7 @@ class ConfigMerger(object): for c_fn in self._fns: try: cfgs.append(util.read_conf(c_fn)) - except: + except Exception: util.logexc(LOG, "Failed loading of configuration from %s", c_fn) @@ -328,6 +328,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') + self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") self.upstart_conf_d = path_cfgs.get('upstart_dir') @@ -349,26 +350,19 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", + "instance_id": ".instance-id", } # Set when a datasource becomes active self.datasource = ds # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): - ipath = self.instance_link - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath + return self._get_path(self.instance_link, name) # get_cpath : get the "clouddir" (/var/lib/cloud/<name>) # for a name in dirmap def get_cpath(self, name=None): - cpath = self.cloud_dir - add_on = self.lookups.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath + return self._get_path(self.cloud_dir, name) # _get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances/<instance>/<name>) @@ -378,7 +372,8 @@ class Paths(object): iid = self.datasource.get_instance_id() if iid is None: return None - ipath = os.path.join(self.cloud_dir, 'instances', str(iid)) + path_safe_iid = str(iid).replace(os.sep, '_') + ipath = os.path.join(self.cloud_dir, 'instances', path_safe_iid) add_on = self.lookups.get(name) if add_on: ipath = os.path.join(ipath, add_on) @@ -396,6 +391,14 @@ class Paths(object): else: return ipath + def _get_path(self, base, name=None): + if name is None: + return base + return os.path.join(base, self.lookups[name]) + + def get_runpath(self, name=None): + return self._get_path(self.run_dir, name) + # This config parser will not throw when sections don't exist # and you are setting values on those sections which is useful diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..63e54f91 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -16,41 +16,15 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see <http://www.gnu.org/licenses/>. -import base64 import errno -import glob -import gzip -import io +import logging import os import re -import shlex -from cloudinit import log as logging from cloudinit import util -from .udev import generate_udev_rule -from . import network_state LOG = logging.getLogger(__name__) - SYS_CLASS_NET = "/sys/class/net/" -LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" - -NET_CONFIG_OPTIONS = [ - "address", "netmask", "broadcast", "network", "metric", "gateway", - "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", - "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", - "netnum", "endpoint", "local", "ttl", - ] - -NET_CONFIG_COMMANDS = [ - "pre-up", "up", "post-up", "down", "pre-down", "post-down", - ] - -NET_CONFIG_BRIDGE_OPTIONS = [ - "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", - "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", - ] - DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -60,23 +34,22 @@ def sys_dev_path(devname, path=""): def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise - except OSError as e: - if e.errno == errno.ENOENT and enoent is not None: - return enoent + contents = util.load_file(sys_dev_path(devname, path)) + except (OSError, IOError) as e: + if getattr(e, 'errno', None) == errno.ENOENT: + if enoent is not None: + return enoent + raise + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror raise @@ -127,505 +100,7 @@ def get_devicelist(): class ParserError(Exception): - """Raised when parser has issue parsing the interfaces file.""" - - -def parse_deb_config_data(ifaces, contents, src_dir, src_path): - """Parses the file contents, placing result into ifaces. - - '_source_path' is added to every dictionary entry to define which file - the configration information came from. - - :param ifaces: interface dictionary - :param contents: contents of interfaces file - :param src_dir: directory interfaces file was located - :param src_path: file path the `contents` was read - """ - currif = None - for line in contents.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - split = line.split(' ') - option = split[0] - if option == "source-directory": - parsed_src_dir = split[1] - if not parsed_src_dir.startswith("/"): - parsed_src_dir = os.path.join(src_dir, parsed_src_dir) - for expanded_path in glob.glob(parsed_src_dir): - dir_contents = os.listdir(expanded_path) - dir_contents = [ - os.path.join(expanded_path, path) - for path in dir_contents - if (os.path.isfile(os.path.join(expanded_path, path)) and - re.match("^[a-zA-Z0-9_-]+$", path) is not None) - ] - for entry in dir_contents: - with open(entry, "r") as fp: - src_data = fp.read().strip() - abs_entry = os.path.abspath(entry) - parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_entry), abs_entry) - elif option == "source": - new_src_path = split[1] - if not new_src_path.startswith("/"): - new_src_path = os.path.join(src_dir, new_src_path) - for expanded_path in glob.glob(new_src_path): - with open(expanded_path, "r") as fp: - src_data = fp.read().strip() - abs_path = os.path.abspath(expanded_path) - parse_deb_config_data( - ifaces, src_data, - os.path.dirname(abs_path), abs_path) - elif option == "auto": - for iface in split[1:]: - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - ifaces[iface]['auto'] = True - elif option == "iface": - iface, family, method = split[1:4] - if iface not in ifaces: - ifaces[iface] = { - # Include the source path this interface was found in. - "_source_path": src_path - } - elif 'family' in ifaces[iface]: - raise ParserError( - "Interface %s can only be defined once. " - "Re-defined in '%s'." % (iface, src_path)) - ifaces[iface]['family'] = family - ifaces[iface]['method'] = method - currif = iface - elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] - elif option in NET_CONFIG_OPTIONS: - ifaces[currif][option] = split[1] - elif option in NET_CONFIG_COMMANDS: - if option not in ifaces[currif]: - ifaces[currif][option] = [] - ifaces[currif][option].append(' '.join(split[1:])) - elif option.startswith('dns-'): - if 'dns' not in ifaces[currif]: - ifaces[currif]['dns'] = {} - if option == 'dns-search': - ifaces[currif]['dns']['search'] = [] - for domain in split[1:]: - ifaces[currif]['dns']['search'].append(domain) - elif option == 'dns-nameservers': - ifaces[currif]['dns']['nameservers'] = [] - for server in split[1:]: - ifaces[currif]['dns']['nameservers'].append(server) - elif option.startswith('bridge_'): - if 'bridge' not in ifaces[currif]: - ifaces[currif]['bridge'] = {} - if option in NET_CONFIG_BRIDGE_OPTIONS: - bridge_option = option.replace('bridge_', '', 1) - ifaces[currif]['bridge'][bridge_option] = split[1] - elif option == "bridge_ports": - ifaces[currif]['bridge']['ports'] = [] - for iface in split[1:]: - ifaces[currif]['bridge']['ports'].append(iface) - elif option == "bridge_hw" and split[1].lower() == "mac": - ifaces[currif]['bridge']['mac'] = split[2] - elif option == "bridge_pathcost": - if 'pathcost' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['pathcost'] = {} - ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] - elif option == "bridge_portprio": - if 'portprio' not in ifaces[currif]['bridge']: - ifaces[currif]['bridge']['portprio'] = {} - ifaces[currif]['bridge']['portprio'][split[1]] = split[2] - elif option.startswith('bond-'): - if 'bond' not in ifaces[currif]: - ifaces[currif]['bond'] = {} - bond_option = option.replace('bond-', '', 1) - ifaces[currif]['bond'][bond_option] = split[1] - for iface in ifaces.keys(): - if 'auto' not in ifaces[iface]: - ifaces[iface]['auto'] = False - - -def parse_deb_config(path): - """Parses a debian network configuration file.""" - ifaces = {} - with open(path, "r") as fp: - contents = fp.read().strip() - abs_path = os.path.abspath(path) - parse_deb_config_data( - ifaces, contents, - os.path.dirname(abs_path), abs_path) - return ifaces - - -def parse_net_config_data(net_config): - """Parses the config, returns NetworkState dictionary - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config() - state = ns.network_state - - return state - - -def parse_net_config(path): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network')) - - return ns - - -def _load_shell_content(content, add_empty=False, empty_val=None): - """Given shell like syntax (key=value\nkey2=value2\n) in content - return the data in dictionary form. If 'add_empty' is True - then add entries in to the returned dictionary for 'VAR=' - variables. Set their value to empty_val.""" - data = {} - for line in shlex.split(content): - key, value = line.split("=", 1) - if not value: - value = empty_val - if add_empty or value: - data[key] = value - - return data - - -def _klibc_to_config_entry(content, mac_addrs=None): - """Convert a klibc writtent shell content file to a 'config' entry - When ip= is seen on the kernel command line in debian initramfs - and networking is brought up, ipconfig will populate - /run/net-<name>.cfg. - - The files are shell style syntax, and examples are in the tests - provided here. There is no good documentation on this unfortunately. - - DEVICE=<name> is expected/required and PROTO should indicate if - this is 'static' or 'dhcp'. - """ - - if mac_addrs is None: - mac_addrs = {} - - data = _load_shell_content(content) - try: - name = data['DEVICE'] - except KeyError: - raise ValueError("no 'DEVICE' entry in data") - - # ipconfig on precise does not write PROTO - proto = data.get('PROTO') - if not proto: - if data.get('filename'): - proto = 'dhcp' - else: - proto = 'static' - - if proto not in ('static', 'dhcp'): - raise ValueError("Unexpected value for PROTO: %s" % proto) - - iface = { - 'type': 'physical', - 'name': name, - 'subnets': [], - } - - if name in mac_addrs: - iface['mac_address'] = mac_addrs[name] - - # originally believed there might be IPV6* values - for v, pre in (('ipv4', 'IPV4'),): - # if no IPV4ADDR or IPV6ADDR, then go on. - if pre + "ADDR" not in data: - continue - subnet = {'type': proto, 'control': 'manual'} - - # these fields go right on the subnet - for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): - if pre + key in data: - subnet[key.lower()] = data[pre + key] - - dns = [] - # handle IPV4DNS0 or IPV6DNS0 - for nskey in ('DNS0', 'DNS1'): - ns = data.get(pre + nskey) - # verify it has something other than 0.0.0.0 (or ipv6) - if ns and len(ns.strip(":.0")): - dns.append(data[pre + nskey]) - if dns: - subnet['dns_nameservers'] = dns - # add search to both ipv4 and ipv6, as it has no namespace - search = data.get('DOMAINSEARCH') - if search: - if ',' in search: - subnet['dns_search'] = search.split(",") - else: - subnet['dns_search'] = search.split() - - iface['subnets'].append(subnet) - - return name, iface - - -def config_from_klibc_net_cfg(files=None, mac_addrs=None): - if files is None: - files = glob.glob('/run/net*.conf') - - entries = [] - names = {} - for cfg_file in files: - name, entry = _klibc_to_config_entry(util.load_file(cfg_file), - mac_addrs=mac_addrs) - if name in names: - raise ValueError( - "device '%s' defined multiple times: %s and %s" % ( - name, names[name], cfg_file)) - - names[name] = cfg_file - entries.append(entry) - return {'config': entries, 'version': 1} - - -def render_persistent_net(network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' - content = "" - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - # for physical interfaces write out a persist net udev rule - if iface['type'] == 'physical' and \ - 'name' in iface and iface.get('mac_address'): - content += generate_udev_rule(iface['name'], - iface['mac_address']) - - return content - - -# TODO: switch valid_map based on mode inet/inet6 -def iface_add_subnet(iface, subnet): - content = "" - valid_map = [ - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_search', - 'dns_nameservers', - ] - for key, value in subnet.items(): - if value and key in valid_map: - if type(value) == list: - value = " ".join(value) - if '_' in key: - key = key.replace('_', '-') - content += " {} {}\n".format(key, value) - - return content - - -# TODO: switch to valid_map for attrs -def iface_add_attrs(iface): - content = "" - ignore_map = [ - 'control', - 'index', - 'inet', - 'mode', - 'name', - 'subnets', - 'type', - ] - if iface['type'] not in ['bond', 'bridge', 'vlan']: - ignore_map.append('mac_address') - - for key, value in iface.items(): - if value and key not in ignore_map: - if type(value) == list: - value = " ".join(value) - content += " {} {}\n".format(key, value) - - return content - - -def render_route(route, indent=""): - """ When rendering routes for an iface, in some cases applying a route - may result in the route command returning non-zero which produces - some confusing output for users manually using ifup/ifdown[1]. To - that end, we will optionally include an '|| true' postfix to each - route line allowing users to work with ifup/ifdown without using - --force option. - - We may at somepoint not want to emit this additional postfix, and - add a 'strict' flag to this function. When called with strict=True, - then we will not append the postfix. - - 1. http://askubuntu.com/questions/168033/ - how-to-set-static-routes-in-ubuntu-server - """ - content = "" - up = indent + "post-up route add" - down = indent + "pre-down route del" - eol = " || true\n" - mapping = { - 'network': '-net', - 'netmask': 'netmask', - 'gateway': 'gw', - 'metric': 'metric', - } - if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': - default_gw = " default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - elif route['network'] == '::' and route['netmask'] == 0: - # ipv6! - default_gw = " -A inet6 default gw %s" % route['gateway'] - content += up + default_gw + eol - content += down + default_gw + eol - else: - route_line = "" - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - route_line += " %s %s" % (mapping[k], route[k]) - content += up + route_line + eol - content += down + route_line + eol - - return content - - -def iface_start_entry(iface, index): - fullname = iface['name'] - if index != 0: - fullname += ":%s" % index - - control = iface['control'] - if control == "auto": - cverb = "auto" - elif control in ("hotplug",): - cverb = "allow-" + control - else: - cverb = "# control-" + control - - subst = iface.copy() - subst.update({'fullname': fullname, 'cverb': cverb}) - - return ("{cverb} {fullname}\n" - "iface {fullname} {inet} {mode}\n").format(**subst) - - -def render_interfaces(network_state): - ''' Given state, emit etc/network/interfaces content ''' - - content = "" - interfaces = network_state.get('interfaces') - ''' Apply a sort order to ensure that we write out - the physical interfaces first; this is critical for - bonding - ''' - order = { - 'physical': 0, - 'bond': 1, - 'bridge': 2, - 'vlan': 3, - } - content += "auto lo\niface lo inet loopback\n" - for dnskey, value in network_state.get('dns', {}).items(): - if len(value): - content += " dns-{} {}\n".format(dnskey, " ".join(value)) - - for iface in sorted(interfaces.values(), - key=lambda k: (order[k['type']], k['name'])): - - if content[-2:] != "\n\n": - content += "\n" - subnets = iface.get('subnets', {}) - if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): - if content[-2:] != "\n\n": - content += "\n" - iface['index'] = index - iface['mode'] = subnet['type'] - iface['control'] = subnet.get('control', 'auto') - if iface['mode'].endswith('6'): - iface['inet'] += '6' - elif iface['mode'] == 'static' and ":" in subnet['address']: - iface['inet'] += '6' - if iface['mode'].startswith('dhcp'): - iface['mode'] = 'dhcp' - - content += iface_start_entry(iface, index) - content += iface_add_subnet(iface, subnet) - content += iface_add_attrs(iface) - else: - # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: - content += "auto {name}\n".format(**iface) - content += "iface {name} {inet} {mode}\n".format(**iface) - content += iface_add_attrs(iface) - - for route in network_state.get('routes'): - content += render_route(route) - - # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') - return content - - -def render_network_state(target, network_state, eni="etc/network/interfaces", - links_prefix=LINKS_FNAME_PREFIX, - netrules='etc/udev/rules.d/70-persistent-net.rules'): - - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(render_interfaces(network_state)) - - if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(render_persistent_net(network_state)) - - if links_prefix: - render_systemd_links(target, network_state, links_prefix) - - -def render_systemd_links(target, network_state, - links_prefix=LINKS_FNAME_PREFIX): - fp_prefix = os.path.sep.join((target, links_prefix)) - for f in glob.glob(fp_prefix + "*"): - os.unlink(f) - - interfaces = network_state.get('interfaces') - for iface in interfaces.values(): - if (iface['type'] == 'physical' and 'name' in iface and - iface.get('mac_address')): - fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + """Raised when a parser has issue parsing a file/content.""" def is_disabled_cfg(cfg): @@ -638,7 +113,6 @@ def sys_netdev_info(name, field): if not os.path.exists(os.path.join(SYS_CLASS_NET, name)): raise OSError("%s: interface does not exist in %s" % (name, SYS_CLASS_NET)) - fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) @@ -718,56 +192,179 @@ def generate_fallback_config(): return nconf -def _decomp_gzip(blob, strict=True): - # decompress blob. raise exception if not compressed unless strict=False. - with io.BytesIO(blob) as iobuf: - gzfp = None - try: - gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) - return gzfp.read() - except IOError: - if strict: - raise - return blob - finally: - if gzfp: - gzfp.close() +def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): + """read the network config and rename devices accordingly. + if strict_present is false, then do not raise exception if no devices + match. if strict_busy is false, then do not raise exception if the + device cannot be renamed because it is currently configured.""" + renames = [] + for ent in netcfg.get('config', {}): + if ent.get('type') != 'physical': + continue + mac = ent.get('mac_address') + name = ent.get('name') + if not mac: + continue + renames.append([mac, name]) + return _rename_interfaces(renames) -def _b64dgz(b64str, gzipped="try"): - # decode a base64 string. If gzipped is true, transparently uncompresss - # if gzipped is 'try', then try gunzip, returning the original on fail. - try: - blob = base64.b64decode(b64str) - except TypeError: - raise ValueError("Invalid base64 text: %s" % b64str) - if not gzipped: - return blob +def _get_current_rename_info(check_downable=True): + """Collect information necessary for rename_interfaces.""" + names = get_devicelist() + bymac = {} + for n in names: + bymac[get_interface_mac(n)] = { + 'name': n, 'up': is_up(n), 'downable': None} + + if check_downable: + nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") + ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + 'scope', 'global'], capture=True) + ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + + nics_with_addresses = set() + for bytes_out in (ipv6, ipv4): + nics_with_addresses.update(nmatch.findall(bytes_out)) + + for d in bymac.values(): + d['downable'] = (d['up'] is False or + d['name'] not in nics_with_addresses) - return _decomp_gzip(blob, strict=gzipped != "try") + return bymac -def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() +def _rename_interfaces(renames, strict_present=True, strict_busy=True, + current_info=None): - if 'network-config=' in cmdline: - data64 = None - for tok in cmdline.split(): - if tok.startswith("network-config="): - data64 = tok.split("=", 1)[1] - if data64: - return util.load_yaml(_b64dgz(data64)) + if not len(renames): + LOG.debug("no interfaces to rename") + return + + if current_info is None: + current_info = _get_current_rename_info() + + cur_bymac = {} + for mac, data in current_info.items(): + cur = data.copy() + cur['mac'] = mac + cur_bymac[mac] = cur + + def update_byname(bymac): + return {data['name']: data for data in bymac.values()} + + def rename(cur, new): + util.subp(["ip", "link", "set", cur, "name", new], capture=True) - if 'ip=' not in cmdline: - return None + def down(name): + util.subp(["ip", "link", "set", name, "down"], capture=True) - if mac_addrs is None: - mac_addrs = {k: sys_netdev_info(k, 'address') - for k in get_devicelist()} + def up(name): + util.subp(["ip", "link", "set", name, "up"], capture=True) - return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) + ops = [] + errors = [] + ups = [] + cur_byname = update_byname(cur_bymac) + tmpname_fmt = "cirename%d" + tmpi = -1 + + for mac, new_name in renames: + cur = cur_bymac.get(mac, {}) + cur_name = cur.get('name') + cur_ops = [] + if cur_name == new_name: + # nothing to do + continue + + if not cur_name: + if strict_present: + errors.append( + "[nic not present] Cannot rename mac=%s to %s" + ", not available." % (mac, new_name)) + continue + if cur['up']: + msg = "[busy] Error renaming mac=%s from %s to %s" + if not cur['downable']: + if strict_busy: + errors.append(msg % (mac, cur_name, new_name)) + continue + cur['up'] = False + cur_ops.append(("down", mac, new_name, (cur_name,))) + ups.append(("up", mac, new_name, (new_name,))) + + if new_name in cur_byname: + target = cur_byname[new_name] + if target['up']: + msg = "[busy-target] Error renaming mac=%s from %s to %s." + if not target['downable']: + if strict_busy: + errors.append(msg % (mac, cur_name, new_name)) + continue + else: + cur_ops.append(("down", mac, new_name, (new_name,))) + + tmp_name = None + while tmp_name is None or tmp_name in cur_byname: + tmpi += 1 + tmp_name = tmpname_fmt % tmpi + + cur_ops.append(("rename", mac, new_name, (new_name, tmp_name))) + target['name'] = tmp_name + cur_byname = update_byname(cur_bymac) + if target['up']: + ups.append(("up", mac, new_name, (tmp_name,))) + + cur_ops.append(("rename", mac, new_name, (cur['name'], new_name))) + cur['name'] = new_name + cur_byname = update_byname(cur_bymac) + ops += cur_ops + + opmap = {'rename': rename, 'down': down, 'up': up} + + if len(ops) + len(ups) == 0: + if len(errors): + LOG.debug("unable to do any work for renaming of %s", renames) + else: + LOG.debug("no work necessary for renaming of %s", renames) + else: + LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) + + for op, mac, new_name, params in ops + ups: + try: + opmap.get(op)(*params) + except Exception as e: + errors.append( + "[unknown] Error performing %s%s for %s, %s: %s" % + (op, params, mac, new_name, e)) + + if len(errors): + raise Exception('\n'.join(errors)) + + +def get_interface_mac(ifname): + """Returns the string value of an interface's MAC Address""" + return read_sys_net(ifname, "address", enoent=False) + + +def get_interfaces_by_mac(devs=None): + """Build a dictionary of tuples {mac: name}""" + if devs is None: + try: + devs = get_devicelist() + except OSError as e: + if e.errno == errno.ENOENT: + devs = [] + else: + raise + ret = {} + for name in devs: + mac = get_interface_mac(name) + # some devices may not have a mac (tun0) + if mac: + ret[mac] = name + return ret # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py new file mode 100644 index 00000000..822a020b --- /dev/null +++ b/cloudinit/net/cmdline.py @@ -0,0 +1,203 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Blake Rouse <blake.rouse@canonical.com> +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see <http://www.gnu.org/licenses/>. + +import base64 +import glob +import gzip +import io +import shlex +import sys + +import six + +from . import get_devicelist +from . import sys_netdev_info + +from cloudinit import util + +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + + +def _load_shell_content(content, add_empty=False, empty_val=None): + """Given shell like syntax (key=value\nkey2=value2\n) in content + return the data in dictionary form. If 'add_empty' is True + then add entries in to the returned dictionary for 'VAR=' + variables. Set their value to empty_val.""" + data = {} + for line in _shlex_split(content): + key, value = line.split("=", 1) + if not value: + value = empty_val + if add_empty or value: + data[key] = value + + return data + + +def _klibc_to_config_entry(content, mac_addrs=None): + """Convert a klibc writtent shell content file to a 'config' entry + When ip= is seen on the kernel command line in debian initramfs + and networking is brought up, ipconfig will populate + /run/net-<name>.cfg. + + The files are shell style syntax, and examples are in the tests + provided here. There is no good documentation on this unfortunately. + + DEVICE=<name> is expected/required and PROTO should indicate if + this is 'static' or 'dhcp'. + """ + + if mac_addrs is None: + mac_addrs = {} + + data = _load_shell_content(content) + try: + name = data['DEVICE'] + except KeyError: + raise ValueError("no 'DEVICE' entry in data") + + # ipconfig on precise does not write PROTO + proto = data.get('PROTO') + if not proto: + if data.get('filename'): + proto = 'dhcp' + else: + proto = 'static' + + if proto not in ('static', 'dhcp'): + raise ValueError("Unexpected value for PROTO: %s" % proto) + + iface = { + 'type': 'physical', + 'name': name, + 'subnets': [], + } + + if name in mac_addrs: + iface['mac_address'] = mac_addrs[name] + + # originally believed there might be IPV6* values + for v, pre in (('ipv4', 'IPV4'),): + # if no IPV4ADDR or IPV6ADDR, then go on. + if pre + "ADDR" not in data: + continue + subnet = {'type': proto, 'control': 'manual'} + + # these fields go right on the subnet + for key in ('NETMASK', 'BROADCAST', 'GATEWAY'): + if pre + key in data: + subnet[key.lower()] = data[pre + key] + + dns = [] + # handle IPV4DNS0 or IPV6DNS0 + for nskey in ('DNS0', 'DNS1'): + ns = data.get(pre + nskey) + # verify it has something other than 0.0.0.0 (or ipv6) + if ns and len(ns.strip(":.0")): + dns.append(data[pre + nskey]) + if dns: + subnet['dns_nameservers'] = dns + # add search to both ipv4 and ipv6, as it has no namespace + search = data.get('DOMAINSEARCH') + if search: + if ',' in search: + subnet['dns_search'] = search.split(",") + else: + subnet['dns_search'] = search.split() + + iface['subnets'].append(subnet) + + return name, iface + + +def config_from_klibc_net_cfg(files=None, mac_addrs=None): + if files is None: + files = glob.glob('/run/net*.conf') + + entries = [] + names = {} + for cfg_file in files: + name, entry = _klibc_to_config_entry(util.load_file(cfg_file), + mac_addrs=mac_addrs) + if name in names: + raise ValueError( + "device '%s' defined multiple times: %s and %s" % ( + name, names[name], cfg_file)) + + names[name] = cfg_file + entries.append(entry) + return {'config': entries, 'version': 1} + + +def _decomp_gzip(blob, strict=True): + # decompress blob. raise exception if not compressed unless strict=False. + with io.BytesIO(blob) as iobuf: + gzfp = None + try: + gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf) + return gzfp.read() + except IOError: + if strict: + raise + return blob + finally: + if gzfp: + gzfp.close() + + +def _b64dgz(b64str, gzipped="try"): + # decode a base64 string. If gzipped is true, transparently uncompresss + # if gzipped is 'try', then try gunzip, returning the original on fail. + try: + blob = base64.b64decode(b64str) + except TypeError: + raise ValueError("Invalid base64 text: %s" % b64str) + + if not gzipped: + return blob + + return _decomp_gzip(blob, strict=gzipped != "try") + + +def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + + if 'network-config=' in cmdline: + data64 = None + for tok in cmdline.split(): + if tok.startswith("network-config="): + data64 = tok.split("=", 1)[1] + if data64: + return util.load_yaml(_b64dgz(data64)) + + if 'ip=' not in cmdline: + return None + + if mac_addrs is None: + mac_addrs = dict((k, sys_netdev_info(k, 'address')) + for k in get_devicelist()) + + return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py new file mode 100644 index 00000000..e5ed10fd --- /dev/null +++ b/cloudinit/net/eni.py @@ -0,0 +1,450 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import glob +import os +import re + +from . import ParserError + +from . import renderer + +from cloudinit import util + + +NET_CONFIG_COMMANDS = [ + "pre-up", "up", "post-up", "down", "pre-down", "post-down", +] + +NET_CONFIG_BRIDGE_OPTIONS = [ + "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", + "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", +] + +NET_CONFIG_OPTIONS = [ + "address", "netmask", "broadcast", "network", "metric", "gateway", + "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", + "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", + "netnum", "endpoint", "local", "ttl", +] + + +# TODO: switch valid_map based on mode inet/inet6 +def _iface_add_subnet(iface, subnet): + content = "" + valid_map = [ + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_search', + 'dns_nameservers', + ] + for key, value in subnet.items(): + if value and key in valid_map: + if type(value) == list: + value = " ".join(value) + if '_' in key: + key = key.replace('_', '-') + content += " {} {}\n".format(key, value) + + return content + + +# TODO: switch to valid_map for attrs + +def _iface_add_attrs(iface): + content = "" + ignore_map = [ + 'control', + 'index', + 'inet', + 'mode', + 'name', + 'subnets', + 'type', + ] + if iface['type'] not in ['bond', 'bridge', 'vlan']: + ignore_map.append('mac_address') + + for key, value in iface.items(): + if value and key not in ignore_map: + if type(value) == list: + value = " ".join(value) + content += " {} {}\n".format(key, value) + + return content + + +def _iface_start_entry(iface, index): + fullname = iface['name'] + if index != 0: + fullname += ":%s" % index + + control = iface['control'] + if control == "auto": + cverb = "auto" + elif control in ("hotplug",): + cverb = "allow-" + control + else: + cverb = "# control-" + control + + subst = iface.copy() + subst.update({'fullname': fullname, 'cverb': cverb}) + + return ("{cverb} {fullname}\n" + "iface {fullname} {inet} {mode}\n").format(**subst) + + +def _parse_deb_config_data(ifaces, contents, src_dir, src_path): + """Parses the file contents, placing result into ifaces. + + '_source_path' is added to every dictionary entry to define which file + the configration information came from. + + :param ifaces: interface dictionary + :param contents: contents of interfaces file + :param src_dir: directory interfaces file was located + :param src_path: file path the `contents` was read + """ + currif = None + for line in contents.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + split = line.split(' ') + option = split[0] + if option == "source-directory": + parsed_src_dir = split[1] + if not parsed_src_dir.startswith("/"): + parsed_src_dir = os.path.join(src_dir, parsed_src_dir) + for expanded_path in glob.glob(parsed_src_dir): + dir_contents = os.listdir(expanded_path) + dir_contents = [ + os.path.join(expanded_path, path) + for path in dir_contents + if (os.path.isfile(os.path.join(expanded_path, path)) and + re.match("^[a-zA-Z0-9_-]+$", path) is not None) + ] + for entry in dir_contents: + with open(entry, "r") as fp: + src_data = fp.read().strip() + abs_entry = os.path.abspath(entry) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_entry), abs_entry) + elif option == "source": + new_src_path = split[1] + if not new_src_path.startswith("/"): + new_src_path = os.path.join(src_dir, new_src_path) + for expanded_path in glob.glob(new_src_path): + with open(expanded_path, "r") as fp: + src_data = fp.read().strip() + abs_path = os.path.abspath(expanded_path) + _parse_deb_config_data( + ifaces, src_data, + os.path.dirname(abs_path), abs_path) + elif option == "auto": + for iface in split[1:]: + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + ifaces[iface]['auto'] = True + elif option == "iface": + iface, family, method = split[1:4] + if iface not in ifaces: + ifaces[iface] = { + # Include the source path this interface was found in. + "_source_path": src_path + } + elif 'family' in ifaces[iface]: + raise ParserError( + "Interface %s can only be defined once. " + "Re-defined in '%s'." % (iface, src_path)) + ifaces[iface]['family'] = family + ifaces[iface]['method'] = method + currif = iface + elif option == "hwaddress": + if split[1] == "ether": + val = split[2] + else: + val = split[1] + ifaces[currif]['hwaddress'] = val + elif option in NET_CONFIG_OPTIONS: + ifaces[currif][option] = split[1] + elif option in NET_CONFIG_COMMANDS: + if option not in ifaces[currif]: + ifaces[currif][option] = [] + ifaces[currif][option].append(' '.join(split[1:])) + elif option.startswith('dns-'): + if 'dns' not in ifaces[currif]: + ifaces[currif]['dns'] = {} + if option == 'dns-search': + ifaces[currif]['dns']['search'] = [] + for domain in split[1:]: + ifaces[currif]['dns']['search'].append(domain) + elif option == 'dns-nameservers': + ifaces[currif]['dns']['nameservers'] = [] + for server in split[1:]: + ifaces[currif]['dns']['nameservers'].append(server) + elif option.startswith('bridge_'): + if 'bridge' not in ifaces[currif]: + ifaces[currif]['bridge'] = {} + if option in NET_CONFIG_BRIDGE_OPTIONS: + bridge_option = option.replace('bridge_', '', 1) + ifaces[currif]['bridge'][bridge_option] = split[1] + elif option == "bridge_ports": + ifaces[currif]['bridge']['ports'] = [] + for iface in split[1:]: + ifaces[currif]['bridge']['ports'].append(iface) + elif option == "bridge_hw" and split[1].lower() == "mac": + ifaces[currif]['bridge']['mac'] = split[2] + elif option == "bridge_pathcost": + if 'pathcost' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['pathcost'] = {} + ifaces[currif]['bridge']['pathcost'][split[1]] = split[2] + elif option == "bridge_portprio": + if 'portprio' not in ifaces[currif]['bridge']: + ifaces[currif]['bridge']['portprio'] = {} + ifaces[currif]['bridge']['portprio'][split[1]] = split[2] + elif option.startswith('bond-'): + if 'bond' not in ifaces[currif]: + ifaces[currif]['bond'] = {} + bond_option = option.replace('bond-', '', 1) + ifaces[currif]['bond'][bond_option] = split[1] + for iface in ifaces.keys(): + if 'auto' not in ifaces[iface]: + ifaces[iface]['auto'] = False + + +def parse_deb_config(path): + """Parses a debian network configuration file.""" + ifaces = {} + with open(path, "r") as fp: + contents = fp.read().strip() + abs_path = os.path.abspath(path) + _parse_deb_config_data( + ifaces, contents, + os.path.dirname(abs_path), abs_path) + return ifaces + + +def convert_eni_data(eni_data): + # return a network config representation of what is in eni_data + ifaces = {} + _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + return _ifaces_to_net_config_data(ifaces) + + +def _ifaces_to_net_config_data(ifaces): + """Return network config that represents the ifaces data provided. + ifaces = parse_deb_config("/etc/network/interfaces") + config = ifaces_to_net_config_data(ifaces) + state = parse_net_config_data(config).""" + devs = {} + for name, data in ifaces.items(): + # devname is 'eth0' for name='eth0:1' + devname = name.partition(":")[0] + if devname == "lo": + # currently provding 'lo' in network config results in duplicate + # entries. in rendered interfaces file. so skip it. + continue + if devname not in devs: + devs[devname] = {'type': 'physical', 'name': devname, + 'subnets': []} + # this isnt strictly correct, but some might specify + # hwaddress on a nic for matching / declaring name. + if 'hwaddress' in data: + devs[devname]['mac_address'] = data['hwaddress'] + subnet = {'_orig_eni_name': name, 'type': data['method']} + if data.get('auto'): + subnet['control'] = 'auto' + else: + subnet['control'] = 'manual' + + if data.get('method') == 'static': + subnet['address'] = data['address'] + + for copy_key in ('netmask', 'gateway', 'broadcast'): + if copy_key in data: + subnet[copy_key] = data[copy_key] + + if 'dns' in data: + for n in ('nameservers', 'search'): + if n in data['dns'] and data['dns'][n]: + subnet['dns_' + n] = data['dns'][n] + devs[devname]['subnets'].append(subnet) + + return {'version': 1, + 'config': [devs[d] for d in sorted(devs)]} + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/network/interfaces format.""" + + def __init__(self, config=None): + if not config: + config = {} + self.eni_path = config.get('eni_path', 'etc/network/interfaces') + self.eni_header = config.get('eni_header', None) + self.links_path_prefix = config.get( + 'links_path_prefix', 'etc/systemd/network/50-cloud-init-') + self.netrules_path = config.get( + 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') + + def _render_route(self, route, indent=""): + """When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" + mapping = { + 'network': '-net', + 'netmask': 'netmask', + 'gateway': 'gw', + 'metric': 'metric', + } + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol + return content + + def _render_interfaces(self, network_state): + '''Given state, emit etc/network/interfaces content.''' + + content = "" + content += "auto lo\niface lo inet loopback\n" + + nameservers = network_state.dns_nameservers + if nameservers: + content += " dns-nameservers %s\n" % (" ".join(nameservers)) + searchdomains = network_state.dns_searchdomains + if searchdomains: + content += " dns-search %s\n" % (" ".join(searchdomains)) + + ''' Apply a sort order to ensure that we write out + the physical interfaces first; this is critical for + bonding + ''' + order = { + 'physical': 0, + 'bond': 1, + 'bridge': 2, + 'vlan': 3, + } + for iface in sorted(network_state.iter_interfaces(), + key=lambda k: (order[k['type']], k['name'])): + + if content[-2:] != "\n\n": + content += "\n" + subnets = iface.get('subnets', {}) + if subnets: + for index, subnet in zip(range(0, len(subnets)), subnets): + if content[-2:] != "\n\n": + content += "\n" + iface['index'] = index + iface['mode'] = subnet['type'] + iface['control'] = subnet.get('control', 'auto') + if iface['mode'].endswith('6'): + iface['inet'] += '6' + elif (iface['mode'] == 'static' and + ":" in subnet['address']): + iface['inet'] += '6' + if iface['mode'].startswith('dhcp'): + iface['mode'] = 'dhcp' + + content += _iface_start_entry(iface, index) + content += _iface_add_subnet(iface, subnet) + content += _iface_add_attrs(iface) + for route in subnet.get('routes', []): + content += self._render_route(route, indent=" ") + else: + # ifenslave docs say to auto the slave devices + if 'bond-master' in iface: + content += "auto {name}\n".format(**iface) + content += "iface {name} {inet} {mode}\n".format(**iface) + content += _iface_add_attrs(iface) + + for route in network_state.iter_routes(): + content += self._render_route(route) + + # global replacements until v2 format + content = content.replace('mac_address', 'hwaddress') + return content + + def render_network_state(self, target, network_state): + fpeni = os.path.join(target, self.eni_path) + util.ensure_dir(os.path.dirname(fpeni)) + header = self.eni_header if self.eni_header else "" + util.write_file(fpeni, header + self._render_interfaces(network_state)) + + if self.netrules_path: + netrules = os.path.join(target, self.netrules_path) + util.ensure_dir(os.path.dirname(netrules)) + util.write_file(netrules, + self._render_persistent_net(network_state)) + + if self.links_path_prefix: + self._render_systemd_links(target, network_state, + links_prefix=self.links_path_prefix) + + def _render_systemd_links(self, target, network_state, links_prefix): + fp_prefix = os.path.join(target, links_prefix) + for f in glob.glob(fp_prefix + "*"): + os.unlink(f) + for iface in network_state.iter_interfaces(): + if (iface['type'] == 'physical' and 'name' in iface and + iface.get('mac_address')): + fname = fp_prefix + iface['name'] + ".link" + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + util.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..8ca5106f 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -15,9 +15,13 @@ # You should have received a copy of the GNU Affero General Public License # along with Curtin. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import log as logging +import copy +import functools +import logging + +import six + from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config LOG = logging.getLogger(__name__) @@ -27,80 +31,198 @@ NETWORK_STATE_REQUIRED_KEYS = { } -def from_state_file(state_file): - network_state = None - state = util.read_conf(state_file) - network_state = NetworkState() - network_state.load(state) +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object - return network_state + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + nsi = NetworkStateInterpreter(version=net_config.get('version'), + config=net_config.get('config')) + nsi.parse_config(skip_broken=skip_broken) + state = nsi.network_state + return state -class NetworkState: - def __init__(self, version=NETWORK_STATE_VERSION, config=None): - self.version = version - self.config = config - self.network_state = { - 'interfaces': {}, - 'routes': [], - 'dns': { - 'nameservers': [], - 'search': [], - } +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = util.read_conf(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + +def from_state_file(state_file): + state = util.read_conf(state_file) + nsi = NetworkStateInterpreter() + nsi.load(state) + return nsi + + +def diff_keys(expected, actual): + missing = set(expected) + for key in actual: + missing.discard(key) + return missing + + +class InvalidCommand(Exception): + pass + + +def ensure_command_keys(required_keys): + + def wrapper(func): + + @functools.wraps(func) + def decorator(self, command, *args, **kwargs): + if required_keys: + missing_keys = diff_keys(required_keys, command) + if missing_keys: + raise InvalidCommand("Command missing %s of required" + " keys %s" % (missing_keys, + required_keys)) + return func(self, command, *args, **kwargs) + + return decorator + + return wrapper + + +class CommandHandlerMeta(type): + """Metaclass that dynamically creates a 'command_handlers' attribute. + + This will scan the to-be-created class for methods that start with + 'handle_' and on finding those will populate a class attribute mapping + so that those methods can be quickly located and called. + """ + def __new__(cls, name, parents, dct): + command_handlers = {} + for attr_name, attr in dct.items(): + if callable(attr) and attr_name.startswith('handle_'): + handles_what = attr_name[len('handle_'):] + if handles_what: + command_handlers[handles_what] = attr + dct['command_handlers'] = command_handlers + return super(CommandHandlerMeta, cls).__new__(cls, name, + parents, dct) + + +class NetworkState(object): + + def __init__(self, network_state, version=NETWORK_STATE_VERSION): + self._network_state = copy.deepcopy(network_state) + self._version = version + + @property + def version(self): + return self._version + + def iter_routes(self, filter_func=None): + for route in self._network_state.get('routes', []): + if filter_func is not None: + if filter_func(route): + yield route + else: + yield route + + @property + def dns_nameservers(self): + try: + return self._network_state['dns']['nameservers'] + except KeyError: + return [] + + @property + def dns_searchdomains(self): + try: + return self._network_state['dns']['search'] + except KeyError: + return [] + + def iter_interfaces(self, filter_func=None): + ifaces = self._network_state.get('interfaces', {}) + for iface in six.itervalues(ifaces): + if filter_func is None: + yield iface + else: + if filter_func(iface): + yield iface + + +@six.add_metaclass(CommandHandlerMeta) +class NetworkStateInterpreter(object): + + initial_network_state = { + 'interfaces': {}, + 'routes': [], + 'dns': { + 'nameservers': [], + 'search': [], } - self.command_handlers = self.get_command_handlers() + } - def get_command_handlers(self): - METHOD_PREFIX = 'handle_' - methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) - handlers = {} - for m in methods: - key = m.replace(METHOD_PREFIX, '') - handlers[key] = getattr(self, m) + def __init__(self, version=NETWORK_STATE_VERSION, config=None): + self._version = version + self._config = config + self._network_state = copy.deepcopy(self.initial_network_state) + self._parsed = False - return handlers + @property + def network_state(self): + return NetworkState(self._network_state, version=self._version) def dump(self): state = { - 'version': self.version, - 'config': self.config, - 'network_state': self.network_state, + 'version': self._version, + 'config': self._config, + 'network_state': self._network_state, } - return dump_config(state) + return util.yaml_dumps(state) def load(self, state): if 'version' not in state: LOG.error('Invalid state, missing version field') - raise Exception('Invalid state, missing version field') + raise ValueError('Invalid state, missing version field') required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] - if not self.valid_command(state, required_keys): - msg = 'Invalid state, missing keys: {}'.format(required_keys) + missing_keys = diff_keys(required_keys, state) + if missing_keys: + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) - raise Exception(msg) + raise ValueError(msg) # v1 - direct attr mapping, except version for key in [k for k in required_keys if k not in ['version']]: setattr(self, key, state[key]) - self.command_handlers = self.get_command_handlers() def dump_network_state(self): - return dump_config(self.network_state) + return util.yaml_dumps(self._network_state) - def parse_config(self): + def parse_config(self, skip_broken=True): # rebuild network state - for command in self.config: - handler = self.command_handlers.get(command['type']) - handler(command) - - def valid_command(self, command, required_keys): - if not required_keys: - return False - - found_keys = [key for key in command.keys() if key in required_keys] - return len(found_keys) == len(required_keys) - + for command in self._config: + command_type = command['type'] + try: + handler = self.command_handlers[command_type] + except KeyError: + raise RuntimeError("No handler found for" + " command '%s'" % command_type) + try: + handler(self, command) + except InvalidCommand: + if not skip_broken: + raise + else: + LOG.warn("Skipping invalid command: %s", command, + exc_info=True) + LOG.debug(self.dump_network_state()) + + @ensure_command_keys(['name']) def handle_physical(self, command): ''' command = { @@ -112,15 +234,8 @@ class NetworkState: ] } ''' - required_keys = [ - 'name', - ] - if not self.valid_command(command, required_keys): - LOG.warn('Skipping Invalid command: {}'.format(command)) - LOG.debug(self.dump_network_state()) - return - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) @@ -146,9 +261,10 @@ class NetworkState: 'gateway': None, 'subnets': subnets, }) - self.network_state['interfaces'].update({command.get('name'): iface}) + self._network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() + @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) def handle_vlan(self, command): ''' auto eth0.222 @@ -158,23 +274,14 @@ class NetworkState: hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' - required_keys = [ - 'name', - 'vlan_link', - 'vlan_id', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['vlan-raw-device'] = command.get('vlan_link') iface['vlan_id'] = command.get('vlan_id') interfaces.update({iface['name']: iface}) + @ensure_command_keys(['name', 'bond_interfaces', 'params']) def handle_bond(self, command): ''' #/etc/network/interfaces @@ -200,23 +307,14 @@ class NetworkState: bond-updelay 200 bond-lacp-rate 4 ''' - required_keys = [ - 'name', - 'bond_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return self.handle_physical(command) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces') iface = interfaces.get(command.get('name'), {}) for param, val in command.get('params').items(): iface.update({param: val}) iface.update({'bond-slaves': 'none'}) - self.network_state['interfaces'].update({iface['name']: iface}) + self._network_state['interfaces'].update({iface['name']: iface}) # handle bond slaves for ifname in command.get('bond_interfaces'): @@ -228,14 +326,15 @@ class NetworkState: # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) bond_if = interfaces.get(ifname) bond_if['bond-master'] = command.get('name') # copy in bond config into slave for param, val in command.get('params').items(): bond_if.update({param: val}) - self.network_state['interfaces'].update({ifname: bond_if}) + self._network_state['interfaces'].update({ifname: bond_if}) + @ensure_command_keys(['name', 'bridge_interfaces', 'params']) def handle_bridge(self, command): ''' auto br0 @@ -263,19 +362,10 @@ class NetworkState: "bridge_waitport", ] ''' - required_keys = [ - 'name', - 'bridge_interfaces', - 'params', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return # find one of the bridge port ifaces to get mac_addr # handle bridge_slaves - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) for ifname in command.get('bridge_interfaces'): if ifname in interfaces: continue @@ -286,7 +376,7 @@ class NetworkState: # inject placeholder self.handle_physical(cmd) - interfaces = self.network_state.get('interfaces') + interfaces = self._network_state.get('interfaces', {}) self.handle_physical(command) iface = interfaces.get(command.get('name'), {}) iface['bridge_ports'] = command['bridge_interfaces'] @@ -295,16 +385,9 @@ class NetworkState: interfaces.update({iface['name']: iface}) + @ensure_command_keys(['address']) def handle_nameserver(self, command): - required_keys = [ - 'address', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - - dns = self.network_state.get('dns') + dns = self._network_state.get('dns') if 'address' in command: addrs = command['address'] if not type(addrs) == list: @@ -318,16 +401,9 @@ class NetworkState: for path in paths: dns['search'].append(path) + @ensure_command_keys(['destination']) def handle_route(self, command): - required_keys = [ - 'destination', - ] - if not self.valid_command(command, required_keys): - print('Skipping Invalid command: {}'.format(command)) - print(self.dump_network_state()) - return - - routes = self.network_state.get('routes') + routes = self._network_state.get('routes', []) network, cidr = command['destination'].split("/") netmask = cidr2mask(int(cidr)) route = { @@ -376,71 +452,3 @@ def mask2cidr(mask): return ipv4mask2cidr(mask) else: return mask - - -if __name__ == '__main__': - import sys - import random - from cloudinit import net - - def load_config(nc): - version = nc.get('version') - config = nc.get('config') - return (version, config) - - def test_parse(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("----NS1-----") - print(ns1.dump_network_state()) - print() - print("----NS2-----") - print(ns2.dump_network_state()) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni = net.render_interfaces(ns2.network_state) - print(eni) - udev_rules = net.render_persistent_net(ns2.network_state) - print(udev_rules) - - def test_dump_and_load(network_config): - print("Loading network_config into NetworkState") - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - print("Dumping state to file") - ns1_dump = ns1.dump() - ns1_state = "/tmp/ns1.state" - with open(ns1_state, "w+") as f: - f.write(ns1_dump) - - print("Loading state from file") - ns2 = from_state_file(ns1_state) - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - - def test_output(network_config): - (version, config) = load_config(network_config) - ns1 = NetworkState(version=version, config=config) - ns1.parse_config() - random.shuffle(config) - ns2 = NetworkState(version=version, config=config) - ns2.parse_config() - print("NS1 == NS2 ?=> {}".format( - ns1.network_state == ns2.network_state)) - eni_1 = net.render_interfaces(ns1.network_state) - eni_2 = net.render_interfaces(ns2.network_state) - print(eni_1) - print(eni_2) - print("eni_1 == eni_2 ?=> {}".format( - eni_1 == eni_2)) - - y = util.read_conf(sys.argv[1]) - network_config = y.get('network') - test_parse(network_config) - test_dump_and_load(network_config) - test_output(network_config) diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py new file mode 100644 index 00000000..310cbe0d --- /dev/null +++ b/cloudinit/net/renderer.py @@ -0,0 +1,48 @@ +# Copyright (C) 2013-2014 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Blake Rouse <blake.rouse@canonical.com> +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see <http://www.gnu.org/licenses/>. + +import six + +from .udev import generate_udev_rule + + +def filter_by_type(match_type): + return lambda iface: match_type == iface['type'] + + +def filter_by_name(match_name): + return lambda iface: match_name == iface['name'] + + +filter_by_physical = filter_by_type('physical') + + +class Renderer(object): + + @staticmethod + def _render_persistent_net(network_state): + """Given state, emit udev rules to map mac to ifname.""" + # TODO(harlowja): this seems shared between eni renderer and + # this, so move it to a shared location. + content = six.StringIO() + for iface in network_state.iter_interfaces(filter_by_physical): + # for physical interfaces write out a persist net udev rule + if 'name' in iface and iface.get('mac_address'): + content.write(generate_udev_rule(iface['name'], + iface['mac_address'])) + return content.getvalue() diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py new file mode 100644 index 00000000..c53acf71 --- /dev/null +++ b/cloudinit/net/sysconfig.py @@ -0,0 +1,400 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import re + +import six + +from cloudinit.distros.parsers import resolv_conf +from cloudinit import util + +from . import renderer + + +def _make_header(sep='#'): + lines = [ + "Created by cloud-init on instance boot automatically, do not edit.", + "", + ] + for i in range(0, len(lines)): + if lines[i]: + lines[i] = sep + " " + lines[i] + else: + lines[i] = sep + return "\n".join(lines) + + +def _is_default_route(route): + if route['network'] == '::' and route['netmask'] == 0: + return True + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + return True + return False + + +def _quote_value(value): + if re.search(r"\s", value): + # This doesn't handle complex cases... + if value.startswith('"') and value.endswith('"'): + return value + else: + return '"%s"' % value + else: + return value + + +class ConfigMap(object): + """Sysconfig like dictionary object.""" + + # Why does redhat prefer yes/no to true/false?? + _bool_map = { + True: 'yes', + False: 'no', + } + + def __init__(self): + self._conf = {} + + def __setitem__(self, key, value): + self._conf[key] = value + + def drop(self, key): + self._conf.pop(key, None) + + def __len__(self): + return len(self._conf) + + def to_string(self): + buf = six.StringIO() + buf.write(_make_header()) + if self._conf: + buf.write("\n") + for key in sorted(self._conf.keys()): + value = self._conf[key] + if isinstance(value, bool): + value = self._bool_map[value] + if not isinstance(value, six.string_types): + value = str(value) + buf.write("%s=%s\n" % (key, _quote_value(value))) + return buf.getvalue() + + +class Route(ConfigMap): + """Represents a route configuration.""" + + route_fn_tpl = '%(base)s/network-scripts/route-%(name)s' + + def __init__(self, route_name, base_sysconf_dir): + super(Route, self).__init__() + self.last_idx = 1 + self.has_set_default = False + self._route_name = route_name + self._base_sysconf_dir = base_sysconf_dir + + def copy(self): + r = Route(self._route_name, self._base_sysconf_dir) + r._conf = self._conf.copy() + r.last_idx = self.last_idx + r.has_set_default = self.has_set_default + return r + + @property + def path(self): + return self.route_fn_tpl % ({'base': self._base_sysconf_dir, + 'name': self._route_name}) + + +class NetInterface(ConfigMap): + """Represents a sysconfig/networking-script (and its config + children).""" + + iface_fn_tpl = '%(base)s/network-scripts/ifcfg-%(name)s' + + iface_types = { + 'ethernet': 'Ethernet', + 'bond': 'Bond', + 'bridge': 'Bridge', + } + + def __init__(self, iface_name, base_sysconf_dir, kind='ethernet'): + super(NetInterface, self).__init__() + self.children = [] + self.routes = Route(iface_name, base_sysconf_dir) + self._kind = kind + self._iface_name = iface_name + self._conf['DEVICE'] = iface_name + self._conf['TYPE'] = self.iface_types[kind] + self._base_sysconf_dir = base_sysconf_dir + + @property + def name(self): + return self._iface_name + + @name.setter + def name(self, iface_name): + self._iface_name = iface_name + self._conf['DEVICE'] = iface_name + + @property + def kind(self): + return self._kind + + @kind.setter + def kind(self, kind): + self._kind = kind + self._conf['TYPE'] = self.iface_types[kind] + + @property + def path(self): + return self.iface_fn_tpl % ({'base': self._base_sysconf_dir, + 'name': self.name}) + + def copy(self, copy_children=False, copy_routes=False): + c = NetInterface(self.name, self._base_sysconf_dir, kind=self._kind) + c._conf = self._conf.copy() + if copy_children: + c.children = list(self.children) + if copy_routes: + c.routes = self.routes.copy() + return c + + +class Renderer(renderer.Renderer): + """Renders network information in a /etc/sysconfig format.""" + + # See: https://access.redhat.com/documentation/en-US/\ + # Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/\ + # s1-networkscripts-interfaces.html (or other docs for + # details about this) + + iface_defaults = tuple([ + ('ONBOOT', True), + ('USERCTL', False), + ('NM_CONTROLLED', False), + ('BOOTPROTO', 'none'), + ]) + + # If these keys exist, then there values will be used to form + # a BONDING_OPTS grouping; otherwise no grouping will be set. + bond_tpl_opts = tuple([ + ('bond_mode', "mode=%s"), + ('bond_xmit_hash_policy', "xmit_hash_policy=%s"), + ('bond_miimon', "miimon=%s"), + ]) + + bridge_opts_keys = tuple([ + ('bridge_stp', 'STP'), + ('bridge_ageing', 'AGEING'), + ('bridge_bridgeprio', 'PRIO'), + ]) + + def __init__(self, config=None): + if not config: + config = {} + self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/') + self.netrules_path = config.get( + 'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules') + self.dns_path = config.get('dns_path', 'etc/resolv.conf') + + @classmethod + def _render_iface_shared(cls, iface, iface_cfg): + for k, v in cls.iface_defaults: + iface_cfg[k] = v + for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]: + old_value = iface.get(old_key) + if old_value is not None: + iface_cfg[new_key] = old_value + + @classmethod + def _render_subnet(cls, iface_cfg, route_cfg, subnet): + subnet_type = subnet.get('type') + if subnet_type == 'dhcp6': + iface_cfg['DHCPV6C'] = True + iface_cfg['IPV6INIT'] = True + iface_cfg['BOOTPROTO'] = 'dhcp' + elif subnet_type in ['dhcp4', 'dhcp']: + iface_cfg['BOOTPROTO'] = 'dhcp' + elif subnet_type == 'static': + iface_cfg['BOOTPROTO'] = 'static' + if subnet.get('ipv6'): + iface_cfg['IPV6ADDR'] = subnet['address'] + iface_cfg['IPV6INIT'] = True + else: + iface_cfg['IPADDR'] = subnet['address'] + else: + raise ValueError("Unknown subnet type '%s' found" + " for interface '%s'" % (subnet_type, + iface_cfg.name)) + if 'netmask' in subnet: + iface_cfg['NETMASK'] = subnet['netmask'] + for route in subnet.get('routes', []): + if _is_default_route(route): + if route_cfg.has_set_default: + raise ValueError("Duplicate declaration of default" + " route found for interface '%s'" + % (iface_cfg.name)) + # NOTE(harlowja): ipv6 and ipv4 default gateways + gw_key = 'GATEWAY0' + nm_key = 'NETMASK0' + addr_key = 'ADDRESS0' + # The owning interface provides the default route. + # + # TODO(harlowja): add validation that no other iface has + # also provided the default route? + iface_cfg['DEFROUTE'] = True + if 'gateway' in route: + iface_cfg['GATEWAY'] = route['gateway'] + route_cfg.has_set_default = True + else: + gw_key = 'GATEWAY%s' % route_cfg.last_idx + nm_key = 'NETMASK%s' % route_cfg.last_idx + addr_key = 'ADDRESS%s' % route_cfg.last_idx + route_cfg.last_idx += 1 + for (old_key, new_key) in [('gateway', gw_key), + ('netmask', nm_key), + ('network', addr_key)]: + if old_key in route: + route_cfg[new_key] = route[old_key] + + @classmethod + def _render_bonding_opts(cls, iface_cfg, iface): + bond_opts = [] + for (bond_key, value_tpl) in cls.bond_tpl_opts: + # Seems like either dash or underscore is possible? + bond_keys = [bond_key, bond_key.replace("_", "-")] + for bond_key in bond_keys: + if bond_key in iface: + bond_value = iface[bond_key] + if isinstance(bond_value, (tuple, list)): + bond_value = " ".join(bond_value) + bond_opts.append(value_tpl % (bond_value)) + break + if bond_opts: + iface_cfg['BONDING_OPTS'] = " ".join(bond_opts) + + @classmethod + def _render_physical_interfaces(cls, network_state, iface_contents): + physical_filter = renderer.filter_by_physical + for iface in network_state.iter_interfaces(physical_filter): + iface_name = iface['name'] + iface_subnets = iface.get("subnets", []) + iface_cfg = iface_contents[iface_name] + route_cfg = iface_cfg.routes + if len(iface_subnets) == 1: + cls._render_subnet(iface_cfg, route_cfg, iface_subnets[0]) + elif len(iface_subnets) > 1: + for i, iface_subnet in enumerate(iface_subnets, + start=len(iface.children)): + iface_sub_cfg = iface_cfg.copy() + iface_sub_cfg.name = "%s:%s" % (iface_name, i) + iface.children.append(iface_sub_cfg) + cls._render_subnet(iface_sub_cfg, route_cfg, iface_subnet) + + @classmethod + def _render_bond_interfaces(cls, network_state, iface_contents): + bond_filter = renderer.filter_by_type('bond') + for iface in network_state.iter_interfaces(bond_filter): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + cls._render_bonding_opts(iface_cfg, iface) + iface_master_name = iface['bond-master'] + iface_cfg['MASTER'] = iface_master_name + iface_cfg['SLAVE'] = True + # Ensure that the master interface (and any of its children) + # are actually marked as being bond types... + master_cfg = iface_contents[iface_master_name] + master_cfgs = [master_cfg] + master_cfgs.extend(master_cfg.children) + for master_cfg in master_cfgs: + master_cfg['BONDING_MASTER'] = True + master_cfg.kind = 'bond' + + @staticmethod + def _render_vlan_interfaces(network_state, iface_contents): + vlan_filter = renderer.filter_by_type('vlan') + for iface in network_state.iter_interfaces(vlan_filter): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + iface_cfg['VLAN'] = True + iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')] + + @staticmethod + def _render_dns(network_state, existing_dns_path=None): + content = resolv_conf.ResolvConf("") + if existing_dns_path and os.path.isfile(existing_dns_path): + content = resolv_conf.ResolvConf(util.load_file(existing_dns_path)) + for nameserver in network_state.dns_nameservers: + content.add_nameserver(nameserver) + for searchdomain in network_state.dns_searchdomains: + content.add_search_domain(searchdomain) + return "\n".join([_make_header(';'), str(content)]) + + @classmethod + def _render_bridge_interfaces(cls, network_state, iface_contents): + bridge_filter = renderer.filter_by_type('bridge') + for iface in network_state.iter_interfaces(bridge_filter): + iface_name = iface['name'] + iface_cfg = iface_contents[iface_name] + iface_cfg.kind = 'bridge' + for old_key, new_key in cls.bridge_opts_keys: + if old_key in iface: + iface_cfg[new_key] = iface[old_key] + # Is this the right key to get all the connected interfaces? + for bridged_iface_name in iface.get('bridge_ports', []): + # Ensure all bridged interfaces are correctly tagged + # as being bridged to this interface. + bridged_cfg = iface_contents[bridged_iface_name] + bridged_cfgs = [bridged_cfg] + bridged_cfgs.extend(bridged_cfg.children) + for bridge_cfg in bridged_cfgs: + bridge_cfg['BRIDGE'] = iface_name + + @classmethod + def _render_sysconfig(cls, base_sysconf_dir, network_state): + '''Given state, return /etc/sysconfig files + contents''' + iface_contents = {} + for iface in network_state.iter_interfaces(): + iface_name = iface['name'] + iface_cfg = NetInterface(iface_name, base_sysconf_dir) + cls._render_iface_shared(iface, iface_cfg) + iface_contents[iface_name] = iface_cfg + cls._render_physical_interfaces(network_state, iface_contents) + cls._render_bond_interfaces(network_state, iface_contents) + cls._render_vlan_interfaces(network_state, iface_contents) + cls._render_bridge_interfaces(network_state, iface_contents) + contents = {} + for iface_name, iface_cfg in iface_contents.items(): + if iface_cfg or iface_cfg.children: + contents[iface_cfg.path] = iface_cfg.to_string() + for iface_cfg in iface_cfg.children: + if iface_cfg: + contents[iface_cfg.path] = iface_cfg.to_string() + if iface_cfg.routes: + contents[iface_cfg.routes.path] = iface_cfg.routes.to_string() + return contents + + def render_network_state(self, target, network_state): + base_sysconf_dir = os.path.join(target, self.sysconf_dir) + for path, data in self._render_sysconfig(base_sysconf_dir, + network_state).items(): + util.write_file(path, data) + if self.dns_path: + dns_path = os.path.join(target, self.dns_path) + resolv_content = self._render_dns(network_state, + existing_dns_path=dns_path) + util.write_file(dns_path, resolv_content) + if self.netrules_path: + netrules_content = self._render_persistent_net(network_state) + netrules_path = os.path.join(target, self.netrules_path) + util.write_file(netrules_path, netrules_content) diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py index 6435ace0..09188295 100644 --- a/cloudinit/net/udev.py +++ b/cloudinit/net/udev.py @@ -48,7 +48,7 @@ def generate_udev_rule(interface, mac): compose_udev_equality('DRIVERS', '?*'), compose_udev_attr_equality('address', mac), compose_udev_setting('NAME', interface), - ]) + ]) return '%s\n' % rule # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index e30d6fb5..d8698a5d 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -20,10 +20,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import cloudinit.util as util -from cloudinit.log import logging import re +from cloudinit import log as logging +from cloudinit import util + from prettytable import PrettyTable LOG = logging.getLogger() @@ -163,7 +164,7 @@ def route_info(): def getgateway(): try: routes = route_info() - except: + except Exception: pass else: for r in routes.get('ipv4', []): diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py index 2f767f64..df2b9b4a 100644 --- a/cloudinit/reporting/events.py +++ b/cloudinit/reporting/events.py @@ -33,11 +33,13 @@ class ReportingEvent(object): """Encapsulation of event formatting.""" def __init__(self, event_type, name, description, - origin=DEFAULT_EVENT_ORIGIN, timestamp=time.time()): + origin=DEFAULT_EVENT_ORIGIN, timestamp=None): self.event_type = event_type self.name = name self.description = description self.origin = origin + if timestamp is None: + timestamp = time.time() self.timestamp = timestamp def as_string(self): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 3212d173..dff20ecb 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -4,9 +4,9 @@ import abc import json import six -from ..registry import DictRegistry -from .. import (url_helper, util) -from .. import log as logging +from cloudinit import log as logging +from cloudinit.registry import DictRegistry +from cloudinit import (url_helper, util) LOG = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class LogHandler(ReportingHandler): input_level = level try: level = getattr(logging, level.upper()) - except: + except Exception: LOG.warn("invalid level '%s', using WARN", input_level) level = logging.WARN self.level = level @@ -81,7 +81,7 @@ class WebHookHandler(ReportingHandler): self.endpoint, data=json.dumps(event.as_dict()), timeout=self.timeout, retries=self.retries, ssl_details=self.ssl_details) - except: + except Exception: LOG.warn("failed posting event: %s" % event.as_string()) diff --git a/cloudinit/serial.py b/cloudinit/serial.py new file mode 100644 index 00000000..af45c13e --- /dev/null +++ b/cloudinit/serial.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +from __future__ import absolute_import + +try: + from serial import Serial +except ImportError: + # For older versions of python (ie 2.6) pyserial may not exist and/or + # work and/or be installed, so make a dummy/fake serial that blows up + # when used... + class Serial(object): + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def isOpen(): + return False + + @staticmethod + def write(data): + raise IOError("Unable to perform serial `write` operation," + " pyserial not installed.") + + @staticmethod + def readline(): + raise IOError("Unable to perform serial `readline` operation," + " pyserial not installed.") + + @staticmethod + def flush(): + raise IOError("Unable to perform serial `flush` operation," + " pyserial not installed.") + + @staticmethod + def read(size=1): + raise IOError("Unable to perform serial `read` operation," + " pyserial not installed.") diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index cd61df31..a3529609 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -205,8 +205,7 @@ class DataSourceAltCloud(sources.DataSource): _err.message) return False except OSError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False floppy_dev = '/dev/fd0' diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..8c7e8673 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -27,11 +27,12 @@ import xml.etree.ElementTree as ET from xml.dom import minidom +from cloudinit.sources.helpers.azure import get_metadata_from_fabric + from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -40,7 +41,8 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = [ 'sh', '-xc', - "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] + "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x" +] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -51,7 +53,7 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - }, + }, 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } @@ -60,7 +62,7 @@ BUILTIN_CLOUD_CONFIG = { 'ephemeral0': {'table_type': 'gpt', 'layout': [100], 'overwrite': True}, - }, + }, 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0.1', 'replace_fs': 'ntfs'}], @@ -312,7 +314,7 @@ def support_new_ephemeral(cfg): file_count = 0 try: file_count = util.mount_cb(device, count_files) - except: + except Exception: return None LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count) @@ -421,7 +423,7 @@ def write_files(datadir, files, dirmode=None): elem.text = DEF_PASSWD_REDACTION return ET.tostring(root) except Exception: - LOG.critical("failed to redact userpassword in {}".format(fname)) + LOG.critical("failed to redact userpassword in %s", fname) return cnt if not datadir: diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index b5ee4129..f80956a5 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -4,13 +4,13 @@ # Author: Alexandru Sirbu <alexandru.sirbu@bigstep.com> # -import json import errno +import json from cloudinit import log as logging from cloudinit import sources -from cloudinit import util from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index d7d4e844..d1f806d6 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -19,15 +19,14 @@ from base64 import b64decode import os import re +from cloudinit.cs_utils import Cepko + from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.cs_utils import Cepko LOG = logging.getLogger(__name__) -VALID_DSMODES = ("local", "net", "disabled") - class DataSourceCloudSigma(sources.DataSource): """ @@ -37,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource): http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html """ def __init__(self, sys_cfg, distro, paths): - self.dsmode = 'local' self.cepko = Cepko() self.ssh_public_key = '' sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -77,17 +75,15 @@ class DataSourceCloudSigma(sources.DataSource): try: server_context = self.cepko.all().result server_meta = server_context['meta'] - except: + except Exception: # TODO: check for explicit "config on", and then warn # but since no explicit config is available now, just debug. LOG.debug("CloudSigma: Unable to read from serial port") return False - dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) - if dsmode not in VALID_DSMODES: - LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) - dsmode = 'net' - if dsmode == "disabled" or dsmode != self.dsmode: + self.dsmode = self._determine_dsmode( + [server_meta.get('cloudinit-dsmode')]) + if dsmode == sources.DSMODE_DISABLED: return False base64_fields = server_meta.get('base64_fields', '').split(',') @@ -119,17 +115,13 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] -class DataSourceCloudSigmaNet(DataSourceCloudSigma): - def __init__(self, sys_cfg, distro, paths): - DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - +# Legacy: Must be present in case we load an old pkl object +DataSourceCloudSigmaNet = DataSourceCloudSigma # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 455a4652..4de1f563 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,14 +25,15 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os -import time from socket import inet_ntoa from struct import pack +import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit import sources from cloudinit import url_helper as uhelp -from cloudinit import sources, util +from cloudinit import util LOG = logging.getLogger(__name__) @@ -206,7 +207,8 @@ def get_latest_lease(): latest_mtime = -1 latest_file = None for file_name in lease_files: - if file_name.endswith(".lease") or file_name.endswith(".leases"): + if file_name.startswith("dhclient.") and \ + (file_name.endswith(".lease") or file_name.endswith(".leases")): abs_path = os.path.join(lease_d, file_name) mtime = os.path.getmtime(abs_path) if mtime > latest_mtime: diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 3fa62ef3..3130e618 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,13 +18,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import copy import os from cloudinit import log as logging from cloudinit import sources from cloudinit import util +from cloudinit.net import eni + from cloudinit.sources.helpers import openstack LOG = logging.getLogger(__name__) @@ -35,7 +36,6 @@ DEFAULT_MODE = 'pass' DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') @@ -47,12 +47,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None - self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None self._network_config = None self.network_json = None + self.network_eni = None + self.known_macs = None self.files = {} def __str__(self): @@ -98,38 +99,22 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['version'], - ds_cfg=self.ds_cfg.get('dsmode'), - user=user_dsmode) + self.dsmode = self._determine_dsmode( + [results.get('dsmode'), self.ds_cfg.get('dsmode'), + sources.DSMODE_PASS if results['version'] == 1 else None]) - if dsmode == "disabled": - # most likely user specified + if self.dsmode == sources.DSMODE_DISABLED: return False - # TODO(smoser): fix this, its dirty. - # we want to do some things (writing files and network config) - # only on first boot, and even then, we want to do so in the - # local datasource (so they happen earlier) even if the configured - # dsmode is 'net' or 'pass'. To do this, we check the previous - # instance-id + # This is legacy and sneaky. If dsmode is 'pass' then write + # 'injected files' and apply legacy ENI network format. prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if prev_iid != cur_iid and self.dsmode == "local": + if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS: on_first_boot(results, distro=self.distro) - - # dsmode != self.dsmode here if: - # * dsmode = "pass", pass means it should only copy files and then - # pass to another datasource - # * dsmode = "net" and self.dsmode = "local" - # so that user boothooks would be applied with network, the - # local datasource just gets out of the way, and lets the net claim - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) return False self.source = found @@ -147,15 +132,14 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - try: - self.network_json = results.get('networkdata') - except ValueError as e: - LOG.warn("Invalid content in network-data: %s", e) - self.network_json = None - + # network_config is an /etc/network/interfaces formated file and is + # obsolete compared to networkdata (from network_data.json) but both + # might be present. + self.network_eni = results.get("network_config") + self.network_json = results.get('networkdata') return True - def check_instance_id(self): + def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) @@ -163,41 +147,17 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def network_config(self): if self._network_config is None: if self.network_json is not None: - self._network_config = convert_network_data(self.network_json) + LOG.debug("network config provided via network_json") + self._network_config = openstack.convert_net_json( + self.network_json, known_macs=self.known_macs) + elif self.network_eni is not None: + self._network_config = eni.convert_eni_data(self.network_eni) + LOG.debug("network config provided via converted eni data") + else: + LOG.debug("no network configuration available") return self._network_config -class DataSourceConfigDriveNet(DataSourceConfigDrive): - def __init__(self, sys_cfg, distro, paths): - DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - -def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): - """Determine what mode should be used. - valid values are 'pass', 'disabled', 'local', 'net' - """ - # user passed data trumps everything - if user is not None: - return user - - if ds_cfg is not None: - return ds_cfg - - # at config-drive version 1, the default behavior was pass. That - # meant to not use use it as primary data source, but expect a ec2 metadata - # source. for version 2, we default to 'net', which means - # the DataSourceConfigDriveNet, would be used. - # - # this could change in the future. If there was definitive metadata - # that indicated presense of an openstack metadata service, then - # we could change to 'pass' by default also. The motivation for that - # would be 'cloud-init query' as the web service could be more dynamic - if cfgdrv_ver == 1: - return "pass" - return "net" - - def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ @@ -231,9 +191,12 @@ def on_first_boot(data, distro=None): % (type(data))) net_conf = data.get("network_config", '') if net_conf and distro: - LOG.debug("Updating network interfaces from config drive") + LOG.warn("Updating network interfaces from config drive") distro.apply_network(net_conf) - files = data.get('files', {}) + write_injected_files(data.get('files')) + + +def write_injected_files(files): if files: LOG.debug("Writing %s injected files", len(files)) for (filename, content) in files.items(): @@ -293,132 +256,15 @@ def find_candidate_devs(probe_optical=True): return devices +# Legacy: Must be present in case we load an old pkl object +DataSourceConfigDriveNet = DataSourceConfigDrive + # Used to match classes to dependencies datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM,)), ] # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) - - -# Convert OpenStack ConfigDrive NetworkData json to network_config yaml -def convert_network_data(network_json=None): - """Return a dictionary of network_config by parsing provided - OpenStack ConfigDrive NetworkData json format - - OpenStack network_data.json provides a 3 element dictionary - - "links" (links are network devices, physical or virtual) - - "networks" (networks are ip network configurations for one or more - links) - - services (non-ip services, like dns) - - networks and links are combined via network items referencing specific - links via a 'link_id' which maps to a links 'id' field. - - To convert this format to network_config yaml, we first iterate over the - links and then walk the network list to determine if any of the networks - utilize the current link; if so we generate a subnet entry for the device - - We also need to map network_data.json fields to network_config fields. For - example, the network_data links 'id' field is equivalent to network_config - 'name' field for devices. We apply more of this mapping to the various - link types that we encounter. - - There are additional fields that are populated in the network_data.json - from OpenStack that are not relevant to network_config yaml, so we - enumerate a dictionary of valid keys for network_yaml and apply filtering - to drop these superflous keys from the network_config yaml. - """ - if network_json is None: - return None - - # dict of network_config key for filtering network_json - valid_keys = { - 'physical': [ - 'name', - 'type', - 'mac_address', - 'subnets', - 'params', - ], - 'subnet': [ - 'type', - 'address', - 'netmask', - 'broadcast', - 'metric', - 'gateway', - 'pointopoint', - 'mtu', - 'scope', - 'dns_nameservers', - 'dns_search', - 'routes', - ], - } - - links = network_json.get('links', []) - networks = network_json.get('networks', []) - services = network_json.get('services', []) - - config = [] - for link in links: - subnets = [] - cfg = {k: v for k, v in link.items() - if k in valid_keys['physical']} - cfg.update({'name': link['id']}) - for network in [net for net in networks - if net['link'] == link['id']]: - subnet = {k: v for k, v in network.items() - if k in valid_keys['subnet']} - if 'dhcp' in network['type']: - t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' - subnet.update({ - 'type': t, - }) - else: - subnet.update({ - 'type': 'static', - 'address': network.get('ip_address'), - }) - subnets.append(subnet) - cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: - cfg.update({ - 'type': 'physical', - 'mac_address': link['ethernet_mac_address']}) - elif link['type'] in ['bond']: - params = {} - for k, v in link.items(): - if k == 'bond_links': - continue - elif k.startswith('bond'): - params.update({k: v}) - cfg.update({ - 'bond_interfaces': copy.deepcopy(link['bond_links']), - 'params': params, - }) - elif link['type'] in ['vlan']: - cfg.update({ - 'name': "%s.%s" % (link['vlan_link'], - link['vlan_id']), - 'vlan_link': link['vlan_link'], - 'vlan_id': link['vlan_id'], - 'mac_address': link['vlan_mac_address'], - }) - else: - raise ValueError( - 'Unknown network_data link type: %s' % link['type']) - - config.append(cfg) - - for service in services: - cfg = service - cfg.update({'type': 'nameserver'}) - config.append(cfg) - - return {'version': 1, 'config': config} diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 12e863d2..44a17a00 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from cloudinit import ec2_utils from cloudinit import log as logging -from cloudinit import util from cloudinit import sources -from cloudinit import ec2_utils +from cloudinit import util import functools diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 7e7fc033..c660a350 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -18,14 +18,14 @@ from base64 import b64decode from cloudinit import log as logging -from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -71,7 +71,7 @@ class DataSourceGCE(sources.DataSource): index = public_key.index(':') if index > 0: return public_key[(index + 1):] - except: + except Exception: return public_key def get_data(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 74d0e5ec..cdc9eef5 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,6 +24,7 @@ import errno import os from cloudinit import log as logging +from cloudinit.net import eni from cloudinit import sources from cloudinit import util @@ -33,9 +34,7 @@ LOG = logging.getLogger(__name__) class DataSourceNoCloud(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None - self.cmdline_id = "ds=nocloud" self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] self.seed_dir = None @@ -58,10 +57,10 @@ class DataSourceNoCloud(sources.DataSource): try: # Parse the kernel command line, getting data passed in md = {} - if parse_cmdline_data(self.cmdline_id, md): + if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) - except: + except Exception: util.logexc(LOG, "Unable to parse command line data") return False @@ -123,12 +122,6 @@ class DataSourceNoCloud(sources.DataSource): mydata = _merge_new_seed(mydata, seeded) - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. If they want - # dsmode of local, then they must specify that. - if 'dsmode' not in mydata['meta-data']: - mydata['meta-data']['dsmode'] = "net" - LOG.debug("Using data from %s", dev) found.append(dev) break @@ -144,7 +137,6 @@ class DataSourceNoCloud(sources.DataSource): if len(found) == 0: return False - seeded_network = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less @@ -160,10 +152,6 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if (mydata['meta-data'].get('network-interfaces') or - mydata.get('network-config')): - seeded_network = self.dsmode - # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) @@ -179,35 +167,21 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) - netdata = {'format': None, 'data': None} - if mydata['meta-data'].get('network-interfaces'): - netdata['format'] = 'interfaces' - netdata['data'] = mydata['meta-data']['network-interfaces'] - elif mydata.get('network-config'): - netdata['format'] = 'network-config' - netdata['data'] = mydata['network-config'] - - # if this is the local datasource or 'seedfrom' was used - # and the source of the seed was self.dsmode. - # Then see if there is network config to apply. - # note this is obsolete network-interfaces style seeding. - if self.dsmode in ("local", seeded_network): - if mydata['meta-data'].get('network-interfaces'): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network( - mydata['meta-data']['network-interfaces']) - - if mydata['meta-data']['dsmode'] == self.dsmode: - self.seed = ",".join(found) - self.metadata = mydata['meta-data'] - self.userdata_raw = mydata['user-data'] - self.vendordata_raw = mydata['vendor-data'] - self._network_config = mydata['network-config'] - return True + self.dsmode = self._determine_dsmode( + [mydata['meta-data'].get('dsmode')]) - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - mydata['meta-data']['dsmode']) - return False + if self.dsmode == sources.DSMODE_DISABLED: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) + return False + + self.seed = ",".join(found) + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] + self._network_eni = mydata['meta-data'].get('network-interfaces') + return True def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid @@ -219,26 +193,27 @@ class DataSourceNoCloud(sources.DataSource): # LP: #1568150 need getattr in the case that an old class object # has been loaded from a pickled file and now executing new source. dirs = getattr(self, 'seed_dirs', [self.seed_dir]) - quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id, - dirs=dirs) + quick_id = _quick_read_instance_id(dirs=dirs) if not quick_id: return None return quick_id == current @property def network_config(self): + if self._network_config is None: + if self._network_eni is not None: + self._network_config = eni.convert_eni_data(self._network_eni) return self._network_config -def _quick_read_instance_id(cmdline_id, dirs=None): +def _quick_read_instance_id(dirs=None): if dirs is None: dirs = [] iid_key = 'instance-id' - if cmdline_id is None: - fill = {} - if parse_cmdline_data(cmdline_id, fill) and iid_key in fill: - return fill[iid_key] + fill = {} + if load_cmdline_data(fill) and iid_key in fill: + return fill[iid_key] for d in dirs: if d is None: @@ -254,8 +229,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None): return None +def load_cmdline_data(fill, cmdline=None): + pairs = [("ds=nocloud", sources.DSMODE_LOCAL), + ("ds=nocloud-net", sources.DSMODE_NETWORK)] + for idstr, dsmode in pairs: + if parse_cmdline_data(idstr, fill, cmdline): + # if dsmode was explicitly in the commanad line, then + # prefer it to the dsmode based on the command line id + if 'dsmode' not in fill: + fill['dsmode'] = dsmode + return True + return False + + # Returns true or false indicating if cmdline indicated -# that this module should be used +# that this module should be used. Updates dictionary 'fill' +# with data that was found. # Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): @@ -288,7 +277,7 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): continue try: (k, v) = item.split("=", 1) - except: + except Exception: k = item v = None if k in s2l: @@ -319,9 +308,7 @@ def _merge_new_seed(cur, seeded): class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) - self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.dsmode = "net" # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2a6cd050..43347cfb 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -30,16 +30,23 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from .helpers.vmware.imc.config import Config -from .helpers.vmware.imc.config_file import ConfigFile -from .helpers.vmware.imc.config_nic import NicConfigurator -from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum -from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum -from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum -from .helpers.vmware.imc.guestcust_util import ( - set_customization_status, + +from cloudinit.sources.helpers.vmware.imc.config \ + import Config +from cloudinit.sources.helpers.vmware.imc.config_file \ + import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic \ + import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_event \ + import GuestCustEventEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_state \ + import GuestCustStateEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( + enable_nics, get_nics_to_enable, - enable_nics + set_customization_status ) LOG = logging.getLogger(__name__) @@ -262,7 +269,7 @@ def read_ovf_environment(contents): elif prop == "user-data": try: ud = base64.decodestring(val) - except: + except Exception: ud = val return (md, ud, cfg) @@ -277,7 +284,7 @@ def get_ovf_env(dirname): try: contents = util.load_file(full_fn) return (fname, contents) - except: + except Exception: util.logexc(LOG, "Failed loading ovf file %s", full_fn) return (None, False) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 681f3a96..7b3a76b9 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,16 +37,13 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" -DEFAULT_MODE = 'net' DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] -VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') @@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('DSMODE', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s", user_dsmode) - user_dsmode = None - - # decide dsmode - if user_dsmode: - dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): - dsmode = self.ds_cfg.get('dsmode') - else: - dsmode = DEFAULT_MODE - - if dsmode == "disabled": - # most likely user specified - return False - - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) + self.dsmode = self._determine_dsmode( + [results.get('DSMODE'), self.ds_cfg.get('dsmode')]) - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + if self.dsmode == sources.DSMODE_DISABLED: return False self.seed = seed + self.network_eni = results.get("network_config") self.metadata = md self.userdata_raw = results.get('userdata') return True def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == 'net': + if self.dsmode == sources.DSMODE_NETWORK: resolve_ip = True else: resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) -class DataSourceOpenNebulaNet(DataSourceOpenNebula): - def __init__(self, sys_cfg, distro, paths): - DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - class NonContextDiskDir(Exception): pass @@ -443,10 +415,12 @@ def read_context_disk_dir(source_dir, asuser=None): return results +# Legacy: Must be present in case we load an old pkl object +DataSourceOpenNebulaNet = DataSourceOpenNebula + # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), - (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 3af17b10..c06d17f3 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack" DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("net", "disabled") class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) - self.dsmode = 'net' self.metadata_address = None self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None @@ -103,7 +101,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address = url2base.get(avail_url) return bool(avail_url) - def get_data(self): + def get_data(self, retries=5, timeout=5): try: if not self.wait_for_metadata_service(): return False @@ -115,7 +113,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'retries': retries, + 'timeout': timeout}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): @@ -123,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address) return False - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - if user_dsmode == 'disabled': + self.dsmode = self._determine_dsmode([results.get('dsmode')]) + if self.dsmode == sources.DSMODE_DISABLED: return False md = results.get('metadata', {}) @@ -153,8 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return sources.instance_id_matches_system_uuid(self.get_instance_id()) -def read_metadata_service(base_url, ssl_details=None): - reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) +def read_metadata_service(base_url, ssl_details=None, + timeout=5, retries=5): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details, + timeout=timeout, retries=retries) return reader.read_v2() diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6cbd8dfa..08bc132b 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -32,21 +32,19 @@ # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html # Comments with "@datadictionary" are snippets of the definition +import base64 import binascii -import contextlib +import json import os import random import re import socket -import stat - -import serial from cloudinit import log as logging +from cloudinit import serial from cloudinit import sources from cloudinit import util - LOG = logging.getLogger(__name__) SMARTOS_ATTRIB_MAP = { @@ -64,14 +62,36 @@ SMARTOS_ATTRIB_MAP = { 'operator-script': ('sdc:operator-script', False), } +SMARTOS_ATTRIB_JSON = { + # Cloud-init Key : (SmartOS Key known JSON) + 'network-data': 'sdc:nics', +} + +SMARTOS_ENV_LX_BRAND = "lx-brand" +SMARTOS_ENV_KVM = "kvm" + DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] +NO_BASE64_DECODE = [ + 'iptables_disable', + 'motd_sys_info', + 'root_authorized_keys', + 'sdc:datacenter_name', + 'sdc:uuid' + 'user-data', + 'user-script', +] + +METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' +SERIAL_DEVICE = '/dev/ttyS1' +SERIAL_TIMEOUT = 60 + # BUILT-IN DATASOURCE CONFIGURATION # The following is the built-in configuration. If the values # are not set via the system configuration, then these default # will be used: # serial_device: which serial device to use for the meta-data -# seed_timeout: how long to wait on the device +# serial_timeout: how long to wait on the device # no_base64_decode: values which are not base64 encoded and # are fetched directly from SmartOS, not meta-data values # base64_keys: meta-data keys that are delivered in base64 @@ -81,16 +101,10 @@ DS_CFG_PATH = ['datasource', DS_NAME] # fs_setup: describes how to format the ephemeral drive # BUILTIN_DS_CONFIG = { - 'serial_device': '/dev/ttyS1', - 'metadata_sockfile': '/native/.zonecontrol/metadata.sock', - 'seed_timeout': 60, - 'no_base64_decode': ['root_authorized_keys', - 'motd_sys_info', - 'iptables_disable', - 'user-data', - 'user-script', - 'sdc:datacenter_name', - 'sdc:uuid'], + 'serial_device': SERIAL_DEVICE, + 'serial_timeout': SERIAL_TIMEOUT, + 'metadata_sockfile': METADATA_SOCKFILE, + 'no_base64_decode': NO_BASE64_DECODE, 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -154,59 +168,41 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): + _unset = "_unset" + smartos_type = _unset + md_client = _unset + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.is_smartdc = None self.ds_cfg = util.mergemanydict([ self.ds_cfg, util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG]) self.metadata = {} + self.network_data = None + self._network_config = None - # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but - # report 'BrandZ virtual linux' as the kernel version - if os.uname()[3].lower() == 'brandz virtual linux': - LOG.debug("Host is SmartOS, guest in Zone") - self.is_smartdc = True - self.smartos_type = 'lx-brand' - self.cfg = {} - self.seed = self.ds_cfg.get("metadata_sockfile") - else: - self.is_smartdc = True - self.smartos_type = 'kvm' - self.seed = self.ds_cfg.get("serial_device") - self.cfg = BUILTIN_CLOUD_CONFIG - self.seed_timeout = self.ds_cfg.get("serial_timeout") - self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode') - self.b64_keys = self.ds_cfg.get('base64_keys') - self.b64_all = self.ds_cfg.get('base64_all') self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self._init() + def __str__(self): root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def _get_seed_file_object(self): - if not self.seed: - raise AttributeError("seed device is not set") - - if self.smartos_type == 'lx-brand': - if not stat.S_ISSOCK(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a socket", self.seed) - return None - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.seed) - return sock.makefile('rwb') - else: - if not stat.S_ISCHR(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a character device") - return None - ser = serial.Serial(self.seed, timeout=self.seed_timeout) - if not ser.isOpen(): - raise SystemError("Unable to open %s" % self.seed) - return ser - return None + return "%s [client=%s]" % (root, self.md_client) + + def _init(self): + if self.smartos_type == self._unset: + self.smartos_type = get_smartos_environ() + if self.smartos_type is None: + self.md_client = None + + if self.md_client == self._unset: + self.md_client = jmc_client_factory( + smartos_type=self.smartos_type, + metadata_sockfile=self.ds_cfg['metadata_sockfile'], + serial_device=self.ds_cfg['serial_device'], + serial_timeout=self.ds_cfg['serial_timeout']) def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -225,50 +221,26 @@ class DataSourceSmartOS(sources.DataSource): '/'.join([svc_path, 'provision_success'])) def get_data(self): + self._init() + md = {} ud = "" - if not device_exists(self.seed): - LOG.debug("No metadata device '%s' found for SmartOS datasource", - self.seed) - return False - - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmidcode in dmi_data() crashes kvm process - LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)") + if not self.smartos_type: + LOG.debug("Not running on smartos") return False - # SDC KVM instances will provide dmi data, LX-brand does not - if self.smartos_type == 'kvm': - dmi_info = dmi_data() - if dmi_info is None: - LOG.debug("No dmidata utility found") - return False - - system_type = dmi_info - if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS. system_type=%s", - system_type) - return False - LOG.debug("Host is SmartOS, guest in KVM") - - seed_obj = self._get_seed_file_object() - if seed_obj is None: - LOG.debug('Seed file object not found.') + if not self.md_client.exists(): + LOG.debug("No metadata device '%r' found for SmartOS datasource", + self.md_client) return False - with contextlib.closing(seed_obj) as seed: - b64_keys = self.query('base64_keys', seed, strip=True, b64=False) - if b64_keys is not None: - self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] - b64_all = self.query('base64_all', seed, strip=True, b64=False) - if b64_all is not None: - self.b64_all = util.is_true(b64_all) + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): + smartos_noun, strip = attribute + md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): - smartos_noun, strip = attribute - md[ci_noun] = self.query(smartos_noun, seed, strip=strip) + for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): + md[ci_noun] = self.md_client.get_json(smartos_noun) # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then @@ -318,6 +290,7 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] + self.network_data = md['network-data'] self._set_provisioned() return True @@ -326,69 +299,20 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - return self.cfg + if self.smartos_type == SMARTOS_ENV_KVM: + return BUILTIN_CLOUD_CONFIG + return {} def get_instance_id(self): return self.metadata['instance-id'] - def query(self, noun, seed_file, strip=False, default=None, b64=None): - if b64 is None: - if noun in self.smartos_no_base64: - b64 = False - elif self.b64_all or noun in self.b64_keys: - b64 = True - - return self._query_data(noun, seed_file, strip=strip, - default=default, b64=b64) - - def _query_data(self, noun, seed_file, strip=False, - default=None, b64=None): - """Makes a request via "GET <NOUN>" - - In the response, the first line is the status, while subsequent - lines are is the value. A blank line with a "." is used to - indicate end of response. - - If the response is expected to be base64 encoded, then set - b64encoded to true. Unfortantely, there is no way to know if - something is 100% encoded, so this method relies on being told - if the data is base64 or not. - """ - - if not noun: - return False - - response = JoyentMetadataClient(seed_file).get_metadata(noun) - - if response is None: - return default - - if b64 is None: - b64 = self._query_data('b64-%s' % noun, seed_file, b64=False, - default=False, strip=True) - b64 = util.is_true(b64) - - resp = None - if b64 or strip: - resp = "".join(response).rstrip() - else: - resp = "".join(response) - - if b64: - try: - return util.b64d(resp) - # Bogus input produces different errors in Python 2 and 3; - # catch both. - except (TypeError, binascii.Error): - LOG.warn("Failed base64 decoding key '%s'", noun) - return resp - - return resp - - -def device_exists(device): - """Symplistic method to determine if the device exists or not""" - return os.path.exists(device) + @property + def network_config(self): + if self._network_config is None: + if self.network_data is not None: + self._network_config = ( + convert_smartos_network_data(self.network_data)) + return self._network_config class JoyentMetadataFetchException(Exception): @@ -407,8 +331,11 @@ class JoyentMetadataClient(object): r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)' r'( (?P<payload>.+))?)') - def __init__(self, metasource): - self.metasource = metasource + def __init__(self, smartos_type=None, fp=None): + if smartos_type is None: + smartos_type = get_smartos_environ() + self.smartos_type = smartos_type + self.fp = fp def _checksum(self, body): return '{0:08x}'.format( @@ -436,37 +363,229 @@ class JoyentMetadataClient(object): LOG.debug('Value "%s" found.', value) return value - def get_metadata(self, metadata_key): - LOG.debug('Fetching metadata key "%s"...', metadata_key) + def request(self, rtype, param=None): request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) - message_body = '{0} GET {1}'.format(request_id, - util.b64e(metadata_key)) + message_body = ' '.join((request_id, rtype,)) + if param: + message_body += ' ' + base64.b64encode(param.encode()).decode() msg = 'V2 {0} {1} {2}\n'.format( len(message_body), self._checksum(message_body), message_body) LOG.debug('Writing "%s" to metadata transport.', msg) - self.metasource.write(msg.encode('ascii')) - self.metasource.flush() + + need_close = False + if not self.fp: + self.open_transport() + need_close = True + + self.fp.write(msg.encode('ascii')) + self.fp.flush() response = bytearray() - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) while response[-1:] != b'\n': - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) + + if need_close: + self.close_transport() + response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) if 'SUCCESS' not in response: return None - return self._get_value_from_frame(request_id, response) + value = self._get_value_from_frame(request_id, response) + return value + + def get(self, key, default=None, strip=False): + result = self.request(rtype='GET', param=key) + if result is None: + return default + if result and strip: + result = result.strip() + return result + + def get_json(self, key, default=None): + result = self.get(key, default=default) + if result is None: + return default + return json.loads(result) + + def list(self): + result = self.request(rtype='KEYS') + if result: + result = result.split('\n') + return result + + def put(self, key, val): + param = b' '.join([base64.b64encode(i.encode()) + for i in (key, val)]).decode() + return self.request(rtype='PUT', param=param) + + def delete(self, key): + return self.request(rtype='DELETE', param=key) + + def close_transport(self): + if self.fp: + self.fp.close() + self.fp = None + + def __enter__(self): + if self.fp: + return self + self.open_transport() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close_transport() + return + + def open_transport(self): + raise NotImplementedError + + +class JoyentMetadataSocketClient(JoyentMetadataClient): + def __init__(self, socketpath): + self.socketpath = socketpath + + def open_transport(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(self.socketpath) + self.fp = sock.makefile('rwb') + + def exists(self): + return os.path.exists(self.socketpath) + + def __repr__(self): + return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) + + +class JoyentMetadataSerialClient(JoyentMetadataClient): + def __init__(self, device, timeout=10, smartos_type=None): + super(JoyentMetadataSerialClient, self).__init__(smartos_type) + self.device = device + self.timeout = timeout + + def exists(self): + return os.path.exists(self.device) + + def open_transport(self): + ser = serial.Serial(self.device, timeout=self.timeout) + if not ser.isOpen(): + raise SystemError("Unable to open %s" % self.device) + self.fp = ser + + def __repr__(self): + return "%s(device=%s, timeout=%s)" % ( + self.__class__.__name__, self.device, self.timeout) + + +class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): + """V1 of the protocol was not safe for all values. + Thus, we allowed the user to pass values in as base64 encoded. + Users may still reasonably expect to be able to send base64 data + and have it transparently decoded. So even though the V2 format is + now used, and is safe (using base64 itself), we keep legacy support. + + The way for a user to do this was: + a.) specify 'base64_keys' key whose value is a comma delimited + list of keys that were base64 encoded. + b.) base64_all: string interpreted as a boolean that indicates + if all keys are base64 encoded. + c.) set a key named b64-<keyname> with a boolean indicating that + <keyname> is base64 encoded.""" + + def __init__(self, device, timeout=10, smartos_type=None): + s = super(JoyentMetadataLegacySerialClient, self) + s.__init__(device, timeout, smartos_type) + self.base64_keys = None + self.base64_all = None + + def _init_base64_keys(self, reset=False): + if reset: + self.base64_keys = None + self.base64_all = None + + keys = None + if self.base64_all is None: + keys = self.list() + if 'base64_all' in keys: + self.base64_all = util.is_true(self._get("base64_all")) + else: + self.base64_all = False + + if self.base64_all: + # short circuit if base64_all is true + return + + if self.base64_keys is None: + if keys is None: + keys = self.list() + b64_keys = set() + if 'base64_keys' in keys: + b64_keys = set(self._get("base64_keys").split(",")) + + # now add any b64-<keyname> that has a true value + for key in [k[3:] for k in keys if k.startswith("b64-")]: + if util.is_true(self._get(key)): + b64_keys.add(key) + else: + if key in b64_keys: + b64_keys.remove(key) + + self.base64_keys = b64_keys + + def _get(self, key, default=None, strip=False): + return (super(JoyentMetadataLegacySerialClient, self). + get(key, default=default, strip=strip)) + + def is_b64_encoded(self, key, reset=False): + if key in NO_BASE64_DECODE: + return False + + self._init_base64_keys(reset=reset) + if self.base64_all: + return True + + return key in self.base64_keys + + def get(self, key, default=None, strip=False): + mdefault = object() + val = self._get(key, strip=False, default=mdefault) + if val is mdefault: + return default + + if self.is_b64_encoded(key): + try: + val = base64.b64decode(val.encode()).decode() + # Bogus input produces different errors in Python 2 and 3 + except (TypeError, binascii.Error): + LOG.warn("Failed base64 decoding key '%s': %s", key, val) + + if strip: + val = val.strip() + + return val -def dmi_data(): - sys_type = util.read_dmi_data("system-product-name") +def jmc_client_factory( + smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, + serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, + uname_version=None): - if not sys_type: + if smartos_type is None: + smartos_type = get_smartos_environ(uname_version) + + if smartos_type is None: return None + elif smartos_type == SMARTOS_ENV_KVM: + return JoyentMetadataLegacySerialClient( + device=serial_device, timeout=serial_timeout, + smartos_type=smartos_type) + elif smartos_type == SMARTOS_ENV_LX_BRAND: + return JoyentMetadataSocketClient(socketpath=metadata_sockfile) - return sys_type + raise ValueError("Unknown value for smartos_type: %s" % smartos_type) def write_boot_content(content, content_f, link=None, shebang=False, @@ -522,15 +641,141 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.ensure_dir(os.path.dirname(link)) os.symlink(content_f, link) except IOError as e: - util.logexc(LOG, "failed establishing content link", e) + util.logexc(LOG, "failed establishing content link: %s", e) + + +def get_smartos_environ(uname_version=None, product_name=None, + uname_arch=None): + uname = os.uname() + if uname_arch is None: + uname_arch = uname[4] + + if uname_arch.startswith("arm") or uname_arch == "aarch64": + return None + + # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but + # report 'BrandZ virtual linux' as the kernel version + if uname_version is None: + uname_version = uname[3] + if uname_version.lower() == 'brandz virtual linux': + return SMARTOS_ENV_LX_BRAND + + if product_name is None: + system_type = util.read_dmi_data("system-product-name") + else: + system_type = product_name + + if system_type and 'smartdc' in system_type.lower(): + return SMARTOS_ENV_KVM + + return None + + +# Covert SMARTOS 'sdc:nics' data to network_config yaml +def convert_smartos_network_data(network_data=None): + """Return a dictionary of network_config by parsing provided + SMARTOS sdc:nics configuration data + + sdc:nics data is a dictionary of properties of a nic and the ip + configuration desired. Additional nic dictionaries are appended + to the list. + + Converting the format is straightforward though it does include + duplicate information as well as data which appears to be relevant + to the hostOS rather than the guest. + + For each entry in the nics list returned from query sdc:nics, we + create a type: physical entry, and extract the interface properties: + 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining + keys are related to ip configuration. For each ip in the 'ips' list + we create a subnet entry under 'subnets' pairing the ip to a one in + the 'gateways' list. + """ + + valid_keys = { + 'physical': [ + 'mac_address', + 'mtu', + 'name', + 'params', + 'subnets', + 'type', + ], + 'subnet': [ + 'address', + 'broadcast', + 'dns_nameservers', + 'dns_search', + 'gateway', + 'metric', + 'netmask', + 'pointopoint', + 'routes', + 'scope', + 'type', + ], + } + + config = [] + for nic in network_data: + cfg = {k: v for k, v in nic.items() + if k in valid_keys['physical']} + cfg.update({ + 'type': 'physical', + 'name': nic['interface']}) + if 'mac' in nic: + cfg.update({'mac_address': nic['mac']}) + + subnets = [] + for ip, gw in zip(nic['ips'], nic['gateways']): + subnet = {k: v for k, v in nic.items() + if k in valid_keys['subnet']} + subnet.update({ + 'type': 'static', + 'address': ip, + 'gateway': gw, + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + config.append(cfg) + + return {'version': 1, 'config': config} # Used to match classes to dependencies datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), ] # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + import sys + jmc = jmc_client_factory() + if jmc is None: + print("Do not appear to be on smartos.") + sys.exit(1) + if len(sys.argv) == 1: + keys = (list(SMARTOS_ATTRIB_JSON.keys()) + + list(SMARTOS_ATTRIB_MAP.keys())) + else: + keys = sys.argv[1:] + + data = {} + for key in keys: + if key in SMARTOS_ATTRIB_JSON: + keyname = SMARTOS_ATTRIB_JSON[key] + data[key] = jmc.get_json(keyname) + else: + if key in SMARTOS_ATTRIB_MAP: + keyname, strip = SMARTOS_ATTRIB_MAP[key] + else: + keyname, strip = (key, False) + val = jmc.get(keyname, strip=strip) + data[key] = jmc.get(keyname, strip=strip) + + print(json.dumps(data, indent=1)) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 6bf2c33b..2a6b8d90 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -34,6 +34,13 @@ from cloudinit import util from cloudinit.filters import launch_index from cloudinit.reporting import events +DSMODE_DISABLED = "disabled" +DSMODE_LOCAL = "local" +DSMODE_NETWORK = "net" +DSMODE_PASS = "pass" + +VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -45,10 +52,9 @@ class DataSourceNotFoundException(Exception): pass +@six.add_metaclass(abc.ABCMeta) class DataSource(object): - __metaclass__ = abc.ABCMeta - def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro @@ -58,6 +64,7 @@ class DataSource(object): self.userdata_raw = None self.vendordata = None self.vendordata_raw = None + self.dsmode = DSMODE_NETWORK # find the datasource config name. # remove 'DataSource' from classname on front, and remove 'Net' on end. @@ -224,10 +231,35 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @staticmethod + def _determine_dsmode(candidates, default=None, valid=None): + # return the first candidate that is non None, warn if not valid + if default is None: + default = DSMODE_NETWORK + + if valid is None: + valid = VALID_DSMODES + + for candidate in candidates: + if candidate is None: + continue + if candidate in valid: + return candidate + else: + LOG.warn("invalid dsmode '%s', using default=%s", + candidate, default) + return default + + return default + @property def network_config(self): return None + @property + def first_instance_boot(self): + return + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 018cac6d..63ccf10e 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -5,6 +5,7 @@ import socket import struct import tempfile import time + from contextlib import contextmanager from xml.etree import ElementTree @@ -220,7 +221,7 @@ class WALinuxAgentShim(object): if 'unknown-245' in line: value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') if value is None: - raise Exception('No endpoint found in DHCP config.') + raise ValueError('No endpoint found in DHCP config.') endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1aa6bbae..d52cb56a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -28,6 +28,7 @@ import six from cloudinit import ec2_utils from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import url_helper from cloudinit import util @@ -145,8 +146,8 @@ class SourceMixin(object): return device +@six.add_metaclass(abc.ABCMeta) class BaseReader(object): - __metaclass__ = abc.ABCMeta def __init__(self, base_path): self.base_path = base_path @@ -156,7 +157,7 @@ class BaseReader(object): pass @abc.abstractmethod - def _path_read(self, path): + def _path_read(self, path, decode=False): pass @abc.abstractmethod @@ -190,14 +191,14 @@ class BaseReader(object): versions_available) return selected_version - def _read_content_path(self, item): + def _read_content_path(self, item, decode=False): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") valid_pieces = [p for p in path_pieces if len(p)] if not valid_pieces: raise BrokenMetadata("Item %s has no valid content path" % (item)) path = self._path_join(self.base_path, "openstack", *path_pieces) - return self._path_read(path) + return self._path_read(path, decode=decode) def read_v2(self): """Reads a version 2 formatted location. @@ -298,7 +299,8 @@ class BaseReader(object): net_item = metadata.get("network_config", None) if net_item: try: - results['network_config'] = self._read_content_path(net_item) + content = self._read_content_path(net_item, decode=True) + results['network_config'] = content except IOError as e: raise BrokenMetadata("Failed to read network" " configuration: %s" % (e)) @@ -333,8 +335,8 @@ class ConfigDriveReader(BaseReader): components = [base] + list(add_ons) return os.path.join(*components) - def _path_read(self, path): - return util.load_file(path, decode=False) + def _path_read(self, path, decode=False): + return util.load_file(path, decode=decode) def _fetch_available_versions(self): if self._versions is None: @@ -446,7 +448,7 @@ class MetadataReader(BaseReader): self._versions = found return self._versions - def _path_read(self, path): + def _path_read(self, path, decode=False): def should_retry_cb(_request_args, cause): try: @@ -463,7 +465,10 @@ class MetadataReader(BaseReader): ssl_details=self.ssl_details, timeout=self.timeout, exception_cb=should_retry_cb) - return response.contents + if decode: + return response.contents.decode() + else: + return response.contents def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) @@ -474,8 +479,152 @@ class MetadataReader(BaseReader): retries=self.retries) +# Convert OpenStack ConfigDrive NetworkData json to network_config yaml +def convert_net_json(network_json=None, known_macs=None): + """Return a dictionary of network_config by parsing provided + OpenStack ConfigDrive NetworkData json format + + OpenStack network_data.json provides a 3 element dictionary + - "links" (links are network devices, physical or virtual) + - "networks" (networks are ip network configurations for one or more + links) + - services (non-ip services, like dns) + + networks and links are combined via network items referencing specific + links via a 'link_id' which maps to a links 'id' field. + + To convert this format to network_config yaml, we first iterate over the + links and then walk the network list to determine if any of the networks + utilize the current link; if so we generate a subnet entry for the device + + We also need to map network_data.json fields to network_config fields. For + example, the network_data links 'id' field is equivalent to network_config + 'name' field for devices. We apply more of this mapping to the various + link types that we encounter. + + There are additional fields that are populated in the network_data.json + from OpenStack that are not relevant to network_config yaml, so we + enumerate a dictionary of valid keys for network_yaml and apply filtering + to drop these superflous keys from the network_config yaml. + """ + if network_json is None: + return None + + # dict of network_config key for filtering network_json + valid_keys = { + 'physical': [ + 'name', + 'type', + 'mac_address', + 'subnets', + 'params', + 'mtu', + ], + 'subnet': [ + 'type', + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'scope', + 'dns_nameservers', + 'dns_search', + 'routes', + ], + } + + links = network_json.get('links', []) + networks = network_json.get('networks', []) + services = network_json.get('services', []) + + config = [] + for link in links: + subnets = [] + cfg = {k: v for k, v in link.items() + if k in valid_keys['physical']} + # 'name' is not in openstack spec yet, but we will support it if it is + # present. The 'id' in the spec is currently implemented as the host + # nic's name, meaning something like 'tap-adfasdffd'. We do not want + # to name guest devices with such ugly names. + if 'name' in link: + cfg['name'] = link['name'] + + for network in [n for n in networks + if n['link'] == link['id']]: + subnet = {k: v for k, v in network.items() + if k in valid_keys['subnet']} + if 'dhcp' in network['type']: + t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' + subnet.update({ + 'type': t, + }) + else: + subnet.update({ + 'type': 'static', + 'address': network.get('ip_address'), + }) + if network['type'] == 'ipv4': + subnet['ipv4'] = True + if network['type'] == 'ipv6': + subnet['ipv6'] = True + subnets.append(subnet) + cfg.update({'subnets': subnets}) + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']: + cfg.update({ + 'type': 'physical', + 'mac_address': link['ethernet_mac_address']}) + elif link['type'] in ['bond']: + params = {} + for k, v in link.items(): + if k == 'bond_links': + continue + elif k.startswith('bond'): + params.update({k: v}) + cfg.update({ + 'bond_interfaces': copy.deepcopy(link['bond_links']), + 'params': params, + }) + elif link['type'] in ['vlan']: + cfg.update({ + 'name': "%s.%s" % (link['vlan_link'], + link['vlan_id']), + 'vlan_link': link['vlan_link'], + 'vlan_id': link['vlan_id'], + 'mac_address': link['vlan_mac_address'], + }) + else: + raise ValueError( + 'Unknown network_data link type: %s' % link['type']) + + config.append(cfg) + + need_names = [d for d in config + if d.get('type') == 'physical' and 'name' not in d] + + if need_names: + if known_macs is None: + known_macs = net.get_interfaces_by_mac() + + for d in need_names: + mac = d.get('mac_address') + if not mac: + raise ValueError("No mac_address or name entry for %s" % d) + if mac not in known_macs: + raise ValueError("Unable to find a system nic for %s" % d) + d['name'] = known_macs[mac] + + for service in services: + cfg = service + cfg.update({'type': 'nameserver'}) + config.append(cfg) + + return {'version': 1, 'config': config} + + def convert_vendordata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). + """data: a loaded json *object* (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index faba5887..fb53ec1d 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -1,25 +1,25 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2015 Canonical Ltd.
-# Copyright (C) 2015 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-class BootProtoEnum:
- """Specifies the NIC Boot Settings."""
-
- DHCP = 'dhcp'
- STATIC = 'static'
+# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +class BootProtoEnum(object): + """Specifies the NIC Boot Settings.""" + + DHCP = 'dhcp' + STATIC = 'static' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index aebc12a0..d645c497 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -1,95 +1,95 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2015 Canonical Ltd.
-# Copyright (C) 2015 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-from .nic import Nic
-
-
-class Config:
- """
- Stores the Contents specified in the Customization
- Specification file.
- """
-
- DNS = 'DNS|NAMESERVER|'
- SUFFIX = 'DNS|SUFFIX|'
- PASS = 'PASSWORD|-PASS'
- TIMEZONE = 'DATETIME|TIMEZONE'
- UTC = 'DATETIME|UTC'
- HOSTNAME = 'NETWORK|HOSTNAME'
- DOMAINNAME = 'NETWORK|DOMAINNAME'
-
- def __init__(self, configFile):
- self._configFile = configFile
-
- @property
- def host_name(self):
- """Return the hostname."""
- return self._configFile.get(Config.HOSTNAME, None)
-
- @property
- def domain_name(self):
- """Return the domain name."""
- return self._configFile.get(Config.DOMAINNAME, None)
-
- @property
- def timezone(self):
- """Return the timezone."""
- return self._configFile.get(Config.TIMEZONE, None)
-
- @property
- def utc(self):
- """Retrieves whether to set time to UTC or Local."""
- return self._configFile.get(Config.UTC, None)
-
- @property
- def admin_password(self):
- """Return the root password to be set."""
- return self._configFile.get(Config.PASS, None)
-
- @property
- def name_servers(self):
- """Return the list of DNS servers."""
- res = []
- cnt = self._configFile.get_count_with_prefix(Config.DNS)
- for i in range(1, cnt + 1):
- key = Config.DNS + str(i)
- res.append(self._configFile[key])
-
- return res
-
- @property
- def dns_suffixes(self):
- """Return the list of DNS Suffixes."""
- res = []
- cnt = self._configFile.get_count_with_prefix(Config.SUFFIX)
- for i in range(1, cnt + 1):
- key = Config.SUFFIX + str(i)
- res.append(self._configFile[key])
-
- return res
-
- @property
- def nics(self):
- """Return the list of associated NICs."""
- res = []
- nics = self._configFile['NIC-CONFIG|NICS']
- for nic in nics.split(','):
- res.append(Nic(nic, self._configFile))
-
- return res
+# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from .nic import Nic + + +class Config(object): + """ + Stores the Contents specified in the Customization + Specification file. + """ + + DNS = 'DNS|NAMESERVER|' + SUFFIX = 'DNS|SUFFIX|' + PASS = 'PASSWORD|-PASS' + TIMEZONE = 'DATETIME|TIMEZONE' + UTC = 'DATETIME|UTC' + HOSTNAME = 'NETWORK|HOSTNAME' + DOMAINNAME = 'NETWORK|DOMAINNAME' + + def __init__(self, configFile): + self._configFile = configFile + + @property + def host_name(self): + """Return the hostname.""" + return self._configFile.get(Config.HOSTNAME, None) + + @property + def domain_name(self): + """Return the domain name.""" + return self._configFile.get(Config.DOMAINNAME, None) + + @property + def timezone(self): + """Return the timezone.""" + return self._configFile.get(Config.TIMEZONE, None) + + @property + def utc(self): + """Retrieves whether to set time to UTC or Local.""" + return self._configFile.get(Config.UTC, None) + + @property + def admin_password(self): + """Return the root password to be set.""" + return self._configFile.get(Config.PASS, None) + + @property + def name_servers(self): + """Return the list of DNS servers.""" + res = [] + cnt = self._configFile.get_count_with_prefix(Config.DNS) + for i in range(1, cnt + 1): + key = Config.DNS + str(i) + res.append(self._configFile[key]) + + return res + + @property + def dns_suffixes(self): + """Return the list of DNS Suffixes.""" + res = [] + cnt = self._configFile.get_count_with_prefix(Config.SUFFIX) + for i in range(1, cnt + 1): + key = Config.SUFFIX + str(i) + res.append(self._configFile[key]) + + return res + + @property + def nics(self): + """Return the list of associated NICs.""" + res = [] + nics = self._configFile['NIC-CONFIG|NICS'] + for nic in nics.split(','): + res.append(Nic(nic, self._configFile)) + + return res diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py index 7266b699..b28830f5 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py +++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py @@ -1,25 +1,25 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2015 Canonical Ltd.
-# Copyright (C) 2015 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-from .config_source import ConfigSource
-
-
-class ConfigNamespace(ConfigSource):
- """Specifies the Config Namespace."""
- pass
+# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from .config_source import ConfigSource + + +class ConfigNamespace(ConfigSource): + """Specifies the Config Namespace.""" + pass diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77098a05..511cc918 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,7 +26,7 @@ from cloudinit import util logger = logging.getLogger(__name__) -class NicConfigurator: +class NicConfigurator(object): def __init__(self, nics): """ Initialize the Nic Configurator diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index a367e476..28ef306a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -1,23 +1,23 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2015 Canonical Ltd.
-# Copyright (C) 2015 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-class ConfigSource:
- """Specifies a source for the Config Content."""
- pass
+# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# Copyright (C) 2015 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +class ConfigSource(object): + """Specifies a source for the Config Content.""" + pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 1b04161f..d1546852 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -1,24 +1,24 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2016 Canonical Ltd.
-# Copyright (C) 2016 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-class GuestCustErrorEnum:
- """Specifies different errors of Guest Customization engine"""
-
- GUESTCUST_ERROR_SUCCESS = 0
+# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +class GuestCustErrorEnum(object): + """Specifies different errors of Guest Customization engine""" + + GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py index fc22568f..ce90c898 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -1,27 +1,27 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2016 Canonical Ltd.
-# Copyright (C) 2016 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-class GuestCustEventEnum:
- """Specifies different types of Guest Customization Events"""
-
- GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100
- GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101
- GUESTCUST_EVENT_ENABLE_NICS = 103
- GUESTCUST_EVENT_QUERY_NICS = 104
+# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +class GuestCustEventEnum(object): + """Specifies different types of Guest Customization Events""" + + GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 + GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101 + GUESTCUST_EVENT_ENABLE_NICS = 103 + GUESTCUST_EVENT_QUERY_NICS = 104 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py index f255be5f..422a096d 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -1,25 +1,25 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2016 Canonical Ltd.
-# Copyright (C) 2016 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-class GuestCustStateEnum:
- """Specifies different states of Guest Customization engine"""
-
- GUESTCUST_STATE_RUNNING = 4
- GUESTCUST_STATE_DONE = 5
+# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +class GuestCustStateEnum(object): + """Specifies different states of Guest Customization engine""" + + GUESTCUST_STATE_RUNNING = 4 + GUESTCUST_STATE_DONE = 5 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index d39f0a65..c07c5949 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -1,128 +1,128 @@ -# vi: ts=4 expandtab
-#
-# Copyright (C) 2016 Canonical Ltd.
-# Copyright (C) 2016 VMware Inc.
-#
-# Author: Sankar Tanguturi <stanguturi@vmware.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3, as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import logging
-import os
-import time
-
-from cloudinit import util
-
-from .guestcust_state import GuestCustStateEnum
-from .guestcust_event import GuestCustEventEnum
-
-logger = logging.getLogger(__name__)
-
-
-CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log"
-QUERY_NICS_SUPPORTED = "queryNicsSupported"
-NICS_STATUS_CONNECTED = "connected"
-
-
-# This will send a RPC command to the underlying
-# VMware Virtualization Platform.
-def send_rpc(rpc):
- if not rpc:
- return None
-
- out = ""
- err = "Error sending the RPC command"
-
- try:
- logger.debug("Sending RPC command: %s", rpc)
- (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0])
- # Remove the trailing newline in the output.
- if out:
- out = out.rstrip()
- except Exception as e:
- logger.debug("Failed to send RPC command")
- logger.exception(e)
-
- return (out, err)
-
-
-# This will send the customization status to the
-# underlying VMware Virtualization Platform.
-def set_customization_status(custstate, custerror, errormessage=None):
- message = ""
-
- if errormessage:
- message = CLOUDINIT_LOG_FILE + "@" + errormessage
- else:
- message = CLOUDINIT_LOG_FILE
-
- rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message)
- (out, err) = send_rpc(rpc)
- return (out, err)
-
-
-# This will read the file nics.txt in the specified directory
-# and return the content
-def get_nics_to_enable(dirpath):
- if not dirpath:
- return None
-
- NICS_SIZE = 1024
- nicsfilepath = os.path.join(dirpath, "nics.txt")
- if not os.path.exists(nicsfilepath):
- return None
-
- with open(nicsfilepath, 'r') as fp:
- nics = fp.read(NICS_SIZE)
-
- return nics
-
-
-# This will send a RPC command to the underlying VMware Virtualization platform
-# and enable nics.
-def enable_nics(nics):
- if not nics:
- logger.warning("No Nics found")
- return
-
- enableNicsWaitRetries = 5
- enableNicsWaitCount = 5
- enableNicsWaitSeconds = 1
-
- for attempt in range(0, enableNicsWaitRetries):
- logger.debug("Trying to connect interfaces, attempt %d", attempt)
- (out, err) = set_customization_status(
- GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
- GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS,
- nics)
- if not out:
- time.sleep(enableNicsWaitCount * enableNicsWaitSeconds)
- continue
-
- if out != QUERY_NICS_SUPPORTED:
- logger.warning("NICS connection status query is not supported")
- return
-
- for count in range(0, enableNicsWaitCount):
- (out, err) = set_customization_status(
- GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
- GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS,
- nics)
- if out and out == NICS_STATUS_CONNECTED:
- logger.info("NICS are connected on %d second", count)
- return
-
- time.sleep(enableNicsWaitSeconds)
-
- logger.warning("Can't connect network interfaces after %d attempts",
- enableNicsWaitRetries)
+# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi <stanguturi@vmware.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import logging +import os +import time + +from cloudinit import util + +from .guestcust_event import GuestCustEventEnum +from .guestcust_state import GuestCustStateEnum + +logger = logging.getLogger(__name__) + + +CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" +QUERY_NICS_SUPPORTED = "queryNicsSupported" +NICS_STATUS_CONNECTED = "connected" + + +# This will send a RPC command to the underlying +# VMware Virtualization Platform. +def send_rpc(rpc): + if not rpc: + return None + + out = "" + err = "Error sending the RPC command" + + try: + logger.debug("Sending RPC command: %s", rpc) + (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + # Remove the trailing newline in the output. + if out: + out = out.rstrip() + except Exception as e: + logger.debug("Failed to send RPC command") + logger.exception(e) + + return (out, err) + + +# This will send the customization status to the +# underlying VMware Virtualization Platform. +def set_customization_status(custstate, custerror, errormessage=None): + message = "" + + if errormessage: + message = CLOUDINIT_LOG_FILE + "@" + errormessage + else: + message = CLOUDINIT_LOG_FILE + + rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) + (out, err) = send_rpc(rpc) + return (out, err) + + +# This will read the file nics.txt in the specified directory +# and return the content +def get_nics_to_enable(dirpath): + if not dirpath: + return None + + NICS_SIZE = 1024 + nicsfilepath = os.path.join(dirpath, "nics.txt") + if not os.path.exists(nicsfilepath): + return None + + with open(nicsfilepath, 'r') as fp: + nics = fp.read(NICS_SIZE) + + return nics + + +# This will send a RPC command to the underlying VMware Virtualization platform +# and enable nics. +def enable_nics(nics): + if not nics: + logger.warning("No Nics found") + return + + enableNicsWaitRetries = 5 + enableNicsWaitCount = 5 + enableNicsWaitSeconds = 1 + + for attempt in range(0, enableNicsWaitRetries): + logger.debug("Trying to connect interfaces, attempt %d", attempt) + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, + nics) + if not out: + time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) + continue + + if out != QUERY_NICS_SUPPORTED: + logger.warning("NICS connection status query is not supported") + return + + for count in range(0, enableNicsWaitCount): + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, + nics) + if out and out == NICS_STATUS_CONNECTED: + logger.info("NICS are connected on %d second", count) + return + + time.sleep(enableNicsWaitSeconds) + + logger.warning("Can't connect network interfaces after %d attempts", + enableNicsWaitRetries) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 33f88726..873ddc3b 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -class Ipv4ModeEnum: +class Ipv4ModeEnum(object): """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py index 030ba311..3c892db0 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic_base.py +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -18,7 +18,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -class NicBase: +class NicBase(object): """ Define what are expected of each nic. The following properties should be provided in an implementation class. @@ -93,7 +93,7 @@ class NicBase: raise NotImplementedError('Check constraints on properties') -class StaticIpv4Base: +class StaticIpv4Base(object): """ Define what are expected of a static IPv4 setting The following properties should be provided in an implementation class. @@ -124,7 +124,7 @@ class StaticIpv4Base: raise NotImplementedError('Ipv4 GATEWAY') -class StaticIpv6Base: +class StaticIpv6Base(object): """Define what are expected of a static IPv6 setting The following properties should be provided in an implementation class. """ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..47deac6e 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,14 +44,16 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net +from cloudinit.net import cmdline +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util -from cloudinit.reporting import events LOG = logging.getLogger(__name__) NULL_DATA_SOURCE = None +NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID" class Init(object): @@ -67,6 +69,7 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE self.ds_restored = False + self._previous_iid = None if reporter is None: reporter = events.ReportEventStack( @@ -213,6 +216,31 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) + def _restore_from_checked_cache(self, existing): + if existing not in ("check", "trust"): + raise ValueError("Unexpected value for existing: %s" % existing) + + ds = self._restore_from_cache() + if not ds: + return (None, "no cache found") + + run_iid_fn = self.paths.get_runpath('instance_id') + if os.path.exists(run_iid_fn): + run_iid = util.load_file(run_iid_fn).strip() + else: + run_iid = None + + if run_iid == ds.get_instance_id(): + return (ds, "restored from cache with run check: %s" % ds) + elif existing == "trust": + return (ds, "restored from cache: %s" % ds) + else: + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): + return (ds, "restored from checked cache: %s" % ds) + else: + return (None, "cache invalid in datasource: %s" % ds) + def _get_data_source(self, existing): if self.datasource is not NULL_DATA_SOURCE: return self.datasource @@ -221,19 +249,9 @@ class Init(object): name="check-cache", description="attempting to read from cache [%s]" % existing, parent=self.reporter) as myrep: - ds = self._restore_from_cache() - if ds and existing == "trust": - myrep.description = "restored from cache: %s" % ds - elif ds and existing == "check": - if (hasattr(ds, 'check_instance_id') and - ds.check_instance_id(self.cfg)): - myrep.description = "restored from checked cache: %s" % ds - else: - myrep.description = "cache invalid in datasource: %s" % ds - ds = None - else: - myrep.description = "no cache found" + ds, desc = self._restore_from_checked_cache(existing) + myrep.description = desc self.ds_restored = bool(ds) LOG.debug(myrep.description) @@ -301,23 +319,41 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = None iid_fn = os.path.join(dp, 'instance-id') - try: - previous_iid = util.load_file(iid_fn).strip() - except Exception: - pass - if not previous_iid: - previous_iid = iid + + previous_iid = self.previous_iid() util.write_file(iid_fn, "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) + + self._write_to_cache() # Ensure needed components are regenerated # after change of instance which may cause # change of configuration self._reset() return iid + def previous_iid(self): + if self._previous_iid is not None: + return self._previous_iid + + dp = self.paths.get_cpath('data') + iid_fn = os.path.join(dp, 'instance-id') + try: + self._previous_iid = util.load_file(iid_fn).strip() + except Exception: + self._previous_iid = NO_PREVIOUS_INSTANCE_ID + + LOG.debug("previous iid found to be %s", self._previous_iid) + return self._previous_iid + + def is_new_instance(self): + previous = self.previous_iid() + ret = (previous == NO_PREVIOUS_INSTANCE_ID or + previous != self.datasource.get_instance_id()) + return ret + def fetch(self, existing="check"): return self._get_data_source(existing=existing) @@ -332,8 +368,6 @@ class Init(object): reporter=self.reporter) def update(self): - if not self._write_to_cache(): - return self._store_userdata() self._store_vendordata() @@ -483,7 +517,7 @@ class Init(object): c_handlers.initialized.remove(mod) try: handlers.call_end(mod, data, frequency) - except: + except Exception: util.logexc(LOG, "Failed to finalize handler: %s", mod) try: @@ -579,13 +613,13 @@ class Init(object): if os.path.exists(disable_file): return (None, disable_file) - cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config()) + cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config()) dscfg = ('ds', None) if self.datasource and hasattr(self.datasource, 'network_config'): dscfg = ('ds', self.datasource.network_config) sys_cfg = ('system_cfg', self.cfg.get('network')) - for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg): + for loc, ncfg in (cmdline_cfg, sys_cfg, dscfg): if net.is_disabled_cfg(ncfg): LOG.debug("network config disabled by %s", loc) return (None, loc) @@ -593,15 +627,27 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self): + def apply_network_config(self, bring_up): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return - LOG.info("Applying network configuration from %s: %s", src, netcfg) try: - return self.distro.apply_network_config(netcfg) + LOG.debug("applying net config names for %s" % netcfg) + self.distro.apply_network_config_names(netcfg) + except Exception as e: + LOG.warn("Failed to rename devices: %s", e) + + if (self.datasource is not NULL_DATA_SOURCE and + not self.is_new_instance()): + LOG.debug("not a new instance. network config is not applied.") + return + + LOG.info("Applying network configuration from %s bringup=%s: %s", + src, bring_up, netcfg) + try: + return self.distro.apply_network_config(netcfg, bring_up=bring_up) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % @@ -794,16 +840,16 @@ class Modules(object): def fetch_base_config(): base_cfgs = [] default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) # Anything in your conf.d location?? # or the 'default' cloud.cfg location??? base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # Kernel/cmdline parameters override system config + kern_contents = util.read_cc_from_cmdline() + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + # And finally the default gets to play if default_cfg: base_cfgs.append(default_cfg) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index a9231482..41ef27e3 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -3,10 +3,12 @@ # Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2016 Amazon.com, Inc. or its affiliates. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> # Author: Joshua Harlow <harlowja@yahoo-inc.com> +# Author: Andrew Jorgensen <ajorgens@amazon.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -102,12 +104,11 @@ def detect_template(text): rest = '' type_match = TYPE_MATCHER.match(ident) if not type_match: - if not CHEETAH_AVAILABLE: - LOG.warn("Cheetah not available as the default renderer for" - " unknown template, reverting to the basic renderer.") - return ('basic', basic_render, text) - else: + if CHEETAH_AVAILABLE: + LOG.debug("Using Cheetah as the renderer for unknown template.") return ('cheetah', cheetah_render, text) + else: + return ('basic', basic_render, text) else: template_type = type_match.group(1).lower().strip() if template_type not in ('jinja', 'cheetah', 'basic'): @@ -142,6 +143,11 @@ def render_to_file(fn, outfn, params, mode=0o644): util.write_file(outfn, contents, mode=mode) +def render_string_to_file(content, outfn, params, mode=0o644): + contents = render_string(content, params) + util.write_file(outfn, contents, mode=mode) + + def render_string(content, params): if not params: params = {} diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 936f7da5..c05e9d90 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -28,8 +28,9 @@ import time from email.utils import parsedate from functools import partial -from requests import exceptions + import oauthlib.oauth1 as oauth1 +from requests import exceptions from six.moves.urllib.parse import ( urlparse, urlunparse, @@ -61,7 +62,7 @@ try: SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): CONFIG_ENABLED = True -except: +except ImportError: pass diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f7c5787c..393bf0bb 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -178,7 +178,7 @@ class UserDataProcessor(object): payload = util.load_yaml(msg.get_payload(decode=True)) if payload: payload_idx = payload.get('launch-index') - except: + except Exception: pass # Header overrides contents, for now (?) or the other way around? if header_idx is not None: @@ -334,19 +334,23 @@ def is_skippable(part): # Coverts a raw string into a mime message -def convert_string(raw_data, headers=None): +def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE): if not raw_data: raw_data = '' - if not headers: - headers = {} - data = util.decode_binary(util.decomp_gzip(raw_data)) - if "mime-version:" in data[0:4096].lower(): - msg = util.message_from_string(data) - for (key, val) in headers.items(): - _replace_header(msg, key, val) - else: - mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE) - maintype, subtype = mtype.split("/", 1) - msg = MIMEBase(maintype, subtype, *headers) + + def create_binmsg(data, content_type): + maintype, subtype = content_type.split("/", 1) + msg = MIMEBase(maintype, subtype) msg.set_payload(data) + return msg + + try: + data = util.decode_binary(util.decomp_gzip(raw_data)) + if "mime-version:" in data[0:4096].lower(): + msg = util.message_from_string(data) + else: + msg = create_binmsg(data, content_type) + except UnicodeDecodeError: + msg = create_binmsg(raw_data, content_type) + return msg diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..e5dd61a0 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -171,7 +171,8 @@ class ProcessExecutionError(IOError): def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None, reason=None): + description=None, reason=None, + errno=None): if not cmd: self.cmd = '-' else: @@ -202,6 +203,7 @@ class ProcessExecutionError(IOError): else: self.reason = '-' + self.errno = errno message = self.MESSAGE_TMPL % { 'description': self.description, 'cmd': self.cmd, @@ -288,7 +290,7 @@ def fork_cb(child_cb, *args, **kwargs): try: child_cb(*args, **kwargs) os._exit(0) - except: + except Exception: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) @@ -336,6 +338,16 @@ def rand_str(strlen=32, select_from=None): return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey + + def read_conf(fname): try: return load_yaml(load_file(fname), default={}) @@ -472,7 +484,7 @@ def is_ipv4(instr): try: toks = [x for x in toks if int(x) < 256 and int(x) >= 0] - except: + except Exception: return False return len(toks) == 4 @@ -1147,7 +1159,14 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options # See man blkid for why 2 is added - (out, _err) = subp(cmd, rcs=[0, 2]) + try: + (out, _err) = subp(cmd, rcs=[0, 2]) + except ProcessExecutionError as e: + if e.errno == errno.ENOENT: + # blkid not found... + out = "" + else: + raise entries = [] for line in out.splitlines(): line = line.strip() @@ -1210,7 +1229,7 @@ def get_cmdline(): else: try: cmdline = load_file("/proc/cmdline").strip() - except: + except Exception: cmdline = "" PROC_CMDLINE = cmdline @@ -1380,7 +1399,7 @@ def read_write_cmdline_url(target_fn): if not os.path.exists(target_fn): try: (key, url, content) = get_cmdline_url() - except: + except Exception: logexc(LOG, "Failed fetching command line url") return try: @@ -1391,7 +1410,7 @@ def read_write_cmdline_url(target_fn): elif key and not content: LOG.debug(("Command line key %s with url" " %s had no contents"), key, url) - except: + except Exception: logexc(LOG, "Failed writing url content to %s", target_fn) @@ -1449,7 +1468,7 @@ def mounts(): mp = m.group(2) fstype = m.group(3) opts = m.group(4) - except: + except Exception: continue # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. @@ -1575,7 +1594,7 @@ def copy(src, dest): def time_rfc2822(): try: ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: + except Exception: ts = "??" return ts @@ -1601,7 +1620,7 @@ def uptime(): bootup = buf.value uptime_str = now - bootup - except: + except Exception: logexc(LOG, "Unable to read uptime using method: %s" % method) return uptime_str @@ -1696,7 +1715,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, sp = subprocess.Popen(args, **kws) (out, err) = sp.communicate(data) except OSError as e: - raise ProcessExecutionError(cmd=args, reason=e) + raise ProcessExecutionError(cmd=args, reason=e, + errno=e.errno) rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, @@ -2055,7 +2075,7 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): tmsg += " (N/A)" try: logfunc(msg + tmsg) - except: + except Exception: pass return ret @@ -2190,7 +2210,7 @@ def _call_dmidecode(key, dmidecode_path): return "" return result except (IOError, OSError) as _err: - LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message) + LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err) return None diff --git a/doc/examples/cloud-config-boot-cmds.txt b/doc/examples/cloud-config-boot-cmds.txt index b281d327..3e59755d 100644 --- a/doc/examples/cloud-config-boot-cmds.txt +++ b/doc/examples/cloud-config-boot-cmds.txt @@ -7,9 +7,9 @@ # bootcmd should really only be used for things that could not be # done later in the boot process. bootcmd is very much like # boothook, but possibly with more friendly. -# * bootcmd will run on every boot -# * the INSTANCE_ID variable will be set to the current instance id. -# * you can use 'cloud-init-boot-per' command to help only run once +# - bootcmd will run on every boot +# - the INSTANCE_ID variable will be set to the current instance id. +# - you can use 'cloud-init-boot-per' command to help only run once bootcmd: - echo 192.168.1.130 us.archive.ubuntu.com > /etc/hosts - [ cloud-init-per, once, mymkfs, mkfs, /dev/vdb ] diff --git a/doc/examples/cloud-config-run-cmds.txt b/doc/examples/cloud-config-run-cmds.txt index 61b3bd63..3bb06864 100644 --- a/doc/examples/cloud-config-run-cmds.txt +++ b/doc/examples/cloud-config-run-cmds.txt @@ -5,12 +5,13 @@ # runcmd contains a list of either lists or a string # each item will be executed in order at rc.local like level with # output to the console +# - runcmd only runs during the first boot # - if the item is a list, the items will be properly executed as if # passed to execve(3) (with the first arg as the command). # - if the item is a string, it will be simply written to the file and # will be interpreted by 'sh' # -# Note, that the list has to be proper yaml, so you have to escape +# Note, that the list has to be proper yaml, so you have to quote # any characters yaml would eat (':' can be problematic) runcmd: - [ ls, -l, / ] diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1236796c..3cc9c055 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -72,14 +72,87 @@ apt_pipelining: False # then apt_mirror above will have no effect apt_preserve_sources_list: true +# Provide a custom template for rendering sources.list +# Default: a default template for Ubuntu/Debain will be used as packaged in +# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl +# Debian: /etc/cloud/templates/sources.list.debian.tmpl +# Others: n/a +# This will follow the normal mirror/codename replacement rules before +# being written to disk. +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + + # could drop some of the usually used entries + + # could refer to other mirrors + deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse + + # or even more uncommon examples like local or NFS mounted repos, + # eventually whatever is compatible with sources.list syntax + deb file:/home/apt/debian unstable main contrib non-free + # 'source' entries in apt-sources that match this python regex # expression will be passed to add-apt-repository add_apt_repo_match: '^[\w-]+:\w' +# 'apt_sources' is a dictionary +# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if +# it doesn't start with a '/'. +# There are certain cases - where no content is written into a source.list file +# where the filename will be ignored - yet it can still be used as index for +# merging. +# The value it maps to is a dictionary with the following optional entries: +# source: a sources.list entry (some variable replacements apply) +# keyid: providing a key to import via shortid or fingerprint +# key: providing a raw PGP key +# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com +# filename: for compatibility with the older format (now the key to this +# dictionary is the filename). If specified this overwrites the +# filename given as key. + +# the new "filename: {specification-dictionary}, filename2: ..." format allows +# better merging between multiple input files than a list like: +# cloud-config1 +# sources: +# s1: {'key': 'key1', 'source': 'source1'} +# cloud-config2 +# sources: +# s2: {'key': 'key2'} +# s1: {filename: 'foo'} +# this would be merged to +#sources: +# s1: +# filename: foo +# key: key1 +# source: source1 +# s2: +# key: key2 +# Be aware that this style of merging is not the default (for backward +# compatibility reasons). You should specify the following merge_how to get +# this more complete and modern merging behaviour: +# merge_how: "list()+dict()+str()" +# This would then also be equivalent to the config merging used in curtin +# (https://launchpad.net/curtin). + +# for more details see below in the various examples + apt_sources: - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + byobu-ppa.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server - filename: byobu-ppa.list + # adding a source.list line, importing a gpg key for a given key id and + # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list # PPA shortcut: # * Setup correct apt sources.list line @@ -87,7 +160,9 @@ apt_sources: # # See https://help.launchpad.net/Packaging/PPA for more information # this requires 'add-apt-repository' - - source: "ppa:smoser/ppa" # Quote the string + # due to that the filename key is ignored in this case + ignored1: + source: "ppa:smoser/ppa" # Quote the string # Custom apt repository: # * all that is required is 'source' @@ -95,29 +170,60 @@ apt_sources: # * [optional] Import the apt signing key from the keyserver # * Defaults: # + keyserver: keyserver.ubuntu.com - # + filename: cloud_config_sources.list # # See sources.list man page for more information about the format - - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted + my-repo.list: + source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted # sources can use $MIRROR and $RELEASE and they will be replaced # with the local mirror for this cloud, and the running release # the entry below would be possibly turned into: - # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse - - source: deb $MIRROR $RELEASE multiverse + # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse + my-repo.list: + source: deb $MIRROR $RELEASE multiverse # this would have the same end effect as 'ppa:byobu/ppa' - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + my-repo.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server filename: byobu-ppa.list + # this would only import the key without adding a ppa or other source spec + # since this doesn't generate a source.list file the filename key is ignored + ignored2: + keyid: F430BBA5 # GPG key ID published on a key server + + # In general keyid's can also be specified via their long fingerprints + # since this doesn't generate a source.list file the filename key is ignored + ignored3: + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block - # * Providing the PBG key here is the most robust method for + # * Providing the PGP key here is the most robust method for # specifying a key, as it removes dependency on a remote key server + my-repo.list: + source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- - - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main + # Custom gpg key: + # * As with keyid, a key may also be specified without a related source. + # * all other facts mentioned above still apply + # since this doesn't generate a source.list file the filename key is ignored + ignored4: key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.0.10 @@ -132,6 +238,7 @@ apt_sources: =Y2oI -----END PGP PUBLIC KEY BLOCK----- + ## apt config via system_info: # under the 'system_info', you can further customize cloud-init's interaction # with apt. diff --git a/doc/merging.rst b/doc/merging.rst index d4d5cd05..afe1a6dd 100644 --- a/doc/merging.rst +++ b/doc/merging.rst @@ -1,15 +1,16 @@ Overview -------- -This was done because it has been a common feature request that there be a -way to specify how cloud-config yaml "dictionaries" are merged together when -there are multiple yamls to merge together (say when performing an #include). +This was implemented because it has been a common feature request that there be +a way to specify how cloud-config yaml "dictionaries" provided as user-data are +merged together when there are multiple yamls to merge together (say when +performing an #include). Since previously the merging algorithm was very simple and would only overwrite and not append lists, or strings, and so on it was decided to create a new and improved way to merge dictionaries (and there contained objects) together in a -way that is customizable, thus allowing for users who provide cloud-config data -to determine exactly how there objects will be merged. +way that is customizable, thus allowing for users who provide cloud-config +user-data to determine exactly how there objects will be merged. For example. @@ -19,13 +20,13 @@ For example. run_cmd: - bash1 - bash2 - + #cloud-config (2) run_cmd: - bash3 - bash4 -The previous way of merging the following 2 objects would result in a final +The previous way of merging the following 2 objects would result in a final cloud-config object that contains the following. .. code-block:: yaml @@ -56,7 +57,7 @@ Customizability Since the above merging algorithm may not always be the desired merging algorithm (like how the previous merging algorithm was not always the preferred one) the concept of customizing how merging can be done was introduced through -a new concept call 'merge classes'. +a new concept call 'merge classes'. A merge class is a class defintion which provides functions that can be used to merge a given type with another given type. @@ -69,7 +70,7 @@ An example of one of these merging classes is the following: def __init__(self, merger, opts): self._merger = merger self._overwrite = 'overwrite' in opts - + # This merging algorithm will attempt to merge with # another dictionary, on encountering any other type of object # it will not merge with said object, but will instead return @@ -129,12 +130,12 @@ for your own usage. definition are the following (in order), 'merge_how', 'merge_type'. String format -******** +************* The string format that is expected is the following. :: - + classname1(option1,option2)+classname2(option3,option4).... The class name there will be connected to class names used when looking for the @@ -144,11 +145,11 @@ on construction of that class. For example, the default string that is used when none is provided is the following: :: - + list()+dict()+str() Dictionary format -******** +***************** In cases where a dictionary can be used to specify the same information as the string format (ie option #2 of above) it can be used, for example. @@ -171,7 +172,7 @@ for every cloud-config that I provide, what exactly happens? The answer is that when merging, a stack of 'merging classes' is kept, the first one on that stack is the default merging classes, this set of mergers will be used when the first cloud-config is merged with the initial empty -cloud-config dictionary. If the cloud-config that was just merged provided a +cloud-config dictionary. If the cloud-config that was just merged provided a set of merging classes (via the above formats) then those merging classes will be pushed onto the stack. Now if there is a second cloud-config to be merged then the merging classes from the cloud-config before the first will be used (not the @@ -181,8 +182,13 @@ cloud-config dictionary coming after it. Other uses ---------- -The default merging algorithm for merging 'conf.d' yaml files (which form a initial -yaml config for cloud-init) was also changed to use this mechanism so its full +In addition to being used for merging user-data sections, the default merging +algorithm for merging 'conf.d' yaml files (which form an initial yaml config +for cloud-init) was also changed to use this mechanism so its full benefits (and customization) can also be used there as well. Other places that -used the previous merging are also similar now extensible (metadata merging for -example). +used the previous merging are also, similarly, now extensible (metadata +merging, for example). + +Note, however, that merge algorithms are not used *across* types of +configuration. As was the case before merging was implemented, +user-data will overwrite conf.d configuration without merging. diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst index 8a03f3c7..2bd87b16 100644 --- a/doc/rtd/topics/merging.rst +++ b/doc/rtd/topics/merging.rst @@ -1,5 +1,5 @@ -========= -Merging -========= +========================== +Merging User-Data Sections +========================== .. include:: ../../merging.rst diff --git a/packages/bddeb b/packages/bddeb index c141b1ab..3c77ce1d 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -40,6 +40,9 @@ STD_NAMED_PACKAGES = [ 'mock', 'nose', 'setuptools', + 'flake8', + 'hacking', + 'unittest2', ] NONSTD_NAMED_PACKAGES = { 'argparse': ('python-argparse', None), @@ -146,11 +149,17 @@ def main(): parser.add_argument("--sign", default=False, action='store_true', help="sign result. do not pass -us -uc to debuild") + parser.add_argument("--signuser", default=False, action='store', + help="user to sign, see man dpkg-genchanges") + args = parser.parse_args() if not args.sign: args.debuild_args.extend(['-us', '-uc']) + if args.signuser: + args.debuild_args.extend(['-e%s' % args.signuser]) + os.environ['INIT_SYSTEM'] = args.init_system capture = True diff --git a/requirements.txt b/requirements.txt index 19c88857..cc1dc05f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,12 @@ PrettyTable oauthlib # This one is currently used only by the CloudSigma and SmartOS datasources. -# If these datasources are removed, this is no longer needed -pyserial +# If these datasources are removed, this is no longer needed. +# +# This will not work in py2.6 so it is only optionally installed on +# python 2.7 and later. +# +# pyserial # This is only needed for places where we need to support configs in a manner # that the built-in config parser is not sufficent (ie @@ -57,15 +57,15 @@ def tiny_p(cmd, capture=True): def pkg_config_read(library, var): fallbacks = { - 'systemd': { - 'systemdsystemunitdir': '/lib/systemd/system', - 'systemdsystemgeneratordir': '/lib/systemd/system-generators', - } + 'systemd': { + 'systemdsystemunitdir': '/lib/systemd/system', + 'systemdsystemgeneratordir': '/lib/systemd/system-generators', + } } cmd = ['pkg-config', '--variable=%s' % var, library] try: (path, err) = tiny_p(cmd) - except: + except Exception: return fallbacks[library][var] return str(path).strip() @@ -184,7 +184,6 @@ else: (USR + '/share/doc/cloud-init/examples/seed', [f for f in glob('doc/examples/seed/*') if is_f(f)]), (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]), - (LIB + '/udev', ['udev/cloud-init-wait']), ] # Use a subclass for install that handles # adding on the right init system configuration files @@ -197,7 +196,6 @@ requirements = read_requires() if sys.version_info < (3,): requirements.append('cheetah') - setuptools.setup( name='cloud-init', version=get_version(), @@ -206,10 +204,14 @@ setuptools.setup( author_email='scott.moser@canonical.com', url='http://launchpad.net/cloud-init/', packages=setuptools.find_packages(exclude=['tests']), - scripts=['bin/cloud-init', - 'tools/cloud-init-per'], + scripts=['tools/cloud-init-per'], license='GPLv3', data_files=data_files, install_requires=requirements, cmdclass=cmdclass, - ) + entry_points={ + 'console_scripts': [ + 'cloud-init = cloudinit.cmd.main:main' + ], + } +) diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index ae286d58..2d319695 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -107,9 +107,6 @@ main() { "ln $CLOUD_SYSTEM_TARGET $link_path" fi fi - # this touches /run/cloud-init/enabled, which is read by - # udev/cloud-init-wait. If not present, it will exit quickly. - touch "$LOG_D/$ENABLE" elif [ "$result" = "$DISABLE" ]; then if [ -f "$link_path" ]; then if rm -f "$link_path"; then diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..6bf38940 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,18 @@ +# Needed generally in tests httpretty>=0.7.1 mock nose -pep8==1.5.7 -pyflakes +unittest2 + +# Only needed if you want to know the test times +# nose-timer + +# Only really needed on older versions of python contextlib2 setuptools + +# Used for syle checking +pep8==1.7.0 +pyflakes==1.1.0 +flake8==2.5.4 +hacking==0.10.2 diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 7f4b8784..8d46a8bf 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -2,18 +2,16 @@ from __future__ import print_function import functools import os -import sys import shutil +import sys import tempfile import unittest +import mock import six +import unittest2 try: - from unittest import mock -except ImportError: - import mock -try: from contextlib import ExitStack except ImportError: from contextlib2 import ExitStack @@ -21,6 +19,9 @@ except ImportError: from cloudinit import helpers as ch from cloudinit import util +# Used for skipping tests +SkipTest = unittest2.SkipTest + # Used for detecting different python versions PY2 = False PY26 = False @@ -44,78 +45,6 @@ else: if _PY_MINOR == 4 and _PY_MICRO < 3: FIX_HTTPRETTY = True -if PY26: - # For now add these on, taken from python 2.7 + slightly adjusted. Drop - # all this once Python 2.6 is dropped as a minimum requirement. - class TestCase(unittest.TestCase): - def setUp(self): - super(TestCase, self).setUp() - self.__all_cleanups = ExitStack() - - def tearDown(self): - self.__all_cleanups.close() - unittest.TestCase.tearDown(self) - - def addCleanup(self, function, *args, **kws): - self.__all_cleanups.callback(function, *args, **kws) - - def assertIs(self, expr1, expr2, msg=None): - if expr1 is not expr2: - standardMsg = '%r is not %r' % (expr1, expr2) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIn(self, member, container, msg=None): - if member not in container: - standardMsg = '%r not found in %r' % (member, container) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertNotIn(self, member, container, msg=None): - if member in container: - standardMsg = '%r unexpectedly found in %r' - standardMsg = standardMsg % (member, container) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIsNone(self, value, msg=None): - if value is not None: - standardMsg = '%r is not None' - standardMsg = standardMsg % (value) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertIsInstance(self, obj, cls, msg=None): - """Same as self.assertTrue(isinstance(obj, cls)), with a nicer - default message.""" - if not isinstance(obj, cls): - standardMsg = '%s is not an instance of %r' % (repr(obj), cls) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertDictContainsSubset(self, expected, actual, msg=None): - missing = [] - mismatched = [] - for k, v in expected.items(): - if k not in actual: - missing.append(k) - elif actual[k] != v: - mismatched.append('%r, expected: %r, actual: %r' - % (k, v, actual[k])) - - if len(missing) == 0 and len(mismatched) == 0: - return - - standardMsg = '' - if missing: - standardMsg = 'Missing: %r' % ','.join(m for m in missing) - if mismatched: - if standardMsg: - standardMsg += '; ' - standardMsg += 'Mismatched values: %s' % ','.join(mismatched) - - self.fail(self._formatMessage(msg, standardMsg)) - - -else: - class TestCase(unittest.TestCase): - pass - # Makes the old path start # with new base instead of whatever @@ -151,6 +80,10 @@ def retarget_many_wrapper(new_base, am, old_func): return wrapper +class TestCase(unittest2.TestCase): + pass + + class ResourceUsingTestCase(TestCase): def setUp(self): super(ResourceUsingTestCase, self).setUp() @@ -180,13 +113,12 @@ class ResourceUsingTestCase(TestCase): with open(where, 'r') as fh: return fh.read() - def getCloudPaths(self): + def getCloudPaths(self, ds=None): tmpdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmpdir) - cp = ch.Paths({ - 'cloud_dir': tmpdir, - 'templates_dir': self.resourceLocation(), - }) + cp = ch.Paths({'cloud_dir': tmpdir, + 'templates_dir': self.resourceLocation()}, + ds=ds) return cp diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 153f1658..0154784a 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -1,16 +1,6 @@ import os import shutil import tempfile -import unittest - -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack from cloudinit import handlers from cloudinit import helpers @@ -18,7 +8,7 @@ from cloudinit import settings from cloudinit import url_helper from cloudinit import util -from .helpers import TestCase +from .helpers import TestCase, ExitStack, mock class FakeModule(handlers.Handler): @@ -99,9 +89,10 @@ class TestWalkerHandleHandler(TestCase): self.assertEqual(self.data['handlercount'], 0) -class TestHandlerHandlePart(unittest.TestCase): +class TestHandlerHandlePart(TestCase): def setUp(self): + super(TestHandlerHandlePart, self).setUp() self.data = "fake data" self.ctype = "fake ctype" self.filename = "fake filename" @@ -177,7 +168,7 @@ class TestHandlerHandlePart(unittest.TestCase): self.data, self.ctype, self.filename, self.payload) -class TestCmdlineUrl(unittest.TestCase): +class TestCmdlineUrl(TestCase): def test_invalid_content(self): url = "http://example.com/foo" key = "mykey" diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index ad32d0b2..dea908d7 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -40,7 +40,7 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): 'test.conf', 'blah', freq) h.handle_part('', handlers.CONTENT_END, None, None, None) - self.assertEquals(0, len(os.listdir(up_root))) + self.assertEqual(0, len(os.listdir(up_root))) def test_upstart_frequency_single(self): # files should be written out when frequency is ! per-instance @@ -67,7 +67,7 @@ class TestBuiltins(test_helpers.FilesystemMockingTestCase): h.handle_part('', handlers.CONTENT_END, None, None, None) - self.assertEquals(len(os.listdir('/etc/upstart')), 1) + self.assertEqual(len(os.listdir('/etc/upstart')), 1) mockobj.assert_called_once_with( ['initctl', 'reload-configuration'], capture=False) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index ed863399..5fa252f7 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -1,17 +1,10 @@ -import imp -import os -import sys import six from . import helpers as test_helpers -try: - from unittest import mock -except ImportError: - import mock +from cloudinit.cmd import main as cli - -BIN_CLOUDINIT = "bin/cloud-init" +mock = test_helpers.mock class TestCLI(test_helpers.FilesystemMockingTestCase): @@ -20,35 +13,22 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): super(TestCLI, self).setUp() self.stderr = six.StringIO() self.patchStdoutAndStderr(stderr=self.stderr) - self.sys_exit = mock.MagicMock() - self.patched_funcs.enter_context( - mock.patch.object(sys, 'exit', self.sys_exit)) - - def _call_main(self): - self.patched_funcs.enter_context( - mock.patch.object(sys, 'argv', ['cloud-init'])) - cli = imp.load_module( - 'cli', open(BIN_CLOUDINIT), '', ('', 'r', imp.PY_SOURCE)) + + def _call_main(self, sysv_args=None): + if not sysv_args: + sysv_args = ['cloud-init'] try: - return cli.main() - except: - pass + return cli.main(sysv_args=sysv_args) + except SystemExit as e: + return e.code - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") def test_no_arguments_shows_usage(self): - self._call_main() - self.assertIn('usage: cloud-init', self.stderr.getvalue()) - - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") - def test_no_arguments_exits_2(self): exit_code = self._call_main() - if self.sys_exit.call_count: - self.assertEqual(mock.call(2), self.sys_exit.call_args) - else: - self.assertEqual(2, exit_code) + self.assertIn('usage: cloud-init', self.stderr.getvalue()) + self.assertEqual(2, exit_code) - @test_helpers.skipIf(not os.path.isfile(BIN_CLOUDINIT), "no bin/cloudinit") def test_no_arguments_shows_error_message(self): - self._call_main() + exit_code = self._call_main() self.assertIn('cloud-init: error: too few arguments', self.stderr.getvalue()) + self.assertEqual(2, exit_code) diff --git a/tests/unittests/test_cs_util.py b/tests/unittests/test_cs_util.py index d7273035..56c9ce9e 100644 --- a/tests/unittests/test_cs_util.py +++ b/tests/unittests/test_cs_util.py @@ -1,21 +1,9 @@ from __future__ import print_function -import sys -import unittest +from . import helpers as test_helpers from cloudinit.cs_utils import Cepko -try: - skip = unittest.skip -except AttributeError: - # Python 2.6. Doesn't have to be high fidelity. - def skip(reason): - def decorator(func): - def wrapper(*args, **kws): - print(reason, file=sys.stderr) - return wrapper - return decorator - SERVER_CONTEXT = { "cpu": 1000, @@ -43,18 +31,9 @@ class CepkoMock(Cepko): # 2015-01-22 BAW: This test is completely useless because it only ever tests # the CepkoMock object. Even in its original form, I don't think it ever # touched the underlying Cepko class methods. -@skip('This test is completely useless') -class CepkoResultTests(unittest.TestCase): +class CepkoResultTests(test_helpers.TestCase): def setUp(self): - pass - # self.mocked = self.mocker.replace("cloudinit.cs_utils.Cepko", - # spec=CepkoMock, - # count=False, - # passthrough=False) - # self.mocked() - # self.mocker.result(CepkoMock()) - # self.mocker.replay() - # self.c = Cepko() + raise test_helpers.SkipTest('This test is completely useless') def test_getitem(self): result = self.c.all() diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 9c1ec1d4..13db8a4c 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -106,9 +106,9 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) - self.assertEquals(2, len(cc)) - self.assertEquals('qux', cc['baz']) - self.assertEquals('qux2', cc['bar']) + self.assertEqual(2, len(cc)) + self.assertEqual('qux', cc['baz']) + self.assertEqual('qux2', cc['bar']) def test_simple_jsonp_vendor_and_user(self): # test that user-data wins over vendor @@ -145,9 +145,9 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) - self.assertEquals('qux', cfg['baz']) - self.assertEquals('qux2', cfg['bar']) - self.assertEquals('quxC', cfg['foo']) + self.assertEqual('qux', cfg['baz']) + self.assertEqual('qux2', cfg['bar']) + self.assertEqual('quxC', cfg['foo']) def test_simple_jsonp_no_vendor_consumed(self): # make sure that vendor data is not consumed @@ -184,8 +184,8 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): mods = stages.Modules(initer) (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg - self.assertEquals('qux', cfg['baz']) - self.assertEquals('qux2', cfg['bar']) + self.assertEqual('qux', cfg['baz']) + self.assertEqual('qux2', cfg['bar']) self.assertNotIn('foo', cfg) def test_mixed_cloud_config(self): @@ -222,8 +222,8 @@ c: d ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) cc = util.load_yaml(cc_contents) - self.assertEquals(1, len(cc)) - self.assertEquals('c', cc['a']) + self.assertEqual(1, len(cc)) + self.assertEqual('c', cc['a']) def test_vendor_user_yaml_cloud_config(self): vendor_blob = ''' @@ -263,8 +263,8 @@ run: (_which_ran, _failures) = mods.run_section('cloud_init_modules') cfg = mods.cfg self.assertIn('vendor_data', cfg) - self.assertEquals('c', cfg['a']) - self.assertEquals('user', cfg['name']) + self.assertEqual('c', cfg['a']) + self.assertEqual('user', cfg['name']) self.assertNotIn('x', cfg['run']) self.assertNotIn('y', cfg['run']) self.assertIn('z', cfg['run']) @@ -358,10 +358,10 @@ p: 1 None) contents = util.load_file(paths.get_ipath('cloud_config')) contents = util.load_yaml(contents) - self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff']) - self.assertEquals(contents['a'], 'be') - self.assertEquals(contents['e'], [1, 2, 3]) - self.assertEquals(contents['p'], 1) + self.assertEqual(contents['run'], ['b', 'c', 'stuff', 'morestuff']) + self.assertEqual(contents['a'], 'be') + self.assertEqual(contents['e'], [1, 2, 3]) + self.assertEqual(contents['p'], 1) def test_unhandled_type_warning(self): """Raw text without magic is ignored but shows warning.""" @@ -411,10 +411,10 @@ c: 4 contents = util.load_file(ci.paths.get_ipath("cloud_config")) contents = util.load_yaml(contents) self.assertTrue(isinstance(contents, dict)) - self.assertEquals(3, len(contents)) - self.assertEquals(2, contents['a']) - self.assertEquals(3, contents['b']) - self.assertEquals(4, contents['c']) + self.assertEqual(3, len(contents)) + self.assertEqual(2, contents['a']) + self.assertEqual(3, contents['b']) + self.assertEqual(4, contents['c']) def test_mime_text_plain(self): """Mime message of type text/plain is ignored but shows warning.""" @@ -449,8 +449,7 @@ c: 4 mockobj.assert_has_calls([ mock.call(outpath, script, 0o700), - mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), - ]) + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600)]) def test_mime_text_x_shellscript(self): """Mime message of type text/x-shellscript is treated as script.""" @@ -470,8 +469,7 @@ c: 4 mockobj.assert_has_calls([ mock.call(outpath, script, 0o700), - mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), - ]) + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600)]) def test_mime_text_plain_shell(self): """Mime type text/plain starting #!/bin/sh is treated as script.""" @@ -491,8 +489,7 @@ c: 4 mockobj.assert_has_calls([ mock.call(outpath, script, 0o700), - mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600), - ]) + mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600)]) def test_mime_application_octet_stream(self): """Mime type application/octet-stream is ignored but shows warning.""" @@ -560,3 +557,20 @@ class TestUDProcess(helpers.ResourceUsingTestCase): ud_proc = ud.UserDataProcessor(self.getCloudPaths()) message = ud_proc.process(msg) self.assertTrue(count_messages(message) == 1) + + +class TestConvertString(helpers.TestCase): + def test_handles_binary_non_utf8_decodable(self): + blob = b'\x32\x99' + msg = ud.convert_string(blob) + self.assertEqual(blob, msg.get_payload(decode=True)) + + def test_handles_binary_utf8_decodable(self): + blob = b'\x32\x32' + msg = ud.convert_string(blob) + self.assertEqual(blob, msg.get_payload(decode=True)) + + def test_handle_headers(self): + text = "hi mom" + msg = ud.convert_string(text) + self.assertEqual(text, msg.get_payload(decode=False)) diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index 85759c68..12966563 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -134,7 +134,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('RHEV', dsrc.get_cloud_type()) + self.assertEqual('RHEV', dsrc.get_cloud_type()) def test_vsphere(self): ''' @@ -143,7 +143,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('VSPHERE', dsrc.get_cloud_type()) + self.assertEqual('VSPHERE', dsrc.get_cloud_type()) def test_unknown(self): ''' @@ -152,7 +152,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('UNKNOWN', dsrc.get_cloud_type()) + self.assertEqual('UNKNOWN', dsrc.get_cloud_type()) class TestGetDataCloudInfoFile(TestCase): @@ -187,7 +187,7 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_vsphere(self): '''Success Test module get_data() forcing VSPHERE.''' @@ -195,7 +195,7 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('VSPHERE') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_fail_rhev(self): '''Failure Test module get_data() forcing RHEV.''' @@ -203,7 +203,7 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: False - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) def test_fail_vsphere(self): '''Failure Test module get_data() forcing VSPHERE.''' @@ -211,14 +211,14 @@ class TestGetDataCloudInfoFile(TestCase): _write_cloud_info_file('VSPHERE') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: False - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) def test_unrecognized(self): '''Failure Test module get_data() forcing unrecognized.''' _write_cloud_info_file('unrecognized') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) class TestGetDataNoCloudInfoFile(TestCase): @@ -250,7 +250,7 @@ class TestGetDataNoCloudInfoFile(TestCase): util.read_dmi_data = _dmi_data('RHEV Hypervisor') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_rhevm = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_vsphere_no_cloud_file(self): '''Test No cloud info file module get_data() forcing VSPHERE.''' @@ -258,14 +258,14 @@ class TestGetDataNoCloudInfoFile(TestCase): util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) dsrc.user_data_vsphere = lambda: True - self.assertEquals(True, dsrc.get_data()) + self.assertEqual(True, dsrc.get_data()) def test_failure_no_cloud_file(self): '''Test No cloud info file module get_data() forcing unrecognized.''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.get_data()) + self.assertEqual(False, dsrc.get_data()) class TestUserDataRhevm(TestCase): @@ -305,7 +305,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_modprobe_fails(self): '''Test user_data_rhevm() where modprobe fails.''' @@ -315,7 +315,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_no_modprobe_cmd(self): '''Test user_data_rhevm() with no modprobe command.''' @@ -325,7 +325,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_udevadm_fails(self): '''Test user_data_rhevm() where udevadm fails.''' @@ -335,7 +335,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) def test_no_udevadm_cmd(self): '''Test user_data_rhevm() with no udevadm command.''' @@ -345,7 +345,7 @@ class TestUserDataRhevm(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_rhevm()) + self.assertEqual(False, dsrc.user_data_rhevm()) class TestUserDataVsphere(TestCase): @@ -380,7 +380,7 @@ class TestUserDataVsphere(TestCase): dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals(False, dsrc.user_data_vsphere()) + self.assertEqual(False, dsrc.user_data_vsphere()) class TestReadUserDataCallback(TestCase): @@ -408,8 +408,8 @@ class TestReadUserDataCallback(TestCase): def test_callback_both(self): '''Test read_user_data_callback() with both files.''' - self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + self.assertEqual('test user data', + read_user_data_callback(self.mount_dir)) def test_callback_dc(self): '''Test read_user_data_callback() with only DC file.''' @@ -418,8 +418,8 @@ class TestReadUserDataCallback(TestCase): dc_file=False, non_dc_file=True) - self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + self.assertEqual('test user data', + read_user_data_callback(self.mount_dir)) def test_callback_non_dc(self): '''Test read_user_data_callback() with only non-DC file.''' @@ -428,14 +428,14 @@ class TestReadUserDataCallback(TestCase): dc_file=True, non_dc_file=False) - self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + self.assertEqual('test user data', + read_user_data_callback(self.mount_dir)) def test_callback_none(self): '''Test read_user_data_callback() no files are found.''' _remove_user_data_files(self.mount_dir) - self.assertEquals(None, read_user_data_callback(self.mount_dir)) + self.assertEqual(None, read_user_data_callback(self.mount_dir)) def force_arch(arch=None): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 444e2799..e90e903c 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,24 +1,16 @@ from cloudinit import helpers from cloudinit.util import b64e, decode_binary, load_file from cloudinit.sources import DataSourceAzure -from ..helpers import TestCase, populate_dir -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import TestCase, populate_dir, mock, ExitStack, PY26, SkipTest import crypt import os -import stat -import yaml import shutil +import stat import tempfile import xml.etree.ElementTree as ET +import yaml def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): @@ -83,6 +75,8 @@ class TestAzureDataSource(TestCase): def setUp(self): super(TestAzureDataSource, self).setUp() + if PY26: + raise SkipTest("Does not work on python 2.6") self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) @@ -165,7 +159,7 @@ class TestAzureDataSource(TestCase): def tags_equal(x, y): for x_tag, x_val in x.items(): y_val = y.get(x_val.tag) - self.assertEquals(x_val.text, y_val.text) + self.assertEqual(x_val.text, y_val.text) old_cnt = create_tag_index(oxml) new_cnt = create_tag_index(nxml) @@ -354,8 +348,8 @@ class TestAzureDataSource(TestCase): self.assertTrue(ret) cfg = dsrc.get_config_obj() - self.assertEquals(dsrc.device_name_to_device("ephemeral0"), - "/dev/sdb") + self.assertEqual(dsrc.device_name_to_device("ephemeral0"), + "/dev/sdb") assert 'disk_setup' in cfg assert 'fs_setup' in cfg self.assertIsInstance(cfg['disk_setup'], dict) @@ -404,15 +398,15 @@ class TestAzureDataSource(TestCase): self.xml_notequals(data['ovfcontent'], on_disk_ovf) # Make sure that the redacted password on disk is not used by CI - self.assertNotEquals(dsrc.cfg.get('password'), - DataSourceAzure.DEF_PASSWD_REDACTION) + self.assertNotEqual(dsrc.cfg.get('password'), + DataSourceAzure.DEF_PASSWD_REDACTION) # Make sure that the password was really encrypted et = ET.fromstring(on_disk_ovf) for elem in et.iter(): if 'UserPassword' in elem.tag: - self.assertEquals(DataSourceAzure.DEF_PASSWD_REDACTION, - elem.text) + self.assertEqual(DataSourceAzure.DEF_PASSWD_REDACTION, + elem.text) def test_ovf_env_arrives_in_waagent_dir(self): xml = construct_valid_ovf_env(data={}, userdata="FOODATA") diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 1134199b..65202ff0 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -1,17 +1,8 @@ import os from cloudinit.sources.helpers import azure as azure_helper -from ..helpers import TestCase -try: - from unittest import mock -except ImportError: - import mock - -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import ExitStack, mock, TestCase GOAL_STATE_TEMPLATE = """\ @@ -70,7 +61,7 @@ class TestFindEndpoint(TestCase): def test_missing_special_azure_line(self): self.load_file.return_value = '' - self.assertRaises(Exception, + self.assertRaises(ValueError, azure_helper.WALinuxAgentShim.find_endpoint) @staticmethod @@ -287,6 +278,7 @@ class TestOpenSSLManager(TestCase): self.subp.side_effect = capture_directory manager = azure_helper.OpenSSLManager() self.assertEqual(manager.tmpdir, subp_directory['path']) + manager.clean_up() @mock.patch.object(azure_helper, 'cd', mock.MagicMock()) @mock.patch.object(azure_helper.tempfile, 'mkdtemp', mock.MagicMock()) diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index 772d189a..2a42ce0c 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -1,4 +1,5 @@ # coding: utf-8 + import copy from cloudinit.cs_utils import Cepko @@ -6,7 +7,6 @@ from cloudinit.sources import DataSourceCloudSigma from .. import helpers as test_helpers - SERVER_CONTEXT = { "cpu": 1000, "cpus_instead_of_cores": False, diff --git a/tests/unittests/test_datasource/test_cloudstack.py b/tests/unittests/test_datasource/test_cloudstack.py index 656d80d1..b1aab17b 100644 --- a/tests/unittests/test_datasource/test_cloudstack.py +++ b/tests/unittests/test_datasource/test_cloudstack.py @@ -1,15 +1,7 @@ from cloudinit import helpers from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack -from ..helpers import TestCase -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +from ..helpers import TestCase, mock, ExitStack class TestCloudStackPasswordFetching(TestCase): diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 89b15f54..18551b92 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -5,22 +5,15 @@ import shutil import six import tempfile -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack - from cloudinit import helpers +from cloudinit.net import eni +from cloudinit.net import network_state from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from ..helpers import TestCase +from ..helpers import TestCase, ExitStack, mock PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' @@ -73,7 +66,7 @@ NETWORK_DATA = { 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', 'ethernet_mac_address': 'fa:16:3e:05:30:fe', - 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04', 'name': 'nic0'} ], 'networks': [ {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', @@ -88,6 +81,35 @@ NETWORK_DATA = { ] } +NETWORK_DATA_2 = { + "services": [ + {"type": "dns", "address": "1.1.1.191"}, + {"type": "dns", "address": "1.1.1.4"}], + "networks": [ + {"network_id": "d94bbe94-7abc-48d4-9c82-4628ea26164a", "type": "ipv4", + "netmask": "255.255.255.248", "link": "eth0", + "routes": [{"netmask": "0.0.0.0", "network": "0.0.0.0", + "gateway": "2.2.2.9"}], + "ip_address": "2.2.2.10", "id": "network0-ipv4"}, + {"network_id": "ca447c83-6409-499b-aaef-6ad1ae995348", "type": "ipv4", + "netmask": "255.255.255.224", "link": "eth1", + "routes": [], "ip_address": "3.3.3.24", "id": "network1-ipv4"}], + "links": [ + {"ethernet_mac_address": "fa:16:3e:dd:50:9a", "mtu": 1500, + "type": "vif", "id": "eth0", "vif_id": "vif-foo1"}, + {"ethernet_mac_address": "fa:16:3e:a8:14:69", "mtu": 1500, + "type": "vif", "id": "eth1", "vif_id": "vif-foo2"}] +} + + +KNOWN_MACS = { + 'fa:16:3e:69:b0:58': 'enp0s1', + 'fa:16:3e:d4:57:ad': 'enp0s2', + 'fa:16:3e:dd:50:9a': 'foo1', + 'fa:16:3e:a8:14:69': 'foo2', + 'fa:16:3e:ed:9a:59': 'foo3', +} + CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), 'ec2/2009-04-04/user-data': USER_DATA, @@ -151,7 +173,7 @@ class TestConfigDriveDataSource(TestCase): mock.patch.object(os.path, 'exists', side_effect=exists_side_effect())) device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) find_mock.assert_called_once_with(mock.ANY) self.assertEqual(exists_mock.call_count, 2) @@ -179,7 +201,7 @@ class TestConfigDriveDataSource(TestCase): mock.patch.object(os.path, 'exists', return_value=True)) device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) find_mock.assert_called_once_with(mock.ANY) exists_mock.assert_called_once_with(mock.ANY) @@ -214,7 +236,7 @@ class TestConfigDriveDataSource(TestCase): with mock.patch.object(os.path, 'exists', side_effect=exists_side_effect()): device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) # We don't assert the call count for os.path.exists() because # not all of the entries in name_tests results in two calls to # that function. Specifically, 'root2k' doesn't seem to call @@ -242,7 +264,7 @@ class TestConfigDriveDataSource(TestCase): for name, dev_name in name_tests.items(): with mock.patch.object(os.path, 'exists', return_value=True): device = cfg_ds.device_name_to_device(name) - self.assertEquals(dev_name, device) + self.assertEqual(dev_name, device) def test_dir_valid(self): """Verify a dir is read as such.""" @@ -348,32 +370,200 @@ class TestConfigDriveDataSource(TestCase): util.find_devs_with = orig_find_devs_with util.is_partition = orig_is_partition - def test_pubkeys_v2(self): + @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot') + def test_pubkeys_v2(self, on_first_boot): """Verify that public-keys work in config-drive-v2.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) self.assertEqual(myds.get_public_ssh_keys(), [OSTACK_META['public_keys']['mykey']]) - def test_network_data_is_found(self): + +class TestNetJson(TestCase): + def setUp(self): + super(TestNetJson, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.maxDiff = None + + @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot') + def test_network_data_is_found(self, on_first_boot): """Verify that network_data is present in ds in config-drive-v2.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - self.assertEqual(myds.network_json, NETWORK_DATA) + self.assertIsNotNone(myds.network_json) - def test_network_config_is_converted(self): + @mock.patch('cloudinit.sources.DataSourceConfigDrive.on_first_boot') + def test_network_config_is_converted(self, on_first_boot): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - network_config = ds.convert_network_data(NETWORK_DATA) + network_config = openstack.convert_net_json(NETWORK_DATA, + known_macs=KNOWN_MACS) self.assertEqual(myds.network_config, network_config) + def test_network_config_conversions(self): + """Tests a bunch of input network json and checks the + expected conversions.""" + in_datas = [ + NETWORK_DATA, + { + 'services': [{'type': 'dns', 'address': '172.19.0.12'}], + 'networks': [{ + 'network_id': 'dacd568d-5be6-4786-91fe-750c374b78b4', + 'type': 'ipv4', + 'netmask': '255.255.252.0', + 'link': 'tap1a81968a-79', + 'routes': [{ + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + 'gateway': '172.19.3.254', + }], + 'ip_address': '172.19.1.34', + 'id': 'network0', + }], + 'links': [{ + 'type': 'bridge', + 'vif_id': '1a81968a-797a-400f-8a80-567f997eb93f', + 'ethernet_mac_address': 'fa:16:3e:ed:9a:59', + 'id': 'tap1a81968a-79', + 'mtu': None, + }], + }, + ] + out_datas = [ + { + 'version': 1, + 'config': [ + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:69:b0:58', + 'name': 'enp0s1', + 'mtu': None, + }, + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:d4:57:ad', + 'name': 'enp0s2', + 'mtu': None, + }, + { + 'subnets': [{'type': 'dhcp4'}], + 'type': 'physical', + 'mac_address': 'fa:16:3e:05:30:fe', + 'name': 'nic0', + 'mtu': None, + }, + { + 'type': 'nameserver', + 'address': '199.204.44.24', + }, + { + 'type': 'nameserver', + 'address': '199.204.47.54', + } + ], + + }, + { + 'version': 1, + 'config': [ + { + 'name': 'foo3', + 'mac_address': 'fa:16:3e:ed:9a:59', + 'mtu': None, + 'type': 'physical', + 'subnets': [ + { + 'address': '172.19.1.34', + 'netmask': '255.255.252.0', + 'type': 'static', + 'ipv4': True, + 'routes': [{ + 'gateway': '172.19.3.254', + 'netmask': '0.0.0.0', + 'network': '0.0.0.0', + }], + } + ] + }, + { + 'type': 'nameserver', + 'address': '172.19.0.12', + } + ], + }, + ] + for in_data, out_data in zip(in_datas, out_datas): + conv_data = openstack.convert_net_json(in_data, + known_macs=KNOWN_MACS) + self.assertEqual(out_data, conv_data) + + +class TestConvertNetworkData(TestCase): + def setUp(self): + super(TestConvertNetworkData, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + + def _getnames_in_config(self, ncfg): + return set([n['name'] for n in ncfg['config'] + if n['type'] == 'physical']) + + def test_conversion_fills_names(self): + ncfg = openstack.convert_net_json(NETWORK_DATA, known_macs=KNOWN_MACS) + expected = set(['nic0', 'enp0s1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): + macs = KNOWN_MACS.copy() + macs.update({'fa:16:3e:05:30:fe': 'foonic1', + 'fa:16:3e:69:b0:58': 'ens1'}) + get_interfaces_by_mac.return_value = macs + + ncfg = openstack.convert_net_json(NETWORK_DATA) + expected = set(['nic0', 'ens1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + def test_convert_raises_value_error_on_missing_name(self): + macs = {'aa:aa:aa:aa:aa:00': 'ens1'} + self.assertRaises(ValueError, openstack.convert_net_json, + NETWORK_DATA, known_macs=macs) + + def test_conversion_with_route(self): + ncfg = openstack.convert_net_json(NETWORK_DATA_2, + known_macs=KNOWN_MACS) + # not the best test, but see that we get a route in the + # network config and that it gets rendered to an ENI file + routes = [] + for n in ncfg['config']: + for s in n.get('subnets', []): + routes.extend(s.get('routes', [])) + self.assertIn( + {'network': '0.0.0.0', 'netmask': '0.0.0.0', 'gateway': '2.2.2.9'}, + routes) + eni_renderer = eni.Renderer() + eni_renderer.render_network_state( + self.tmp, network_state.parse_net_config_data(ncfg)) + with open(os.path.join(self.tmp, "etc", + "network", "interfaces"), 'r') as f: + eni_rendering = f.read() + self.assertIn("route add default gw 2.2.2.9", eni_rendering) + def cfg_ds_from_dir(seed_d): - found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - populate_ds_from_read_config(cfg_ds, seed_d, found) + cfg_ds.seed_dir = seed_d + cfg_ds.known_macs = KNOWN_MACS.copy() + if not cfg_ds.get_data(): + raise RuntimeError("Data source did not extract itself from" + " seed directory %s" % seed_d) return cfg_ds @@ -387,7 +577,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') cfg_ds.network_json = results.get('networkdata') - cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) + cfg_ds._network_config = openstack.convert_net_json( + cfg_ds.network_json, known_macs=KNOWN_MACS) def populate_dir(seed_dir, files): @@ -400,7 +591,6 @@ def populate_dir(seed_dir, files): mode = "w" else: mode = "wb" - with open(path, mode) as fp: fp.write(content) diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index 679d1b82..8936a1e3 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -19,8 +19,8 @@ import re from six.moves.urllib_parse import urlparse -from cloudinit import settings from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceDigitalOcean from .. import helpers as test_helpers diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index fa714070..6e62a4d2 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -20,8 +20,8 @@ import re from base64 import b64encode, b64decode from six.moves.urllib_parse import urlparse -from cloudinit import settings from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceGCE from .. import helpers as test_helpers @@ -52,7 +52,7 @@ GCE_META_ENCODING = { HEADERS = {'X-Google-Metadata-Request': 'True'} MD_URL_RE = re.compile( - r'http://metadata.google.internal./computeMetadata/v1/.*') + r'http://metadata.google.internal/computeMetadata/v1/.*') def _set_mock_metadata(gce_meta=None): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 77d15cac..f66f1c6d 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -104,13 +104,13 @@ class TestMAASDataSource(TestCase): 'meta-data/local-hostname': 'test-hostname', 'meta-data/public-keys': 'test-hostname', 'user-data': b'foodata', - } + } valid_order = [ 'meta-data/local-hostname', 'meta-data/instance-id', 'meta-data/public-keys', 'user-data', - ] + ] my_seed = "http://example.com/xmeta" my_ver = "1999-99-99" my_headers = {'header1': 'value1', 'header2': 'value2'} diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 2d5fc37c..b0fa1130 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,22 +1,13 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from ..helpers import TestCase, populate_dir +from ..helpers import TestCase, populate_dir, mock, ExitStack import os -import yaml import shutil import tempfile -import unittest -try: - from unittest import mock -except ImportError: - import mock -try: - from contextlib import ExitStack -except ImportError: - from contextlib2 import ExitStack +import yaml class TestNoCloudDataSource(TestCase): @@ -139,7 +130,7 @@ class TestNoCloudDataSource(TestCase): self.assertTrue(ret) -class TestParseCommandLineData(unittest.TestCase): +class TestParseCommandLineData(TestCase): def test_parse_cmdline_data_valid(self): ds_id = "ds=nocloud" diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 0aa1ba84..5c8592c5 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -22,8 +22,8 @@ import re from .. import helpers as test_helpers -from six import StringIO from six.moves.urllib.parse import urlparse +from six import StringIO from cloudinit import helpers from cloudinit import settings @@ -135,41 +135,45 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): body=get_request_callback) +def _read_metadata_service(): + return ds.read_metadata_service(BASE_URL, retries=0, timeout=0.1) + + class TestOpenStackDataSource(test_helpers.HttprettyTestCase): VERSION = 'latest' @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - f = ds.read_metadata_service(BASE_URL) - self.assertEquals(VENDOR_DATA, f.get('vendordata')) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) - self.assertEquals(2, len(f['files'])) - self.assertEquals(USER_DATA, f.get('userdata')) - self.assertEquals(EC2_META, f.get('ec2-metadata')) - self.assertEquals(2, f.get('version')) + f = _read_metadata_service() + self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEqual(2, len(f['files'])) + self.assertEqual(USER_DATA, f.get('userdata')) + self.assertEqual(EC2_META, f.get('ec2-metadata')) + self.assertEqual(2, f.get('version')) metadata = f['metadata'] - self.assertEquals('nova', metadata.get('availability_zone')) - self.assertEquals('sm-foo-test.novalocal', metadata.get('hostname')) - self.assertEquals('sm-foo-test.novalocal', - metadata.get('local-hostname')) - self.assertEquals('sm-foo-test', metadata.get('name')) - self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', - metadata.get('uuid')) - self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', - metadata.get('instance-id')) + self.assertEqual('nova', metadata.get('availability_zone')) + self.assertEqual('sm-foo-test.novalocal', metadata.get('hostname')) + self.assertEqual('sm-foo-test.novalocal', + metadata.get('local-hostname')) + self.assertEqual('sm-foo-test', metadata.get('name')) + self.assertEqual('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('uuid')) + self.assertEqual('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('instance-id')) @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) - f = ds.read_metadata_service(BASE_URL) - self.assertEquals(VENDOR_DATA, f.get('vendordata')) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) - self.assertEquals(USER_DATA, f.get('userdata')) - self.assertEquals({}, f.get('ec2-metadata')) - self.assertEquals(2, f.get('version')) + f = _read_metadata_service() + self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEqual(USER_DATA, f.get('userdata')) + self.assertEqual({}, f.get('ec2-metadata')) + self.assertEqual(2, f.get('version')) @hp.activate def test_bad_metadata(self): @@ -178,8 +182,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.NonReadable, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.NonReadable, _read_metadata_service) @hp.activate def test_bad_uuid(self): @@ -190,8 +193,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_userdata_empty(self): @@ -200,10 +202,10 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('user_data'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) - self.assertEquals(VENDOR_DATA, f.get('vendordata')) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + f = _read_metadata_service() + self.assertEqual(VENDOR_DATA, f.get('vendordata')) + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('userdata')) @hp.activate @@ -213,9 +215,9 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) - self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) - self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + f = _read_metadata_service() + self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @hp.activate @@ -225,8 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_metadata_invalid(self): @@ -235,8 +236,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_datasource(self): @@ -245,18 +245,18 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertTrue(found) - self.assertEquals(2, ds_os.version) + self.assertEqual(2, ds_os.version) md = dict(ds_os.metadata) md.pop('instance-id', None) md.pop('local-hostname', None) - self.assertEquals(OSTACK_META, md) - self.assertEquals(EC2_META, ds_os.ec2_metadata) - self.assertEquals(USER_DATA, ds_os.userdata_raw) - self.assertEquals(2, len(ds_os.files)) - self.assertEquals(VENDOR_DATA, ds_os.vendordata_pure) - self.assertEquals(ds_os.vendordata_raw, None) + self.assertEqual(OSTACK_META, md) + self.assertEqual(EC2_META, ds_os.ec2_metadata) + self.assertEqual(USER_DATA, ds_os.userdata_raw) + self.assertEqual(2, len(ds_os.files)) + self.assertEqual(VENDOR_DATA, ds_os.vendordata_pure) + self.assertEqual(ds_os.vendordata_raw, None) @hp.activate def test_bad_datasource_meta(self): @@ -269,7 +269,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -288,7 +288,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -311,7 +311,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..9c6c8768 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,6 +24,8 @@ from __future__ import print_function +from binascii import crc32 +import json import os import os.path import re @@ -31,21 +33,58 @@ import shutil import stat import tempfile import uuid -from binascii import crc32 -import serial +from cloudinit import serial +from cloudinit.sources import DataSourceSmartOS + import six from cloudinit import helpers as c_helpers -from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e -from .. import helpers - -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import mock, FilesystemMockingTestCase, TestCase + +SDC_NICS = json.loads(""" +[ + { + "nic_tag": "external", + "primary": true, + "mtu": 1500, + "model": "virtio", + "gateway": "8.12.42.1", + "netmask": "255.255.255.0", + "ip": "8.12.42.102", + "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe", + "gateways": [ + "8.12.42.1" + ], + "vlan_id": 324, + "mac": "90:b8:d0:f5:e4:f5", + "interface": "net0", + "ips": [ + "8.12.42.102/24" + ] + }, + { + "nic_tag": "sdc_overlay/16187209", + "gateway": "192.168.128.1", + "model": "virtio", + "mac": "90:b8:d0:a5:ff:cd", + "netmask": "255.255.252.0", + "ip": "192.168.128.93", + "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9", + "gateways": [ + "192.168.128.1" + ], + "vlan_id": 2, + "mtu": 8500, + "interface": "net1", + "ips": [ + "192.168.128.93/22" + ] + } +] +""") MOCK_RETURNS = { 'hostname': 'test-host', @@ -60,79 +99,68 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), + 'sdc:nics': json.dumps(SDC_NICS), } DMI_DATA_RETURN = 'smartdc' -def get_mock_client(mockdata): - class MockMetadataClient(object): +class PsuedoJoyentClient(object): + def __init__(self, data=None): + if data is None: + data = MOCK_RETURNS.copy() + self.data = data + return + + def get(self, key, default=None, strip=False): + if key in self.data: + r = self.data[key] + if strip: + r = r.strip() + else: + r = default + return r - def __init__(self, serial): - pass + def get_json(self, key, default=None): + result = self.get(key, default=default) + if result is None: + return default + return json.loads(result) - def get_metadata(self, metadata_key): - return mockdata.get(metadata_key) - return MockMetadataClient + def exists(self): + return True -class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): +class TestSmartOSDataSource(FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() + dsmos = 'cloudinit.sources.DataSourceSmartOS' + patcher = mock.patch(dsmos + ".jmc_client_factory") + self.jmc_cfact = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch(dsmos + ".get_smartos_environ") + self.get_smartos_environ = patcher.start() + self.addCleanup(patcher.stop) + self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) - self.legacy_user_d = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.legacy_user_d) - - # If you should want to watch the logs... - self._log = None - self._log_file = None - self._log_handler = None - - # patch cloud_dir, so our 'seed_dir' is guaranteed empty self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) - self.unapply = [] - super(TestSmartOSDataSource, self).setUp() + self.legacy_user_d = os.path.join(self.tmp, 'legacy_user_tmp') + os.mkdir(self.legacy_user_d) + + self.orig_lud = DataSourceSmartOS.LEGACY_USER_D + DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d def tearDown(self): - helpers.FilesystemMockingTestCase.tearDown(self) - if self._log_handler and self._log: - self._log.removeHandler(self._log_handler) - apply_patches([i for i in reversed(self.unapply)]) + DataSourceSmartOS.LEGACY_USER_D = self.orig_lud super(TestSmartOSDataSource, self).tearDown() - def _patchIn(self, root): - self.restore() - self.patchOS(root) - self.patchUtils(root) - - def apply_patches(self, patches): - ret = apply_patches(patches) - self.unapply += ret - - def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None, - is_lxbrand=False): - mod = DataSourceSmartOS - - if mockdata is None: - mockdata = MOCK_RETURNS - - if dmi_data is None: - dmi_data = DMI_DATA_RETURN - - def _dmi_data(): - return dmi_data - - def _os_uname(): - if not is_lxbrand: - # LP: #1243287. tests assume this runs, but running test on - # arm would cause them all to fail. - return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64') - else: - return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX', - 'X86_64') + def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM, + sys_cfg=None, ds_cfg=None): + self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) + self.get_smartos_environ.return_value = mode if sys_cfg is None: sys_cfg = {} @@ -141,44 +169,8 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): sys_cfg['datasource'] = sys_cfg.get('datasource', {}) sys_cfg['datasource']['SmartOS'] = ds_cfg - self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) - self.apply_patches([ - (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(mod, 'dmi_data', _dmi_data)]) - self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(mod, 'device_exists', lambda d: True)]) - dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, - paths=self.paths) - self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) - return dsrc - - def test_seed(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals('kvm', dsrc.smartos_type) - self.assertEquals('/dev/ttyS1', dsrc.seed) - - def test_seed_lxbrand(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals('lx-brand', dsrc.smartos_type) - self.assertEquals('/native/.zonecontrol/metadata.sock', dsrc.seed) - - def test_issmartdc(self): - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_issmartdc_lxbrand(self): - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) + return DataSourceSmartOS.DataSourceSmartOS( + sys_cfg, distro=None, paths=self.paths) def test_no_base64(self): ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} @@ -190,110 +182,65 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:uuid'], - dsrc.metadata['instance-id']) + self.assertEqual(MOCK_RETURNS['sdc:uuid'], + dsrc.metadata['instance-id']) def test_root_keys(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) def test_hostname_b64(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) def test_hostname(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_base64_all(self): - # metadata provided base64_all of true - my_returns = MOCK_RETURNS.copy() - my_returns['base64_all'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEquals(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_b64_userdata(self): - my_returns = MOCK_RETURNS.copy() - my_returns['b64-cloud-init:user-data'] = "true" - my_returns['b64-hostname'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEquals(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_b64_keys(self): - my_returns = MOCK_RETURNS.copy() - my_returns['base64_keys'] = 'hostname,ignored' - for k in ('hostname',): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) + def test_userdata(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) + self.assertEqual(MOCK_RETURNS['user-data'], + dsrc.metadata['legacy-user-data']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) - def test_userdata(self): + def test_sdc_nics(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-data'], - dsrc.metadata['legacy-user-data']) - self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) + self.assertEqual(json.loads(MOCK_RETURNS['sdc:nics']), + dsrc.metadata['network-data']) def test_sdc_scripts(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d self.assertTrue(os.path.exists(legacy_script_f)) self.assertTrue(os.path.islink(legacy_script_f)) user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEquals(user_script_perm, '700') + self.assertEqual(user_script_perm, '700') def test_scripts_shebanged(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d self.assertTrue(os.path.exists(legacy_script_f)) @@ -301,9 +248,9 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): shebang = None with open(legacy_script_f, 'r') as f: shebang = f.readlines()[0].strip() - self.assertEquals(shebang, "#!/bin/bash") + self.assertEqual(shebang, "#!/bin/bash") user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEquals(user_script_perm, '700') + self.assertEqual(user_script_perm, '700') def test_scripts_shebang_not_added(self): """ @@ -319,8 +266,8 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(my_returns['user-script'], - dsrc.metadata['user-script']) + self.assertEqual(my_returns['user-script'], + dsrc.metadata['user-script']) legacy_script_f = "%s/user-script" % self.legacy_user_d self.assertTrue(os.path.exists(legacy_script_f)) @@ -328,7 +275,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): shebang = None with open(legacy_script_f, 'r') as f: shebang = f.readlines()[0].strip() - self.assertEquals(shebang, "#!/usr/bin/perl") + self.assertEqual(shebang, "#!/usr/bin/perl") def test_userdata_removed(self): """ @@ -358,7 +305,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): if re.match(r'.*\/mdata-user-data$', name_f): found_new = True print(name_f) - self.assertEquals(permissions, '400') + self.assertEqual(permissions, '400') self.assertFalse(found_new) @@ -366,8 +313,8 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:vendor-data'], - dsrc.metadata['vendor-data']) + self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], + dsrc.metadata['vendor-data']) def test_default_vendor_data(self): my_returns = MOCK_RETURNS.copy() @@ -376,7 +323,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertNotEquals(def_op_script, dsrc.metadata['vendor-data']) + self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) # we expect default vendor-data is a boothook self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) @@ -385,15 +332,15 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) + self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) def test_motd_sys_info(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) + self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) def test_default_ephemeral(self): # Test to make sure that the builtin config has the ephemeral @@ -430,21 +377,11 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) -def apply_patches(patches): - ret = [] - for (ref, name, replace) in patches: - if replace is None: - continue - orig = getattr(ref, name) - setattr(ref, name, replace) - ret.append((ref, name, orig)) - return ret - - -class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): +class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() + self.serial = mock.MagicMock(spec=serial.Serial) self.request_id = 0xabcdef12 self.metadata_value = 'value' @@ -481,7 +418,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataClient(self.serial) + return DataSourceSmartOS.JoyentMetadataClient( + fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -495,7 +433,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_writes_a_single_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(1, self.serial.write.call_count) written_line = self.serial.write.call_args[0][0] print(type(written_line)) @@ -505,7 +443,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def _get_written_line(self, key='some_key'): client = self._get_client() - client.get_metadata(key) + client.get(key) return self.serial.write.call_args[0][0] def test_get_metadata_writes_bytes(self): @@ -549,32 +487,32 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_reads_a_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(self.metasource_data_len, self.serial.read.call_count) def test_get_metadata_returns_valid_value(self): client = self._get_client() - value = client.get_metadata('some_key') + value = client.get('some_key') self.assertEqual(self.metadata_value, value) def test_get_metadata_throws_exception_for_incorrect_length(self): self.response_parts['length'] = 0 client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_incorrect_crc(self): self.response_parts['crc'] = 'deadbeef' client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_request_id_mismatch(self): self.response_parts['request_id'] = 'deadbeef' client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_returns_None_if_value_not_found(self): self.response_parts['payload'] = '' @@ -582,4 +520,24 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): self.response_parts['length'] = 17 client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] - self.assertIsNone(client.get_metadata('some_key')) + self.assertIsNone(client.get('some_key')) + + +class TestNetworkConversion(TestCase): + + def test_convert_simple(self): + expected = { + 'version': 1, + 'config': [ + {'name': 'net0', 'type': 'physical', + 'subnets': [{'type': 'static', 'gateway': '8.12.42.1', + 'netmask': '255.255.255.0', + 'address': '8.12.42.102/24'}], + 'mtu': 1500, 'mac_address': '90:b8:d0:f5:e4:f5'}, + {'name': 'net1', 'type': 'physical', + 'subnets': [{'type': 'static', 'gateway': '192.168.128.1', + 'netmask': '255.255.252.0', + 'address': '192.168.128.93/22'}], + 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]} + found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS) + self.assertEqual(expected, found) diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 6ed1704c..96fa0811 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -87,13 +87,13 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): rules = 'ALL=(ALL:ALL) ALL' contents = self._write_load_sudoers('harlowja', rules) expected = ['harlowja ALL=(ALL:ALL) ALL'] - self.assertEquals(len(expected), self._count_in(expected, contents)) + self.assertEqual(len(expected), self._count_in(expected, contents)) not_expected = [ 'harlowja A', 'harlowja L', 'harlowja L', ] - self.assertEquals(0, self._count_in(not_expected, contents)) + self.assertEqual(0, self._count_in(not_expected, contents)) def test_sudoers_ensure_rules_list(self): rules = [ @@ -107,13 +107,13 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): 'harlowja B-ALL=(ALL:ALL) ALL', 'harlowja C-ALL=(ALL:ALL) ALL', ] - self.assertEquals(len(expected), self._count_in(expected, contents)) + self.assertEqual(len(expected), self._count_in(expected, contents)) not_expected = [ 'harlowja A', 'harlowja L', 'harlowja L', ] - self.assertEquals(0, self._count_in(not_expected, contents)) + self.assertEqual(0, self._count_in(not_expected, contents)) def test_sudoers_ensure_new(self): cls = distros.fetch("ubuntu") @@ -136,7 +136,7 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): self.assertIn("includedir /b", contents) self.assertTrue(os.path.isdir("/b")) self.assertIn("josh", contents) - self.assertEquals(2, contents.count("josh")) + self.assertEqual(2, contents.count("josh")) def test_arch_package_mirror_info_unknown(self): """for an unknown arch, we should get back that with arch 'default'.""" diff --git a/tests/unittests/test_distros/test_hostname.py b/tests/unittests/test_distros/test_hostname.py index 143e29a9..5f28a868 100644 --- a/tests/unittests/test_distros/test_hostname.py +++ b/tests/unittests/test_distros/test_hostname.py @@ -15,24 +15,24 @@ BASE_HOSTNAME = BASE_HOSTNAME.strip() class TestHostnameHelper(unittest.TestCase): def test_parse_same(self): hn = hostname.HostnameConf(BASE_HOSTNAME) - self.assertEquals(str(hn).strip(), BASE_HOSTNAME) - self.assertEquals(hn.hostname, 'blahblah') + self.assertEqual(str(hn).strip(), BASE_HOSTNAME) + self.assertEqual(hn.hostname, 'blahblah') def test_no_adjust_hostname(self): hn = hostname.HostnameConf(BASE_HOSTNAME) prev_name = hn.hostname hn.set_hostname("") - self.assertEquals(hn.hostname, prev_name) + self.assertEqual(hn.hostname, prev_name) def test_adjust_hostname(self): hn = hostname.HostnameConf(BASE_HOSTNAME) prev_name = hn.hostname - self.assertEquals(prev_name, 'blahblah') + self.assertEqual(prev_name, 'blahblah') hn.set_hostname("bbbbd") - self.assertEquals(hn.hostname, 'bbbbd') + self.assertEqual(hn.hostname, 'bbbbd') expected_out = ''' # My super-duper-hostname bbbbd ''' - self.assertEquals(str(hn).strip(), expected_out.strip()) + self.assertEqual(str(hn).strip(), expected_out.strip()) diff --git a/tests/unittests/test_distros/test_hosts.py b/tests/unittests/test_distros/test_hosts.py index fc701eaa..ab867c6f 100644 --- a/tests/unittests/test_distros/test_hosts.py +++ b/tests/unittests/test_distros/test_hosts.py @@ -17,25 +17,25 @@ BASE_ETC = BASE_ETC.strip() class TestHostsHelper(unittest.TestCase): def test_parse(self): eh = hosts.HostsConf(BASE_ETC) - self.assertEquals(eh.get_entry('127.0.0.1'), [['localhost']]) - self.assertEquals(eh.get_entry('192.168.1.10'), - [['foo.mydomain.org', 'foo'], - ['bar.mydomain.org', 'bar']]) + self.assertEqual(eh.get_entry('127.0.0.1'), [['localhost']]) + self.assertEqual(eh.get_entry('192.168.1.10'), + [['foo.mydomain.org', 'foo'], + ['bar.mydomain.org', 'bar']]) eh = str(eh) self.assertTrue(eh.startswith('# Example')) def test_add(self): eh = hosts.HostsConf(BASE_ETC) eh.add_entry('127.0.0.0', 'blah') - self.assertEquals(eh.get_entry('127.0.0.0'), [['blah']]) + self.assertEqual(eh.get_entry('127.0.0.0'), [['blah']]) eh.add_entry('127.0.0.3', 'blah', 'blah2', 'blah3') - self.assertEquals(eh.get_entry('127.0.0.3'), - [['blah', 'blah2', 'blah3']]) + self.assertEqual(eh.get_entry('127.0.0.3'), + [['blah', 'blah2', 'blah3']]) def test_del(self): eh = hosts.HostsConf(BASE_ETC) eh.add_entry('127.0.0.0', 'blah') - self.assertEquals(eh.get_entry('127.0.0.0'), [['blah']]) + self.assertEqual(eh.get_entry('127.0.0.0'), [['blah']]) eh.del_entries('127.0.0.0') - self.assertEquals(eh.get_entry('127.0.0.0'), []) + self.assertEqual(eh.get_entry('127.0.0.0'), []) diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 2c2a424d..9172e3aa 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -1,4 +1,5 @@ import os +from six import StringIO try: from unittest import mock @@ -9,16 +10,14 @@ try: except ImportError: from contextlib2 import ExitStack -from six import StringIO from ..helpers import TestCase from cloudinit import distros +from cloudinit.distros.parsers.sys_conf import SysConf from cloudinit import helpers from cloudinit import settings from cloudinit import util -from cloudinit.distros.parsers.sys_conf import SysConf - BASE_NET_CFG = ''' auto lo @@ -108,23 +107,23 @@ class TestNetCfgDistro(TestCase): ub_distro.apply_network(BASE_NET_CFG, False) - self.assertEquals(len(write_bufs), 1) + self.assertEqual(len(write_bufs), 1) eni_name = '/etc/network/interfaces.d/50-cloud-init.cfg' self.assertIn(eni_name, write_bufs) write_buf = write_bufs[eni_name] - self.assertEquals(str(write_buf).strip(), BASE_NET_CFG.strip()) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(str(write_buf).strip(), BASE_NET_CFG.strip()) + self.assertEqual(write_buf.mode, 0o644) def assertCfgEquals(self, blob1, blob2): b1 = dict(SysConf(blob1.strip().splitlines())) b2 = dict(SysConf(blob2.strip().splitlines())) - self.assertEquals(b1, b2) + self.assertEqual(b1, b2) for (k, v) in b1.items(): self.assertIn(k, b2) for (k, v) in b2.items(): self.assertIn(k, b1) for (k, v) in b1.items(): - self.assertEquals(v, b2[k]) + self.assertEqual(v, b2[k]) def test_simple_write_rh(self): rh_distro = self._get_distro('rhel') @@ -148,7 +147,7 @@ class TestNetCfgDistro(TestCase): rh_distro.apply_network(BASE_NET_CFG, False) - self.assertEquals(len(write_bufs), 4) + self.assertEqual(len(write_bufs), 4) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] @@ -157,7 +156,7 @@ DEVICE="lo" ONBOOT=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', write_bufs) @@ -172,7 +171,7 @@ GATEWAY="192.168.1.254" BROADCAST="192.168.1.0" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', write_bufs) @@ -183,7 +182,7 @@ BOOTPROTO="dhcp" ONBOOT=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network', write_bufs) write_buf = write_bufs['/etc/sysconfig/network'] @@ -192,7 +191,7 @@ ONBOOT=yes NETWORKING=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) def test_write_ipv6_rhel(self): rh_distro = self._get_distro('rhel') @@ -216,7 +215,7 @@ NETWORKING=yes rh_distro.apply_network(BASE_NET_CFG_IPV6, False) - self.assertEquals(len(write_bufs), 4) + self.assertEqual(len(write_bufs), 4) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-lo', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-lo'] @@ -225,7 +224,7 @@ DEVICE="lo" ONBOOT=yes ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth0', write_bufs) @@ -243,7 +242,7 @@ IPV6ADDR="2607:f0d0:1002:0011::2" IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network-scripts/ifcfg-eth1', write_bufs) write_buf = write_bufs['/etc/sysconfig/network-scripts/ifcfg-eth1'] @@ -260,7 +259,7 @@ IPV6ADDR="2607:f0d0:1002:0011::3" IPV6_DEFAULTGW="2607:f0d0:1002:0011::1" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) self.assertIn('/etc/sysconfig/network', write_bufs) write_buf = write_bufs['/etc/sysconfig/network'] @@ -271,7 +270,7 @@ NETWORKING_IPV6=yes IPV6_AUTOCONF=no ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) def test_simple_write_freebsd(self): fbsd_distro = self._get_distro('freebsd') @@ -319,4 +318,4 @@ ifconfig_vtnet1="DHCP" defaultrouter="192.168.1.254" ''' self.assertCfgEquals(expected_buf, str(write_buf)) - self.assertEquals(write_buf.mode, 0o644) + self.assertEqual(write_buf.mode, 0o644) diff --git a/tests/unittests/test_distros/test_resolv.py b/tests/unittests/test_distros/test_resolv.py index 9edeb6e7..9402b5ea 100644 --- a/tests/unittests/test_distros/test_resolv.py +++ b/tests/unittests/test_distros/test_resolv.py @@ -1,9 +1,10 @@ from cloudinit.distros.parsers import resolv_conf from cloudinit.distros import rhel_util +from ..helpers import TestCase + import re import tempfile -from ..helpers import TestCase BASE_RESOLVE = ''' @@ -19,7 +20,7 @@ class TestResolvHelper(TestCase): def test_parse_same(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) rp_r = str(rp).strip() - self.assertEquals(BASE_RESOLVE, rp_r) + self.assertEqual(BASE_RESOLVE, rp_r) def test_write_works(self): with tempfile.NamedTemporaryFile() as fh: @@ -27,10 +28,10 @@ class TestResolvHelper(TestCase): def test_local_domain(self): rp = resolv_conf.ResolvConf(BASE_RESOLVE) - self.assertEquals(None, rp.local_domain) + self.assertEqual(None, rp.local_domain) rp.local_domain = "bob" - self.assertEquals('bob', rp.local_domain) + self.assertEqual('bob', rp.local_domain) self.assertIn('domain bob', str(rp)) def test_nameservers(self): @@ -41,7 +42,7 @@ class TestResolvHelper(TestCase): self.assertIn('10.2', rp.nameservers) self.assertIn('nameserver 10.2', str(rp)) self.assertNotIn('10.3', rp.nameservers) - self.assertEquals(len(rp.nameservers), 3) + self.assertEqual(len(rp.nameservers), 3) rp.add_nameserver('10.2') self.assertRaises(ValueError, rp.add_nameserver, '10.3') self.assertNotIn('10.3', rp.nameservers) @@ -55,12 +56,12 @@ class TestResolvHelper(TestCase): self.assertTrue(re.search(r'search(.*)bbb.y.com(.*)', str(rp))) self.assertIn('bbb.y.com', rp.search_domains) rp.add_search_domain('bbb.y.com') - self.assertEquals(len(rp.search_domains), 3) + self.assertEqual(len(rp.search_domains), 3) rp.add_search_domain('bbb2.y.com') - self.assertEquals(len(rp.search_domains), 4) + self.assertEqual(len(rp.search_domains), 4) rp.add_search_domain('bbb3.y.com') - self.assertEquals(len(rp.search_domains), 5) + self.assertEqual(len(rp.search_domains), 5) rp.add_search_domain('bbb4.y.com') - self.assertEquals(len(rp.search_domains), 6) + self.assertEqual(len(rp.search_domains), 6) self.assertRaises(ValueError, rp.add_search_domain, 'bbb5.y.com') - self.assertEquals(len(rp.search_domains), 6) + self.assertEqual(len(rp.search_domains), 6) diff --git a/tests/unittests/test_distros/test_sysconfig.py b/tests/unittests/test_distros/test_sysconfig.py index 03d89a10..8cb55522 100644 --- a/tests/unittests/test_distros/test_sysconfig.py +++ b/tests/unittests/test_distros/test_sysconfig.py @@ -1,6 +1,7 @@ import re from cloudinit.distros.parsers.sys_conf import SysConf + from ..helpers import TestCase @@ -27,34 +28,34 @@ IPV6TO4_ROUTING='eth0-:0004::1/64 eth1-:0005::1/64' ETHTOOL_OPTS="-K ${DEVICE} tso on; -G ${DEVICE} rx 256 tx 256" USEMD5=no''' conf = SysConf(contents.splitlines()) - self.assertEquals(conf['HOSTNAME'], 'blahblah') - self.assertEquals(conf['SHORTDATE'], '$(date +%y:%m:%d:%H:%M)') + self.assertEqual(conf['HOSTNAME'], 'blahblah') + self.assertEqual(conf['SHORTDATE'], '$(date +%y:%m:%d:%H:%M)') # Should be unquoted - self.assertEquals(conf['ETHTOOL_OPTS'], ('-K ${DEVICE} tso on; ' - '-G ${DEVICE} rx 256 tx 256')) - self.assertEquals(contents, str(conf)) + self.assertEqual(conf['ETHTOOL_OPTS'], ('-K ${DEVICE} tso on; ' + '-G ${DEVICE} rx 256 tx 256')) + self.assertEqual(contents, str(conf)) def test_parse_shell_vars(self): contents = 'USESMBAUTH=$XYZ' conf = SysConf(contents.splitlines()) - self.assertEquals(contents, str(conf)) + self.assertEqual(contents, str(conf)) conf = SysConf('') conf['B'] = '${ZZ}d apples' # Should be quoted - self.assertEquals('B="${ZZ}d apples"', str(conf)) + self.assertEqual('B="${ZZ}d apples"', str(conf)) conf = SysConf('') conf['B'] = '$? d apples' - self.assertEquals('B="$? d apples"', str(conf)) + self.assertEqual('B="$? d apples"', str(conf)) contents = 'IPMI_WATCHDOG_OPTIONS="timeout=60"' conf = SysConf(contents.splitlines()) - self.assertEquals('IPMI_WATCHDOG_OPTIONS=timeout=60', str(conf)) + self.assertEqual('IPMI_WATCHDOG_OPTIONS=timeout=60', str(conf)) def test_parse_adjust(self): contents = 'IPV6TO4_ROUTING="eth0-:0004::1/64 eth1-:0005::1/64"' conf = SysConf(contents.splitlines()) # Should be unquoted - self.assertEquals('eth0-:0004::1/64 eth1-:0005::1/64', - conf['IPV6TO4_ROUTING']) + self.assertEqual('eth0-:0004::1/64 eth1-:0005::1/64', + conf['IPV6TO4_ROUTING']) conf['IPV6TO4_ROUTING'] = "blah \tblah" contents2 = str(conf).strip() # Should be requoted due to whitespace @@ -65,12 +66,12 @@ USEMD5=no''' conf = SysConf(''.splitlines()) conf['B'] = ' $(time)' contents = str(conf) - self.assertEquals('B= $(time)', contents) + self.assertEqual('B= $(time)', contents) def test_parse_empty(self): contents = '' conf = SysConf(contents.splitlines()) - self.assertEquals('', str(conf).strip()) + self.assertEqual('', str(conf).strip()) def test_parse_add_new(self): contents = 'BLAH=b' diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 4525f487..a887a930 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -33,20 +33,19 @@ class TestUGNormalize(TestCase): def test_group_dict(self): distro = self._make_distro('ubuntu') - g = {'groups': [ - {'ubuntu': ['foo', 'bar'], - 'bob': 'users'}, - 'cloud-users', - {'bob': 'users2'} - ]} + g = {'groups': + [{'ubuntu': ['foo', 'bar'], + 'bob': 'users'}, + 'cloud-users', + {'bob': 'users2'}]} (_users, groups) = self._norm(g, distro) self.assertIn('ubuntu', groups) ub_members = groups['ubuntu'] - self.assertEquals(sorted(['foo', 'bar']), sorted(ub_members)) + self.assertEqual(sorted(['foo', 'bar']), sorted(ub_members)) self.assertIn('bob', groups) b_members = groups['bob'] - self.assertEquals(sorted(['users', 'users2']), - sorted(b_members)) + self.assertEqual(sorted(['users', 'users2']), + sorted(b_members)) def test_basic_groups(self): distro = self._make_distro('ubuntu') @@ -55,7 +54,7 @@ class TestUGNormalize(TestCase): } (users, groups) = self._norm(ug_cfg, distro) self.assertIn('bob', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_csv_groups(self): distro = self._make_distro('ubuntu') @@ -66,7 +65,7 @@ class TestUGNormalize(TestCase): self.assertIn('bob', groups) self.assertIn('joe', groups) self.assertIn('steve', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_more_groups(self): distro = self._make_distro('ubuntu') @@ -77,7 +76,7 @@ class TestUGNormalize(TestCase): self.assertIn('bob', groups) self.assertIn('joe', groups) self.assertIn('steve', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_member_groups(self): distro = self._make_distro('ubuntu') @@ -90,11 +89,11 @@ class TestUGNormalize(TestCase): } (users, groups) = self._norm(ug_cfg, distro) self.assertIn('bob', groups) - self.assertEquals(['s'], groups['bob']) - self.assertEquals([], groups['joe']) + self.assertEqual(['s'], groups['bob']) + self.assertEqual([], groups['joe']) self.assertIn('joe', groups) self.assertIn('steve', groups) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_users_simple_dict(self): distro = self._make_distro('ubuntu', bcfg) @@ -128,14 +127,14 @@ class TestUGNormalize(TestCase): } } (users, _groups) = self._norm(ug_cfg, distro) - self.assertEquals({}, users) + self.assertEqual({}, users) ug_cfg = { 'users': { 'default': 'no', } } (users, _groups) = self._norm(ug_cfg, distro) - self.assertEquals({}, users) + self.assertEqual({}, users) def test_users_simple_csv(self): distro = self._make_distro('ubuntu') @@ -145,8 +144,8 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) def test_users_simple(self): distro = self._make_distro('ubuntu') @@ -159,8 +158,8 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) def test_users_old_user(self): distro = self._make_distro('ubuntu', bcfg) @@ -211,8 +210,8 @@ class TestUGNormalize(TestCase): self.assertIn('zetta', users) ug_cfg = {} (users, groups) = self._norm(ug_cfg, distro) - self.assertEquals({}, users) - self.assertEquals({}, groups) + self.assertEqual({}, users) + self.assertEqual({}, groups) def test_users_dict_default_additional(self): distro = self._make_distro('ubuntu', bcfg) @@ -223,12 +222,10 @@ class TestUGNormalize(TestCase): } (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) - self.assertEquals(",".join(distro.get_default_user()['groups']), - users['bob']['groups']) - self.assertEquals(True, - users['bob']['blah']) - self.assertEquals(True, - users['bob']['default']) + self.assertEqual(",".join(distro.get_default_user()['groups']), + users['bob']['groups']) + self.assertEqual(True, users['bob']['blah']) + self.assertEqual(True, users['bob']['default']) def test_users_dict_extract(self): distro = self._make_distro('ubuntu', bcfg) @@ -240,7 +237,7 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) (name, config) = distros.extract_default(users) - self.assertEquals(name, 'bob') + self.assertEqual(name, 'bob') expected_config = {} def_config = None try: @@ -255,7 +252,7 @@ class TestUGNormalize(TestCase): expected_config.pop('name', None) expected_config.pop('groups', None) config.pop('groups', None) - self.assertEquals(config, expected_config) + self.assertEqual(config, expected_config) def test_users_dict_default(self): distro = self._make_distro('ubuntu', bcfg) @@ -266,10 +263,9 @@ class TestUGNormalize(TestCase): } (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) - self.assertEquals(",".join(distro.get_default_user()['groups']), - users['bob']['groups']) - self.assertEquals(True, - users['bob']['default']) + self.assertEqual(",".join(distro.get_default_user()['groups']), + users['bob']['groups']) + self.assertEqual(True, users['bob']['default']) def test_users_dict_trans(self): distro = self._make_distro('ubuntu') @@ -283,8 +279,8 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'tr_me': True, 'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'tr_me': True, 'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) def test_users_dict(self): distro = self._make_distro('ubuntu') @@ -297,5 +293,5 @@ class TestUGNormalize(TestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'default': False}, users['joe']) - self.assertEquals({'default': False}, users['bob']) + self.assertEqual({'default': False}, users['joe']) + self.assertEqual({'default': False}, users['bob']) diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index 99fc54be..d6cf17fa 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -16,7 +16,7 @@ class TestEc2Util(helpers.HttprettyTestCase): body='stuff', status=200) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('stuff', userdata.decode('utf-8')) + self.assertEqual('stuff', userdata.decode('utf-8')) @hp.activate def test_userdata_fetch_fail_not_found(self): @@ -24,7 +24,7 @@ class TestEc2Util(helpers.HttprettyTestCase): 'http://169.254.169.254/%s/user-data' % (self.VERSION), status=404) userdata = eu.get_instance_userdata(self.VERSION, retries=0) - self.assertEquals('', userdata) + self.assertEqual('', userdata) @hp.activate def test_userdata_fetch_fail_server_dead(self): @@ -32,7 +32,7 @@ class TestEc2Util(helpers.HttprettyTestCase): 'http://169.254.169.254/%s/user-data' % (self.VERSION), status=500) userdata = eu.get_instance_userdata(self.VERSION, retries=0) - self.assertEquals('', userdata) + self.assertEqual('', userdata) @hp.activate def test_userdata_fetch_fail_server_not_found(self): @@ -40,7 +40,7 @@ class TestEc2Util(helpers.HttprettyTestCase): 'http://169.254.169.254/%s/user-data' % (self.VERSION), status=404) userdata = eu.get_instance_userdata(self.VERSION) - self.assertEquals('', userdata) + self.assertEqual('', userdata) @hp.activate def test_metadata_fetch_no_keys(self): @@ -56,9 +56,9 @@ class TestEc2Util(helpers.HttprettyTestCase): hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'), status=200, body='1') md = eu.get_instance_metadata(self.VERSION, retries=0) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') - self.assertEquals(md['ami-launch-index'], '1') + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') + self.assertEqual(md['ami-launch-index'], '1') @hp.activate def test_metadata_fetch_key(self): @@ -77,9 +77,9 @@ class TestEc2Util(helpers.HttprettyTestCase): uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') - self.assertEquals(1, len(md['public-keys'])) + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') + self.assertEqual(1, len(md['public-keys'])) @hp.activate def test_metadata_fetch_with_2_keys(self): @@ -102,9 +102,9 @@ class TestEc2Util(helpers.HttprettyTestCase): uh.combine_url(base_url, 'public-keys/1/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-other-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') - self.assertEquals(2, len(md['public-keys'])) + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') + self.assertEqual(2, len(md['public-keys'])) @hp.activate def test_metadata_fetch_bdm(self): @@ -131,9 +131,9 @@ class TestEc2Util(helpers.HttprettyTestCase): status=200, body="sdc") md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) - self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') - self.assertEquals(md['instance-id'], '123') + self.assertEqual(md['hostname'], 'ec2.fake.host.name.com') + self.assertEqual(md['instance-id'], '123') bdm = md['block-device-mapping'] - self.assertEquals(2, len(bdm)) - self.assertEquals(bdm['ami'], 'sdb') - self.assertEquals(bdm['ephemeral0'], 'sdc') + self.assertEqual(2, len(bdm)) + self.assertEqual(bdm['ami'], 'sdb') + self.assertEqual(bdm['ephemeral0'], 'sdc') diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 95d24b9b..395713e6 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -25,7 +25,7 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): for (index, count) in expected_counts.items(): index = util.safe_int(index) filtered_message = launch_index.Filter(index).apply(message) - self.assertEquals(count_messages(filtered_message), count) + self.assertEqual(count_messages(filtered_message), count) # Ensure original message still ok/not modified self.assertTrue(self.equivalentMessage(message, orig_message)) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 1ed185ca..d1dca2c4 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -1,6 +1,6 @@ +from cloudinit.config import cc_apt_configure from cloudinit import util -from cloudinit.config import cc_apt_configure from ..helpers import TestCase import os diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py new file mode 100644 index 00000000..acde0863 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -0,0 +1,180 @@ +""" test_handler_apt_configure_sources_list +Test templating of sources list +""" +import logging +import os +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import templater +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from cloudinit.distros.debian import Distro + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + +YAML_TEXT_CUSTOM_SL = """ +apt_mirror: http://archive.ubuntu.com/ubuntu/ +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + # FIND_SOMETHING_SPECIAL +""" + +EXPECTED_CONVERTED_CONTENT = ( + """## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +# FIND_SOMETHING_SPECIAL +""") + + +def load_tfile_or_url(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """TestAptSourceConfigSourceList + Main Class to test sources list rendering + """ + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def apt_source_list(self, distro, mirror, mirrorcheck=None): + """apt_source_list + Test rendering of a source.list from template for a given distro + """ + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} + mycloud = self._get_cloud(distro) + + with mock.patch.object(templater, 'render_to_file') as mocktmpl: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + with mock.patch.object(util, 'rename'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocktmpl.assert_called_once_with( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro), + '/etc/apt/sources.list', + {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) + + def test_apt_source_list_debian(self): + """Test rendering of a source.list from template for debian""" + self.apt_source_list('debian', 'http://httpredir.debian.org/debian') + + def test_apt_source_list_ubuntu(self): + """Test rendering of a source.list from template for ubuntu""" + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + @staticmethod + def myresolve(name): + """Fake util.is_resolvable for mirrorfail tests""" + if name == "does.not.exist": + print("Faking FAIL for '%s'" % name) + return False + else: + print("Faking SUCCESS for '%s'" % name) + return True + + def test_apt_srcl_debian_mirrorfail(self): + """Test rendering of a source.list from template for debian""" + with mock.patch.object(util, 'is_resolvable', + side_effect=self.myresolve) as mockresolve: + self.apt_source_list('debian', + ['http://does.not.exist', + 'http://httpredir.debian.org/debian'], + 'http://httpredir.debian.org/debian') + mockresolve.assert_any_call("does.not.exist") + mockresolve.assert_any_call("httpredir.debian.org") + + def test_apt_srcl_ubuntu_mirrorfail(self): + """Test rendering of a source.list from template for ubuntu""" + with mock.patch.object(util, 'is_resolvable', + side_effect=self.myresolve) as mockresolve: + self.apt_source_list('ubuntu', + ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + mockresolve.assert_any_call("does.not.exist") + mockresolve.assert_any_call("archive.ubuntu.com") + + def test_apt_srcl_custom(self): + """Test rendering from a custom source.list template""" + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(cc_apt_configure, 'get_release', + return_value='fakerelease'): + with mock.patch.object(Distro, 'get_primary_arch', + return_value='amd64'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockwrite.assert_called_once_with( + '/etc/apt/sources.list', + EXPECTED_CONVERTED_CONTENT, + mode=420) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py new file mode 100644 index 00000000..99a4d860 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -0,0 +1,516 @@ +""" test_handler_apt_source +Testing various config variations of the apt_source config +""" +import os +import re +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit.config import cc_apt_configure +from cloudinit import gpg +from cloudinit import util + +from ..helpers import TestCase + +EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" + + +def load_tfile_or_url(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfig(TestCase): + """TestAptSourceConfig + Main Class to test apt_source configs + """ + release = "fantastic" + + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") + self.join = os.path.join + # mock fallback filename into writable tmp dir + self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") + get_rel = patcher.start() + get_rel.return_value = self.release + self.addCleanup(patcher.stop) + + @staticmethod + def _get_default_params(): + """get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = cc_apt_configure.get_release() + params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" + return params + + def myjoin(self, *args, **kwargs): + """myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + def apt_src_basic(self, filename, cfg): + """apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + + cc_apt_configure.add_apt_sources(cfg, params) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic(self): + """Test deb source string, overwrite mirror and filename""" + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + self.apt_src_basic(self.aptlistfile, [cfg]) + + def test_apt_src_basic_dict(self): + """Test deb source string, overwrite mirror and filename (dict)""" + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self.apt_src_basic(self.aptlistfile, cfg) + + def apt_src_basic_tri(self, cfg): + """apt_src_basic_tri + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + generic part to check three files with different content + """ + self.apt_src_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic_tri(self): + """Test Fix three deb source string with filenames""" + cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile3} + self.apt_src_basic_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_basic_dict_tri(self): + """Test Fix three deb source string with filenames (dict)""" + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self.apt_src_basic_tri(cfg) + + def test_apt_src_basic_nofn(self): + """Test Fix three deb source string without filenames (dict)""" + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_basic(self.fallbackfn, [cfg]) + + def apt_src_replacement(self, filename, cfg): + """apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + cc_apt_configure.add_apt_sources(cfg, params) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace(self): + """Test Autoreplacement of MIRROR and RELEASE in source specs""" + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + self.apt_src_replacement(self.aptlistfile, [cfg]) + + def apt_src_replace_tri(self, cfg): + """apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + generic part + """ + self.apt_src_replacement(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace_tri(self): + """Test triple Autoreplacement of MIRROR and RELEASE in source specs""" + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_replace_dict_tri(self): + """Test triple Autoreplacement in source specs (dict)""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self.apt_src_replace_tri(cfg) + + def test_apt_src_replace_nofn(self): + """Test Autoreplacement of MIRROR and RELEASE in source specs nofile""" + cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_replacement(self.fallbackfn, [cfg]) + + def apt_src_keyid(self, filename, cfg, keynum): + """apt_src_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.add_apt_sources(cfg, params) + + # check if it added the right ammount of keys + calls = [] + for _ in range(keynum): + calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) + mockobj.assert_has_calls(calls, any_order=True) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid(self): + """Test specification of a source + keyid with filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + self.apt_src_keyid(self.aptlistfile, [cfg], 1) + + def test_apt_src_keyid_tri(self): + """Test 3x specification of a source + keyid with filename being set""" + cfg1 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77", + 'filename': self.aptlistfile3} + + self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid_nofn(self): + """Test specification of a source + keyid without filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_keyid(self.fallbackfn, [cfg], 1) + + def apt_src_key(self, filename, cfg): + """apt_src_key + Test specification of a source + key + """ + params = self._get_default_params() + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_apt_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_key(self): + """Test specification of a source + key with filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321", + 'filename': self.aptlistfile} + self.apt_src_key(self.aptlistfile, cfg) + + def test_apt_src_key_nofn(self): + """Test specification of a source + key without filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_key(self.fallbackfn, cfg) + + def test_apt_src_keyonly(self): + """Test specifying key without source""" + params = self._get_default_params() + cfg = {'key': "fakekey 4242", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_apt_sources([cfg], params) + + mockobj.assert_called_once_with(('apt-key', 'add', '-'), + 'fakekey 4242') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyidonly(self): + """Test specification of a keyid without source""" + params = self._get_default_params() + cfg = {'keyid': "03683F77", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + cc_apt_configure.add_apt_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def apt_src_keyid_real(self, cfg, expectedkey): + """apt_src_keyid_real + Test specification of a keyid without source including + up to addition of the key (add_apt_key_raw mocked to keep the + environment as is) + """ + params = self._get_default_params() + + with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: + with mock.patch.object(gpg, 'get_key_by_id', + return_value=expectedkey) as mockgetkey: + cc_apt_configure.add_apt_sources([cfg], params) + + mockgetkey.assert_called_with(cfg['keyid'], + cfg.get('keyserver', + 'keyserver.ubuntu.com')) + mockkey.assert_called_with(expectedkey) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyid_real(self): + """test_apt_src_keyid_real - Test keyid including key add""" + keyid = "03683F77" + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_longkeyid_real(self): + """test_apt_src_longkeyid_real - Test long keyid including key add""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_longkeyid_ks_real(self): + """test_apt_src_longkeyid_ks_real - Test long keyid from other ks""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {'keyid': keyid, + 'keyserver': 'keys.gnupg.net', + 'filename': self.aptlistfile} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_ppa(self): + """Test adding a ppa""" + params = self._get_default_params() + cfg = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_apt_sources([cfg], params, + aa_repo_match=matcher) + mockobj.assert_called_once_with(['add-apt-repository', + 'ppa:smoser/cloud-init-test']) + + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_ppa_tri(self): + """Test adding three ppa's""" + params = self._get_default_params() + cfg1 = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + cfg2 = {'source': 'ppa:smoser/cloud-init-test2', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'ppa:smoser/cloud-init-test3', + 'filename': self.aptlistfile3} + + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params, + aa_repo_match=matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) + + def test_convert_to_new_format(self): + """Test the conversion of old to new format""" + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, + 'source': 'deb $MIRROR $RELEASE ' + 'multiverse'}, + self.aptlistfile2: {'filename': self.aptlistfile2, + 'source': 'deb $MIRROR $RELEASE main'}, + self.aptlistfile3: {'filename': self.aptlistfile3, + 'source': 'deb $MIRROR $RELEASE ' + 'universe'}} + + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) + self.assertEqual(newcfg, checkcfg) + + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) + self.assertEqual(newcfg2, checkcfg) + + with self.assertRaises(ValueError): + cc_apt_configure.convert_to_new_format(5) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_ca_certs.py b/tests/unittests/test_handler/test_handler_ca_certs.py index a6b9c0fd..5e771731 100644 --- a/tests/unittests/test_handler/test_handler_ca_certs.py +++ b/tests/unittests/test_handler/test_handler_ca_certs.py @@ -1,8 +1,8 @@ from cloudinit import cloud +from cloudinit.config import cc_ca_certs from cloudinit import helpers from cloudinit import util -from cloudinit.config import cc_ca_certs from ..helpers import TestCase import logging @@ -176,8 +176,7 @@ class TestAddCaCerts(TestCase): mock_write.assert_has_calls([ mock.call("/usr/share/ca-certificates/cloud-init-ca-certs.crt", cert, mode=0o644), - mock.call("/etc/ca-certificates.conf", expected, omode="wb"), - ]) + mock.call("/etc/ca-certificates.conf", expected, omode="wb")]) mock_load.assert_called_once_with("/etc/ca-certificates.conf") def test_single_cert_no_trailing_cr(self): @@ -202,8 +201,7 @@ class TestAddCaCerts(TestCase): mock.call("/etc/ca-certificates.conf", "%s\n%s\n" % (ca_certs_content, "cloud-init-ca-certs.crt"), - omode="wb"), - ]) + omode="wb")]) mock_load.assert_called_once_with("/etc/ca-certificates.conf") @@ -228,8 +226,7 @@ class TestAddCaCerts(TestCase): mock.call("/etc/ca-certificates.conf", "%s\n%s\n" % (ca_certs_content, "cloud-init-ca-certs.crt"), - omode='wb'), - ]) + omode='wb')]) mock_load.assert_called_once_with("/etc/ca-certificates.conf") @@ -264,8 +261,7 @@ class TestRemoveDefaultCaCerts(TestCase): mock_delete.assert_has_calls([ mock.call("/usr/share/ca-certificates/"), - mock.call("/etc/ssl/certs/"), - ]) + mock.call("/etc/ssl/certs/")]) mock_write.assert_called_once_with( "/etc/ca-certificates.conf", "", mode=0o644) diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 7763f23b..7a1bc317 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,21 +1,19 @@ import json +import logging import os - -from cloudinit.config import cc_chef +import shutil +import six +import tempfile from cloudinit import cloud +from cloudinit.config import cc_chef from cloudinit import distros from cloudinit import helpers -from cloudinit import util from cloudinit.sources import DataSourceNone +from cloudinit import util from .. import helpers as t_help -import six -import logging -import shutil -import tempfile - LOG = logging.getLogger(__name__) CLIENT_TEMPL = os.path.sep.join(["templates", "chef_client.rb.tmpl"]) diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index bef0d80d..e653488a 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -1,7 +1,7 @@ from cloudinit import cloud +from cloudinit.config import cc_growpart from cloudinit import util -from cloudinit.config import cc_growpart from ..helpers import TestCase import errno diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index de85eff6..c91908f4 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -64,4 +64,4 @@ class TestLocale(t_help.FilesystemMockingTestCase): contents = util.load_file('/etc/sysconfig/language', decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEquals({'RC_LANG': cfg['locale']}, dict(n_cfg)) + self.assertEqual({'RC_LANG': cfg['locale']}, dict(n_cfg)) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 5f61ba6a..6f90defb 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -1,6 +1,6 @@ from cloudinit.config import cc_lxd -from cloudinit import (distros, helpers, cloud) from cloudinit.sources import DataSourceNoCloud +from cloudinit import (distros, helpers, cloud) from .. import helpers as t_help import logging @@ -42,11 +42,11 @@ class TestLxd(t_help.TestCase): cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) self.assertTrue(mock_util.which.called) init_call = mock_util.subp.call_args_list[0][0][0] - self.assertEquals(init_call, - ['lxd', 'init', '--auto', - '--network-address=0.0.0.0', - '--storage-backend=zfs', - '--storage-pool=poolname']) + self.assertEqual(init_call, + ['lxd', 'init', '--auto', + '--network-address=0.0.0.0', + '--storage-backend=zfs', + '--storage-pool=poolname']) @mock.patch("cloudinit.config.cc_lxd.util") def test_lxd_install(self, mock_util): @@ -56,7 +56,7 @@ class TestLxd(t_help.TestCase): cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) self.assertTrue(cc.distro.install_packages.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] - self.assertEquals(sorted(install_pkg), ['lxd', 'zfs']) + self.assertEqual(sorted(install_pkg), ['lxd', 'zfs']) @mock.patch("cloudinit.config.cc_lxd.util") def test_no_init_does_nothing(self, mock_util): @@ -87,7 +87,7 @@ class TestLxd(t_help.TestCase): "ipv6_netmask": "64", "ipv6_nat": "true", "domain": "lxd"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "true", "lxd/bridge-name": "testbr0", @@ -109,7 +109,7 @@ class TestLxd(t_help.TestCase): "ipv6_address": "fd98:9e0:3744::1", "ipv6_netmask": "64", "ipv6_nat": "true"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "true", "lxd/bridge-ipv6": "true", @@ -120,7 +120,7 @@ class TestLxd(t_help.TestCase): def test_lxd_debconf_existing(self): data = {"mode": "existing", "name": "testbr0"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "false", "lxd/use-existing-bridge": "true", @@ -128,7 +128,7 @@ class TestLxd(t_help.TestCase): def test_lxd_debconf_none(self): data = {"mode": "none"} - self.assertEquals( + self.assertEqual( cc_lxd.bridge_to_debconf(data), {"lxd/setup-bridge": "false", "lxd/bridge-name": ""}) diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 04ce5687..feff319d 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -119,7 +119,7 @@ def check_lps_ret(psc_return, mode=None): try: float(timeout) - except: + except Exception: errs.append("timeout failed convert to float") if len(errs): diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index b932165c..38636063 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -31,7 +31,7 @@ class TestLoadConfig(t_help.TestCase): 'config_dir': "mydir", 'config_filename': 'myfilename', 'service_reload_command': 'auto'} - ) + ) self.assertEqual(found, self.basecfg) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 98bc9b81..a0390da9 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -92,7 +92,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("tiny-tim-was-here", contents) + self.assertEqual("tiny-tim-was-here", contents) def test_append_random_unknown_encoding(self): data = self._compress(b"tiny-toe") @@ -117,7 +117,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("tiny-toe", contents) + self.assertEqual("tiny-toe", contents) def test_append_random_gz(self): data = self._compress(b"big-toe") @@ -130,7 +130,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("big-toe", contents) + self.assertEqual("big-toe", contents) def test_append_random_base64(self): data = util.b64e('bubbles') @@ -143,7 +143,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("bubbles", contents) + self.assertEqual("bubbles", contents) def test_append_random_b64(self): data = util.b64e('kit-kat') @@ -156,7 +156,7 @@ class TestRandomSeed(t_help.TestCase): } cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals("kit-kat", contents) + self.assertEqual("kit-kat", contents) def test_append_random_metadata(self): cfg = { @@ -168,7 +168,7 @@ class TestRandomSeed(t_help.TestCase): c = self._get_cloud('ubuntu', {'random_seed': '-so-was-josh'}) cc_seed_random.handle('test', cfg, c, LOG, []) contents = util.load_file(self._seed_file) - self.assertEquals('tiny-tim-was-here-so-was-josh', contents) + self.assertEqual('tiny-tim-was-here-so-was-josh', contents) def test_seed_command_provided_and_available(self): c = self._get_cloud('ubuntu', {}) diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index d358b069..7effa124 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -7,13 +7,11 @@ from cloudinit import util from .. import helpers as t_help -import shutil -import tempfile +from configobj import ConfigObj import logging - +import shutil from six import BytesIO - -from configobj import ConfigObj +import tempfile LOG = logging.getLogger(__name__) @@ -43,8 +41,8 @@ class TestHostname(t_help.FilesystemMockingTestCase): if not distro.uses_systemd(): contents = util.load_file("/etc/sysconfig/network", decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEquals({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, - dict(n_cfg)) + self.assertEqual({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, + dict(n_cfg)) def test_write_hostname_debian(self): cfg = { @@ -58,7 +56,7 @@ class TestHostname(t_help.FilesystemMockingTestCase): cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) contents = util.load_file("/etc/hostname") - self.assertEquals('blah', contents.strip()) + self.assertEqual('blah', contents.strip()) def test_write_hostname_sles(self): cfg = { @@ -71,4 +69,4 @@ class TestHostname(t_help.FilesystemMockingTestCase): self.patchUtils(self.tmp) cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) contents = util.load_file("/etc/HOSTNAME") - self.assertEquals('blah', contents.strip()) + self.assertEqual('blah', contents.strip()) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 8aeff53c..57dce1bc 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -1,6 +1,7 @@ from cloudinit.config.cc_snappy import ( makeop, get_package_ops, render_snap_op) from cloudinit import util + from .. import helpers as t_help import os diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index e3df8759..b7e6b03d 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -28,12 +28,10 @@ from cloudinit.sources import DataSourceNoCloud from .. import helpers as t_help from configobj import ConfigObj - -from six import BytesIO - +import logging import shutil +from six import BytesIO import tempfile -import logging LOG = logging.getLogger(__name__) @@ -72,7 +70,7 @@ class TestTimezone(t_help.FilesystemMockingTestCase): contents = util.load_file('/etc/sysconfig/clock', decode=False) n_cfg = ConfigObj(BytesIO(contents)) - self.assertEquals({'TIMEZONE': cfg['timezone']}, dict(n_cfg)) + self.assertEqual({'TIMEZONE': cfg['timezone']}, dict(n_cfg)) contents = util.load_file('/etc/localtime') - self.assertEquals(dummy_contents, contents.strip()) + self.assertEqual(dummy_contents, contents.strip()) diff --git a/tests/unittests/test_handler/test_handler_write_files.py b/tests/unittests/test_handler/test_handler_write_files.py index f1c7f7b4..466e45f8 100644 --- a/tests/unittests/test_handler/test_handler_write_files.py +++ b/tests/unittests/test_handler/test_handler_write_files.py @@ -1,6 +1,6 @@ -from cloudinit import util -from cloudinit import log as logging from cloudinit.config.cc_write_files import write_files +from cloudinit import log as logging +from cloudinit import util from ..helpers import FilesystemMockingTestCase diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 3a8aa7c1..28b060f8 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -1,16 +1,13 @@ -from cloudinit import util - from cloudinit.config import cc_yum_add_repo +from cloudinit import util from .. import helpers -import shutil -import tempfile +import configobj import logging - +import shutil from six import BytesIO - -import configobj +import tempfile LOG = logging.getLogger(__name__) @@ -68,4 +65,4 @@ class TestConfig(helpers.FilesystemMockingTestCase): 'gpgcheck': '1', } } - self.assertEquals(expected, dict(contents)) + self.assertEqual(expected, dict(contents)) diff --git a/tests/unittests/test_helpers.py b/tests/unittests/test_helpers.py new file mode 100644 index 00000000..943a5723 --- /dev/null +++ b/tests/unittests/test_helpers.py @@ -0,0 +1,33 @@ +"""Tests of the built-in user data handlers.""" + +import os + +from . import helpers as test_helpers + +from cloudinit import sources + + +class MyDataSource(sources.DataSource): + _instance_id = None + + def get_instance_id(self): + return self._instance_id + + +class TestPaths(test_helpers.ResourceUsingTestCase): + def test_get_ipath_and_instance_id_with_slashes(self): + myds = MyDataSource(sys_cfg={}, distro=None, paths={}) + myds._instance_id = "/foo/bar" + safe_iid = "_foo_bar" + mypaths = self.getCloudPaths(myds) + + self.assertEqual( + os.path.join(mypaths.cloud_dir, 'instances', safe_iid), + mypaths.get_ipath()) + + def test_get_ipath_and_empty_instance_id_returns_none(self): + myds = MyDataSource(sys_cfg={}, distro=None, paths={}) + myds._instance_id = None + mypaths = self.getCloudPaths(myds) + + self.assertEqual(None, mypaths.get_ipath()) diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 976d8283..a33ec184 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -125,15 +125,15 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): def test_seed_runs(self): test_dicts = [] - for i in range(1, 50): + for i in range(1, 10): base_dicts = [] - for j in range(1, 50): + for j in range(1, 10): base_dicts.append(make_dict(5, i * j)) test_dicts.append(base_dicts) for test in test_dicts: c = _old_mergemanydict(*test) d = util.mergemanydict(test) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_merge_cc_samples(self): tests = self._load_merge_files() @@ -155,7 +155,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): fail_msg = fail_msg % (expected_fn, ",".join(merging_fns), merged_buf, expected_merge) - self.assertEquals(expected_merge, merged_buf, msg=fail_msg) + self.assertEqual(expected_merge, merged_buf, msg=fail_msg) def test_compat_merges_dict(self): a = { @@ -167,7 +167,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merges_dict2(self): a = { @@ -182,7 +182,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merges_list(self): a = {'b': [1, 2, 3]} @@ -190,7 +190,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): c = {'b': [6, 7]} e = _old_mergemanydict(a, b, c) f = util.mergemanydict([a, b, c]) - self.assertEquals(e, f) + self.assertEqual(e, f) def test_compat_merges_str(self): a = {'b': "hi"} @@ -198,7 +198,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): c = {'b': "hallo"} e = _old_mergemanydict(a, b, c) f = util.mergemanydict([a, b, c]) - self.assertEquals(e, f) + self.assertEqual(e, f) def test_compat_merge_sub_dict(self): a = { @@ -222,7 +222,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merge_sub_dict2(self): a = { @@ -238,7 +238,7 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) def test_compat_merge_sub_list(self): a = { @@ -254,4 +254,4 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): } c = _old_mergedict(a, b) d = util.mergemanydict([a, b]) - self.assertEquals(c, d) + self.assertEqual(c, d) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index 09235c4d..3ae00fc6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -1,13 +1,22 @@ -from cloudinit import util from cloudinit import net +from cloudinit.net import cmdline +from cloudinit.net import eni +from cloudinit.net import network_state +from cloudinit.net import sysconfig +from cloudinit.sources.helpers import openstack +from cloudinit import util + +from .helpers import mock from .helpers import TestCase import base64 import copy -import io import gzip +import io import json import os +import shutil +import tempfile DHCP_CONTENT_1 = """ DEVICE='eth0' @@ -67,22 +76,214 @@ STATIC_EXPECTED_1 = { 'dns_nameservers': ['10.0.1.1']}], } +# Examples (and expected outputs for various renderers). +OS_SAMPLES = [ + { + 'in_data': { + "services": [{"type": "dns", "address": "172.19.0.12"}], + "networks": [{ + "network_id": "dacd568d-5be6-4786-91fe-750c374b78b4", + "type": "ipv4", "netmask": "255.255.252.0", + "link": "tap1a81968a-79", + "routes": [{ + "netmask": "0.0.0.0", + "network": "0.0.0.0", + "gateway": "172.19.3.254", + }], + "ip_address": "172.19.1.34", "id": "network0" + }], + "links": [ + { + "ethernet_mac_address": "fa:16:3e:ed:9a:59", + "mtu": None, "type": "bridge", "id": + "tap1a81968a-79", + "vif_id": "1a81968a-797a-400f-8a80-567f997eb93f" + }, + ], + }, + 'in_macs': { + 'fa:16:3e:ed:9a:59': 'eth0', + }, + 'out_sysconfig': [ + ('etc/sysconfig/network-scripts/ifcfg-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=static +DEFROUTE=yes +DEVICE=eth0 +GATEWAY=172.19.3.254 +HWADDR=fa:16:3e:ed:9a:59 +IPADDR=172.19.1.34 +NETMASK=255.255.252.0 +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip()), + ('etc/sysconfig/network-scripts/route-eth0', + """ +# Created by cloud-init on instance boot automatically, do not edit. +# +ADDRESS0=0.0.0.0 +GATEWAY0=172.19.3.254 +NETMASK0=0.0.0.0 +""".lstrip()), + ('etc/resolv.conf', + """ +; Created by cloud-init on instance boot automatically, do not edit. +; +nameserver 172.19.0.12 +""".lstrip()), + ('etc/udev/rules.d/70-persistent-net.rules', + "".join(['SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ', + 'ATTR{address}=="fa:16:3e:ed:9a:59", NAME="eth0"\n']))] + } +] + + +def _setup_test(tmp_dir, mock_get_devicelist, mock_sys_netdev_info, + mock_sys_dev_path): + mock_get_devicelist.return_value = ['eth1000'] + dev_characteristics = { + 'eth1000': { + "bridge": False, + "carrier": False, + "dormant": False, + "operstate": "down", + "address": "07-1C-C6-75-A4-BE", + } + } + + def netdev_info(name, field): + return dev_characteristics[name][field] + + mock_sys_netdev_info.side_effect = netdev_info + + def sys_dev_path(devname, path=""): + return tmp_dir + devname + "/" + path + + for dev in dev_characteristics: + os.makedirs(os.path.join(tmp_dir, dev)) + with open(os.path.join(tmp_dir, dev, 'operstate'), 'w') as fh: + fh.write("down") + + mock_sys_dev_path.side_effect = sys_dev_path + + +class TestSysConfigRendering(TestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.sys_netdev_info") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_sys_netdev_info, + mock_sys_dev_path): + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + _setup_test(tmp_dir, mock_get_devicelist, + mock_sys_netdev_info, mock_sys_dev_path) + + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + + render_file = 'etc/sysconfig/network-scripts/ifcfg-eth1000' + with open(os.path.join(render_dir, render_file)) as fh: + content = fh.read() + expected_content = """ +# Created by cloud-init on instance boot automatically, do not edit. +# +BOOTPROTO=dhcp +DEVICE=eth1000 +HWADDR=07-1C-C6-75-A4-BE +NM_CONTROLLED=no +ONBOOT=yes +TYPE=Ethernet +USERCTL=no +""".lstrip() + self.assertEqual(expected_content, content) + + def test_openstack_rendering_samples(self): + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + render_dir = os.path.join(tmp_dir, "render") + for os_sample in OS_SAMPLES: + ex_input = os_sample['in_data'] + ex_mac_addrs = os_sample['in_macs'] + network_cfg = openstack.convert_net_json( + ex_input, known_macs=ex_mac_addrs) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + renderer = sysconfig.Renderer() + renderer.render_network_state(render_dir, ns) + for fn, expected_content in os_sample.get('out_sysconfig', []): + with open(os.path.join(render_dir, fn)) as fh: + self.assertEqual(expected_content, fh.read()) + + +class TestEniNetRendering(TestCase): + + @mock.patch("cloudinit.net.sys_dev_path") + @mock.patch("cloudinit.net.sys_netdev_info") + @mock.patch("cloudinit.net.get_devicelist") + def test_default_generation(self, mock_get_devicelist, + mock_sys_netdev_info, + mock_sys_dev_path): + tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp_dir) + _setup_test(tmp_dir, mock_get_devicelist, + mock_sys_netdev_info, mock_sys_dev_path) + + network_cfg = net.generate_fallback_config() + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) + + render_dir = os.path.join(tmp_dir, "render") + os.makedirs(render_dir) + + renderer = eni.Renderer( + {'links_path_prefix': None, + 'eni_path': 'interfaces', 'netrules_path': None, + }) + renderer.render_network_state(render_dir, ns) + + self.assertTrue(os.path.exists(os.path.join(render_dir, + 'interfaces'))) + with open(os.path.join(render_dir, 'interfaces')) as fh: + contents = fh.read() + + expected = """ +auto lo +iface lo inet loopback + +auto eth1000 +iface eth1000 inet dhcp +""" + self.assertEqual(expected.lstrip(), contents.lstrip()) + -class TestNetConfigParsing(TestCase): +class TestCmdlineConfigParsing(TestCase): simple_cfg = { 'config': [{"type": "physical", "name": "eth0", "mac_address": "c0:d6:9f:2c:e8:80", "subnets": [{"type": "dhcp"}]}]} - def test_klibc_convert_dhcp(self): - found = net._klibc_to_config_entry(DHCP_CONTENT_1) + def test_cmdline_convert_dhcp(self): + found = cmdline._klibc_to_config_entry(DHCP_CONTENT_1) self.assertEqual(found, ('eth0', DHCP_EXPECTED_1)) - def test_klibc_convert_static(self): - found = net._klibc_to_config_entry(STATIC_CONTENT_1) + def test_cmdline_convert_static(self): + found = cmdline._klibc_to_config_entry(STATIC_CONTENT_1) self.assertEqual(found, ('eth1', STATIC_EXPECTED_1)) - def test_config_from_klibc_net_cfg(self): + def test_config_from_cmdline_net_cfg(self): files = [] pairs = (('net-eth0.cfg', DHCP_CONTENT_1), ('net-eth1.cfg', STATIC_CONTENT_1)) @@ -103,21 +304,22 @@ class TestNetConfigParsing(TestCase): files.append(fp) util.write_file(fp, content) - found = net.config_from_klibc_net_cfg(files=files, mac_addrs=macs) + found = cmdline.config_from_klibc_net_cfg(files=files, + mac_addrs=macs) self.assertEqual(found, expected) def test_cmdline_with_b64(self): data = base64.b64encode(json.dumps(self.simple_cfg).encode()) encoded_text = data.decode() - cmdline = 'ro network-config=' + encoded_text + ' root=foo' - found = net.read_kernel_cmdline_config(cmdline=cmdline) + raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline) self.assertEqual(found, self.simple_cfg) def test_cmdline_with_b64_gz(self): data = _gzip_data(json.dumps(self.simple_cfg).encode()) encoded_text = base64.b64encode(data).decode() - cmdline = 'ro network-config=' + encoded_text + ' root=foo' - found = net.read_kernel_cmdline_config(cmdline=cmdline) + raw_cmdline = 'ro network-config=' + encoded_text + ' root=foo' + found = cmdline.read_kernel_cmdline_config(cmdline=raw_cmdline) self.assertEqual(found, self.simple_cfg) diff --git a/tests/unittests/test_reporting.py b/tests/unittests/test_reporting.py index 32356ef9..20ca23df 100644 --- a/tests/unittests/test_reporting.py +++ b/tests/unittests/test_reporting.py @@ -4,10 +4,12 @@ # vi: ts=4 expandtab from cloudinit import reporting -from cloudinit.reporting import handlers from cloudinit.reporting import events +from cloudinit.reporting import handlers + +import mock -from .helpers import (mock, TestCase) +from .helpers import TestCase def _fake_registry(): diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 8c586ad7..891dbe77 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -1,11 +1,24 @@ -from cloudinit import util -from cloudinit.config import cc_rh_subscription +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + import logging -import mock -import unittest + +from cloudinit.config import cc_rh_subscription +from cloudinit import util + +from .helpers import TestCase, mock -class GoodTests(unittest.TestCase): +class GoodTests(TestCase): def setUp(self): super(GoodTests, self).setUp() self.name = "cc_rh_subscription" @@ -92,7 +105,7 @@ class GoodTests(unittest.TestCase): self.assertEqual(self.SM._sub_man_cli.call_count, 9) -class TestBadInput(unittest.TestCase): +class TestBadInput(TestCase): name = "cc_rh_subscription" cloud_init = None log = logging.getLogger("bad_tests") diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index d0ec36a9..ce43798e 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -42,13 +42,13 @@ class TestMergeRun(helpers.FilesystemMockingTestCase): args=[PER_INSTANCE], freq=PER_INSTANCE) mirrors = initer.distro.get_option('package_mirrors') - self.assertEquals(1, len(mirrors)) + self.assertEqual(1, len(mirrors)) mirror = mirrors[0] - self.assertEquals(mirror['arches'], ['i386', 'amd64', 'blah']) + self.assertEqual(mirror['arches'], ['i386', 'amd64', 'blah']) mods = stages.Modules(initer) (which_ran, failures) = mods.run_section('cloud_init_modules') self.assertTrue(len(failures) == 0) self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') - self.assertEquals(contents, 'blah') + self.assertEqual(contents, 'blah') diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index e19e65cd..07e7b1a8 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -63,7 +63,7 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): initer.fetch() iid = initer.instancify() - self.assertEquals(iid, 'iid-datasource-none') + self.assertEqual(iid, 'iid-datasource-none') initer.update() self.assertTrue(os.path.islink("var/lib/cloud/instance")) @@ -78,4 +78,4 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.assertTrue(os.path.exists('/etc/blah.ini')) self.assertIn('write-files', which_ran) contents = util.load_file('/etc/blah.ini') - self.assertEquals(contents, 'blah') + self.assertEqual(contents, 'blah') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index b9863650..94b6e061 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -58,7 +58,7 @@ class TestTemplates(test_helpers.TestCase): blob = "blahblah $blah" (template_type, renderer, contents) = templater.detect_template(blob) self.assertIn("cheetah", template_type) - self.assertEquals(blob, contents) + self.assertEqual(blob, contents) blob = '##template:something-new' self.assertRaises(ValueError, templater.detect_template, blob) @@ -67,18 +67,18 @@ class TestTemplates(test_helpers.TestCase): blob = '''## template:cheetah $a,$b''' c = templater.render_string(blob, {"a": 1, "b": 2}) - self.assertEquals("1,2", c) + self.assertEqual("1,2", c) def test_render_jinja(self): blob = '''## template:jinja {{a}},{{b}}''' c = templater.render_string(blob, {"a": 1, "b": 2}) - self.assertEquals("1,2", c) + self.assertEqual("1,2", c) def test_render_default(self): blob = '''$a,$b''' c = templater.render_string(blob, {"a": 1, "b": 2}) - self.assertEquals("1,2", c) + self.assertEqual("1,2", c) def test_render_basic_deeper(self): hn = 'myfoohost.yahoo.com' diff --git a/tools/run-pep8 b/tools/run-pep8 index 086400fc..4bd0bbfb 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,8 +1,7 @@ #!/bin/bash -pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) -# FIXME: cloud-init modifies sys module path, pep8 does not like -# bin_files=( "bin/cloud-init" ) +pycheck_dirs=( "cloudinit/" "tests/" "tools/" ) + CR=" " [ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose="" diff --git a/tools/run-pyflakes b/tools/run-pyflakes index 4bea17f4..b3759a94 100755 --- a/tools/run-pyflakes +++ b/tools/run-pyflakes @@ -3,7 +3,7 @@ PYTHON_VERSION=${PYTHON_VERSION:-2} CR=" " -pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) +pycheck_dirs=( "cloudinit/" "tests/" "tools/" ) set -f if [ $# -eq 0 ]; then @@ -1,32 +1,30 @@ [tox] -envlist = py27,py3,pyflakes +envlist = py27,py3,flake8 recreate = True [testenv] commands = python -m nose {posargs:tests} deps = -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt - -[testenv:py3] -basepython = python3 + -r{toxinidir}/requirements.txt +setenv = + LC_ALL = en_US.utf-8 -[testenv:pyflakes] +[testenv:flake8] basepython = python3 -commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/} - {envpython} -m pep8 {posargs:cloudinit/ tests/ tools/} +commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/} # https://github.com/gabrielfalcao/HTTPretty/issues/223 setenv = LC_ALL = en_US.utf-8 +[testenv:py3] +basepython = python3 + [testenv:py26] commands = nosetests {posargs:tests} -deps = - contextlib2 - httpretty>=0.7.1 - mock - nose - pep8==1.5.7 - pyflakes setenv = LC_ALL = C + +[flake8] +ignore=H404,H405,H105,H301,H104,H403,H101 +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools diff --git a/udev/79-cloud-init-net-wait.rules b/udev/79-cloud-init-net-wait.rules deleted file mode 100644 index 8344222a..00000000 --- a/udev/79-cloud-init-net-wait.rules +++ /dev/null @@ -1,10 +0,0 @@ -# cloud-init cold/hot-plug blocking mechanism -# this file blocks further processing of network events -# until cloud-init local has had a chance to read and apply network -SUBSYSTEM!="net", GOTO="cloudinit_naming_end" -ACTION!="add", GOTO="cloudinit_naming_end" - -IMPORT{program}="/lib/udev/cloud-init-wait" - -LABEL="cloudinit_naming_end" -# vi: ts=4 expandtab syntax=udevrules diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait deleted file mode 100755 index b434005d..00000000 --- a/udev/cloud-init-wait +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/sh - -CI_NET_READY="/run/cloud-init/network-config-ready" -LOG="/run/cloud-init/${0##*/}.log" -LOG_INIT=0 -MAX_WAIT=60 -DEBUG=0 - -block_until_ready() { - local fname="$1" max="$2" - [ -f "$fname" ] && return 0 - # udevadm settle below will exit at the first of 3 conditions - # 1.) timeout 2.) file exists 3.) all in-flight udev events are processed - # since this is being run from a udev event, the 3 wont happen. - # thus, this is essentially a inotify wait or timeout on a file in /run - # that is created by cloud-init-local. - udevadm settle "--timeout=$max" "--exit-if-exists=$fname" -} - -log() { - [ -n "${LOG}" ] || return - [ "${DEBUG:-0}" = "0" ] && return - - if [ $LOG_INIT = 0 ]; then - if [ -d "${LOG%/*}" ] || mkdir -p "${LOG%/*}"; then - LOG_INIT=1 - else - echo "${0##*/}: WARN: log init to ${LOG%/*}" 1>&2 - return - fi - elif [ "$LOG_INIT" = "-1" ]; then - return - fi - local info="$$ $INTERFACE" - if [ "$DEBUG" -gt 1 ]; then - local up idle - read up idle < /proc/uptime - info="$$ $INTERFACE $up" - fi - echo "[$info]" "$@" >> "$LOG" -} - -main() { - local name="" readyfile="$CI_NET_READY" - local info="INTERFACE=${INTERFACE} ID_NET_NAME=${ID_NET_NAME}" - info="$info ID_NET_NAME_PATH=${ID_NET_NAME_PATH}" - info="$info MAC_ADDRESS=${MAC_ADDRESS}" - log "$info" - - ## Check to see if cloud-init.target is set. If cloud-init is - ## disabled we do not want to do anything. - if [ ! -f "/run/cloud-init/enabled" ]; then - log "cloud-init disabled" - return 0 - fi - - if [ "${INTERFACE#lo}" != "$INTERFACE" ]; then - return 0 - fi - - block_until_ready "$readyfile" "$MAX_WAIT" || - { log "failed waiting for ready on $INTERFACE"; return 1; } - - log "net config ready" -} - -main "$@" -exit - -# vi: ts=4 expandtab |