diff options
Diffstat (limited to 'cloudinit')
36 files changed, 1512 insertions, 265 deletions
diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 22d9167e..620b3c07 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -82,9 +82,6 @@ class Cloud(object): def get_locale(self): return self.datasource.get_locale() - def get_local_mirror(self): - return self.datasource.get_local_mirror() - def get_hostname(self, fqdn=False): return self.datasource.get_hostname(fqdn=fqdn) diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index 3426099e..02056ee0 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -16,8 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util frequency = PER_INSTANCE @@ -50,7 +50,7 @@ def handle(_name, cfg, cloud, log, _args): def write_apt_snippet(cloud, setting, log, f_name): - """ Writes f_name with apt pipeline depth 'setting' """ + """Writes f_name with apt pipeline depth 'setting'.""" file_contents = APT_PIPE_TPL % (setting) diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py index 1bffa47d..356bb98d 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -20,6 +20,7 @@ import glob import os +import time from cloudinit import templater from cloudinit import util @@ -50,20 +51,25 @@ def handle(name, cfg, cloud, log, _args): upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) release = get_release() - mirror = find_apt_mirror(cloud, cfg) - if not mirror: + mirrors = find_apt_mirror_info(cloud, cfg) + if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) return - log.debug("Selected mirror at: %s" % mirror) + # backwards compatibility + mirror = mirrors["primary"] + mirrors["mirror"] = mirror + + log.debug("mirror info: %s" % mirrors) if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror, cloud, log) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', - "archive.ubuntu.com/ubuntu") - rename_apt_lists(old_mir, mirror) + generate_sources_list(release, mirrors, cloud, log) + old_mirrors = cfg.get('apt_old_mirrors', + {"primary": "archive.ubuntu.com/ubuntu", + "security": "security.ubuntu.com/ubuntu"}) + rename_apt_lists(old_mirrors, mirrors) # Set up any apt proxy proxy = cfg.get("apt_proxy", None) @@ -81,8 +87,10 @@ def handle(name, cfg, cloud, log, _args): # Process 'apt_sources' if 'apt_sources' in cfg: - errors = add_sources(cloud, cfg['apt_sources'], - {'MIRROR': mirror, 'RELEASE': release}) + params = mirrors + params['RELEASE'] = release + params['MIRROR'] = mirror + errors = add_sources(cloud, cfg['apt_sources'], params) for e in errors: log.warn("Source Error: %s", ':'.join(e)) @@ -118,6 +126,20 @@ def handle(name, cfg, cloud, log, _args): util.logexc(log, "Failed to install packages: %s ", pkglist) errors.append(e) + # kernel and openssl (possibly some other packages) + # write a file /var/run/reboot-required after upgrading. + # if that file exists and configured, then just stop right now and reboot + # TODO(smoser): handle this less voilently + reboot_file = "/var/run/reboot-required" + if ((upgrade or pkglist) and cfg.get("apt_reboot_if_required", False) and + os.path.isfile(reboot_file)): + log.warn("rebooting after upgrade or install per %s" % reboot_file) + time.sleep(1) # give the warning time to get out + util.subp(["/sbin/reboot"]) + time.sleep(60) + log.warn("requested reboot did not happen!") + errors.append(Exception("requested reboot did not happen!")) + if len(errors): log.warn("%s failed with exceptions, re-raising the last one", len(errors)) @@ -146,15 +168,18 @@ def mirror2lists_fileprefix(mirror): return string -def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): - oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirror2lists_fileprefix(new_mirror)) - if oprefix == nprefix: - return - olen = len(oprefix) - for filename in glob.glob("%s_*" % oprefix): - # TODO use the cloud.paths.join... - util.rename(filename, "%s%s" % (nprefix, filename[olen:])) +def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): + for (name, omirror) in old_mirrors.iteritems(): + nmirror = new_mirrors.get(name) + if not nmirror: + continue + oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) + nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror)) + if oprefix == nprefix: + continue + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + util.rename(filename, "%s%s" % (nprefix, filename[olen:])) def get_release(): @@ -162,14 +187,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirror, cloud, log): +def generate_sources_list(codename, mirrors, cloud, log): template_fn = cloud.get_template_filename('sources.list') - if template_fn: - params = {'mirror': mirror, 'codename': codename} - out_fn = cloud.paths.join(False, '/etc/apt/sources.list') - templater.render_to_file(template_fn, out_fn, params) - else: + if not template_fn: log.warn("No template found, not rendering /etc/apt/sources.list") + return + + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + out_fn = cloud.paths.join(False, '/etc/apt/sources.list') + templater.render_to_file(template_fn, out_fn, params) def add_sources(cloud, srclist, template_params=None): @@ -231,43 +259,47 @@ def add_sources(cloud, srclist, template_params=None): return errorlist -def find_apt_mirror(cloud, cfg): - """ find an apt_mirror given the cloud and cfg provided """ +def find_apt_mirror_info(cloud, cfg): + """find an apt_mirror given the cloud and cfg provided.""" mirror = None - cfg_mirror = cfg.get("apt_mirror", None) - if cfg_mirror: - mirror = cfg["apt_mirror"] - elif "apt_mirror_search" in cfg: - mirror = util.search_for_mirror(cfg['apt_mirror_search']) - else: - mirror = cloud.get_local_mirror() + # this is less preferred way of specifying mirror preferred would be to + # use the distro's search or package_mirror. + mirror = cfg.get("apt_mirror", None) - mydom = "" + search = cfg.get("apt_mirror_search", None) + if not mirror and search: + mirror = util.search_for_mirror(search) + if (not mirror and + util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): + mydom = "" doms = [] - if not mirror: - # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - mydom = ".".join(fqdn.split(".")[1:]) - if mydom: - doms.append(".%s" % mydom) + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + doms.extend((".localdomain", "",)) - if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): - doms.extend((".localdomain", "",)) + mirror_list = [] + distro = cloud.distro.name + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % (post)) - mirror_list = [] - distro = cloud.distro.name - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) - for post in doms: - mirror_list.append(mirrorfmt % (post)) + mirror = util.search_for_mirror(mirror_list) - mirror = util.search_for_mirror(mirror_list) + mirror_info = cloud.datasource.get_package_mirror_info() - if not mirror: - mirror = cloud.distro.get_package_mirror() + # this is a bit strange. + # if mirror is set, then one of the legacy options above set it + # but they do not cover security. so we need to get that from + # get_package_mirror_info + if mirror: + mirror_info.update({'primary': mirror}) - return mirror + return mirror_info diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index bae1ea54..896cb4d0 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -20,8 +20,8 @@ import os -from cloudinit import util from cloudinit.settings import PER_ALWAYS +from cloudinit import util frequency = PER_ALWAYS diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py new file mode 100644 index 00000000..6d376184 --- /dev/null +++ b/cloudinit/config/cc_emit_upstart.py @@ -0,0 +1,48 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +from cloudinit.settings import PER_ALWAYS +from cloudinit import util + +frequency = PER_ALWAYS + +distros = ['ubuntu', 'debian'] + + +def handle(name, _cfg, cloud, log, args): + event_names = args + if not event_names: + # Default to the 'cloud-config' + # event for backwards compat. + event_names = ['cloud-config'] + if not os.path.isfile("/sbin/initctl"): + log.debug(("Skipping module named %s," + " no /sbin/initctl located"), name) + return + cfgpath = cloud.paths.get_ipath_cur("cloud_config") + for n in event_names: + cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] + try: + util.subp(cmd) + except Exception as e: + # TODO(harlowja), use log exception from utils?? + log.warn("Emission of upstart event %s failed due to: %s", n, e) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index aff03c4e..6b864fda 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -28,7 +28,7 @@ frequency = PER_ALWAYS # Cheetah formated default message FINAL_MESSAGE_DEF = ("Cloud-init v. ${version} finished at ${timestamp}." - " Up ${uptime} seconds.") + " Datasource ${datasource}. Up ${uptime} seconds") def handle(_name, cfg, cloud, log, args): @@ -51,6 +51,7 @@ def handle(_name, cfg, cloud, log, args): 'uptime': uptime, 'timestamp': ts, 'version': cver, + 'datasource': str(cloud.datasource), } util.multi_log("%s\n" % (templater.render_string(msg_in, subs)), console=False, stderr=True) @@ -63,3 +64,6 @@ def handle(_name, cfg, cloud, log, args): util.write_file(boot_fin_fn, contents) except: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) + + if cloud.datasource.is_disconnected: + log.warn("Used fallback datasource") diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 467c1496..74ee18e1 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -48,7 +48,8 @@ def handle(name, cfg, cloud, log, _args): # Create object for reading puppet.conf values puppet_config = helpers.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to - # mix the rest up. First clean them up (TODO is this really needed??) + # mix the rest up. First clean them up + # (TODO(harlowja) is this really needed??) cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), @@ -80,7 +81,7 @@ def handle(name, cfg, cloud, log, _args): for (o, v) in cfg.iteritems(): if o == 'certname': # Expand %f as the fqdn - # TODO should this use the cloud fqdn?? + # TODO(harlowja) should this use the cloud fqdn?? v = v.replace("%f", socket.getfqdn()) # Expand %i as the instance id v = v.replace("%i", cloud.get_instance_id()) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 256a194f..e7f27944 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -22,8 +22,8 @@ import os import stat import time -from cloudinit import util from cloudinit.settings import PER_ALWAYS +from cloudinit import util frequency = PER_ALWAYS @@ -72,12 +72,12 @@ def handle(name, cfg, cloud, log, args): log.debug("Skipping module named %s, resizing disabled", name) return - # TODO is the directory ok to be used?? + # TODO(harlowja) is the directory ok to be used?? resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") resize_root_d = cloud.paths.join(False, resize_root_d) util.ensure_dir(resize_root_d) - # TODO: allow what is to be resized to be configurable?? + # TODO(harlowja): allow what is to be resized to be configurable?? resize_what = cloud.paths.join(False, "/") with util.ExtendedTemporaryFile(prefix="cloudinit.resizefs.", dir=resize_root_d, delete=True) as tfh: @@ -136,5 +136,5 @@ def do_resize(resize_cmd, log): raise tot_time = time.time() - start log.debug("Resizing took %.3f seconds", tot_time) - # TODO: Should we add a fsck check after this to make + # TODO(harlowja): Should we add a fsck check after this to make # sure we didn't corrupt anything? diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 45d41b3f..4bf18516 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -37,9 +37,9 @@ import os +from cloudinit.settings import PER_INSTANCE from cloudinit import url_helper as uhelp from cloudinit import util -from cloudinit.settings import PER_INSTANCE from urlparse import parse_qs @@ -72,7 +72,7 @@ def handle(name, _cfg, cloud, log, _args): captured_excps = [] # These will eventually be then ran by the cc_scripts_user - # TODO: maybe this should just be a new user data handler?? + # TODO(harlowja): maybe this should just be a new user data handler?? # Instead of a late module that acts like a user data handler? scripts_d = cloud.get_ipath_cur('scripts') urls = mdict[MY_HOOKNAME] diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index ab266741..7d0fbd9f 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -50,8 +50,16 @@ def handle(_name, cfg, cloud, log, args): expire = util.get_cfg_option_bool(chfg, 'expire', expire) if not plist and password: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - plist = "%s:%s" % (user, password) + user = cloud.distro.get_default_user() + + if 'users' in cfg: + user_zero = cfg['users'].keys()[0] + + if user_zero != "default": + user = user_zero + + if user: + plist = "%s:%s" % (user, password) errors = [] if plist: diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 4019ae90..439c8eb8 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -18,11 +18,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import os import glob +import os -from cloudinit import util from cloudinit import ssh_util +from cloudinit import util DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " @@ -76,7 +76,7 @@ def handle(_name, cfg, cloud, log, _args): pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: - # TODO: Is this guard needed? + # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) @@ -94,7 +94,7 @@ def handle(_name, cfg, cloud, log, _args): if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: - # TODO: Is this guard needed? + # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: @@ -102,7 +102,16 @@ def handle(_name, cfg, cloud, log, _args): " %s to file %s"), keytype, keyfile) try: - user = util.get_cfg_option_str(cfg, 'user') + # TODO(utlemming): consolidate this stanza that occurs in: + # cc_ssh_import_id, cc_set_passwords, maybe cc_users_groups.py + user = cloud.distro.get_default_user() + + if 'users' in cfg: + user_zero = cfg['users'].keys()[0] + + if user_zero != "default": + user = user_zero + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) @@ -124,7 +133,9 @@ def apply_credentials(keys, user, paths, disable_root, disable_root_opts): if user: ssh_util.setup_user_keys(keys, user, '', paths) - if disable_root and user: + if disable_root: + if not user: + user = "NONE" key_prefix = disable_root_opts.replace('$USER', user) else: key_prefix = '' diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py new file mode 100644 index 00000000..23f5755a --- /dev/null +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -0,0 +1,96 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import base64 +import hashlib + +from prettytable import PrettyTable + +from cloudinit import ssh_util +from cloudinit import util + + +def _split_hash(bin_hash): + split_up = [] + for i in xrange(0, len(bin_hash), 2): + split_up.append(bin_hash[i:i + 2]) + return split_up + + +def _gen_fingerprint(b64_text, hash_meth='md5'): + if not b64_text: + return '' + # TBD(harlowja): Maybe we should feed this into 'ssh -lf'? + try: + hasher = hashlib.new(hash_meth) + hasher.update(base64.b64decode(b64_text)) + return ":".join(_split_hash(hasher.hexdigest())) + except TypeError: + # Raised when b64 not really b64... + return '?' + + +def _is_printable_key(entry): + if any([entry.keytype, entry.base64, entry.comment, entry.options]): + if (entry.keytype and + entry.keytype.lower().strip() in ['ssh-dss', 'ssh-rsa']): + return True + return False + + +def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', + prefix='ci-info: '): + if not key_entries: + message = ("%sno authorized ssh keys fingerprints found for user %s." + % (prefix, user)) + util.multi_log(message) + return + tbl_fields = ['Keytype', 'Fingerprint (%s)' % (hash_meth), 'Options', + 'Comment'] + tbl = PrettyTable(tbl_fields) + for entry in key_entries: + if _is_printable_key(entry): + row = [] + row.append(entry.keytype or '-') + row.append(_gen_fingerprint(entry.base64, hash_meth) or '-') + row.append(entry.options or '-') + row.append(entry.comment or '-') + tbl.add_row(row) + authtbl_s = tbl.get_string() + authtbl_lines = authtbl_s.splitlines() + max_len = len(max(authtbl_lines, key=len)) + lines = [ + util.center("Authorized keys from %s for user %s" % + (key_fn, user), "+", max_len), + ] + lines.extend(authtbl_lines) + for line in lines: + util.multi_log(text="%s%s\n" % (prefix, line), + stderr=False, console=True) + + +def handle(name, cfg, cloud, log, _args): + if 'no_ssh_fingerprints' in cfg: + log.debug(("Skipping module named %s, " + "logging of ssh fingerprints disabled"), name) + + user_name = util.get_cfg_option_str(cfg, "user", "ubuntu") + hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") + extract = ssh_util.extract_authorized_keys + (auth_key_fn, auth_key_entries) = extract(user_name, cloud.paths) + _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, hash_meth) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index c58b28ec..c5f07376 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -19,35 +19,69 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from cloudinit import util +import pwd # The ssh-import-id only seems to exist on ubuntu (for now) # https://launchpad.net/ssh-import-id distros = ['ubuntu'] -def handle(name, cfg, _cloud, log, args): +def handle(_name, cfg, cloud, log, args): + + # import for "user: XXXXX" if len(args) != 0: user = args[0] ids = [] if len(args) > 1: ids = args[1:] - else: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) - if len(ids) == 0: - log.debug("Skipping module named %s, no ids found to import", name) + import_ssh_ids(ids, user, log) return - if not user: - log.debug("Skipping module named %s, no user found to import", name) + # import for cloudinit created users + elist = [] + for user in cfg['users'].keys(): + if user == "default": + user = cloud.distro.get_default_user() + if not user: + continue + import_ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) + else: + if not isinstance(cfg['users'][user], dict): + log.debug("cfg['users'][%s] not a dict, skipping ssh_import", + user) + import_ids = util.get_cfg_option_list(cfg['users'][user], + "ssh_import_id", []) + + if not len(import_ids): + continue + + try: + import_ssh_ids(import_ids, user, log) + except Exception as exc: + util.logexc(exc, "ssh-import-id failed for: %s %s" % + (user, import_ids)) + elist.append(exc) + + if len(elist): + raise elist[0] + + +def import_ssh_ids(ids, user, log): + if not (user and ids): + log.debug("empty user(%s) or ids(%s). not importing", user, ids) return + try: + _check = pwd.getpwnam(user) + except KeyError as exc: + raise exc + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids log.debug("Importing ssh ids for user %s.", user) try: util.subp(cmd, capture=False) - except util.ProcessExecutionError as e: + except util.ProcessExecutionError as exc: util.logexc(log, "Failed to run command to import %s ssh ids", user) - raise e + raise exc diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 38108da7..4d75000f 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -18,8 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import util from cloudinit import templater +from cloudinit import util from cloudinit.settings import PER_ALWAYS diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index b84a1a06..1d6679ea 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -20,8 +20,8 @@ import os -from cloudinit import util from cloudinit.settings import PER_ALWAYS +from cloudinit import util frequency = PER_ALWAYS diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py new file mode 100644 index 00000000..1e241623 --- /dev/null +++ b/cloudinit/config/cc_users_groups.py @@ -0,0 +1,70 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Ben Howard <ben.howard@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE + + +def handle(name, cfg, cloud, log, _args): + user_zero = None + + if 'groups' in cfg: + for group in cfg['groups']: + if isinstance(group, dict): + for name, values in group.iteritems(): + if isinstance(values, list): + cloud.distro.create_group(name, values) + elif isinstance(values, str): + cloud.distro.create_group(name, values.split(',')) + else: + cloud.distro.create_group(group, []) + + if 'users' in cfg: + user_zero = None + + for name, user_config in cfg['users'].iteritems(): + if not user_zero: + user_zero = name + + # Handle the default user creation + if name == "default" and user_config: + log.info("Creating default user") + + # Create the default user if so defined + try: + cloud.distro.add_default_user() + + if user_zero == name: + user_zero = cloud.distro.get_default_user() + + except NotImplementedError: + + if user_zero == name: + user_zero = None + + log.warn("Distro has not implemented default user " + "creation. No default user will be created") + else: + # Make options friendly for distro.create_user + new_opts = {} + if isinstance(user_config, dict): + for opt in user_config: + new_opts[opt.replace('-', '')] = user_config[opt] + + cloud.distro.create_user(name, **new_opts) diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 1bfa4c25..a73d6f4e 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -19,8 +19,8 @@ import base64 import os -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util frequency = PER_INSTANCE @@ -46,7 +46,7 @@ def canonicalize_extraction(encoding_type, log): return ['application/x-gzip'] if encoding_type in ['gz+base64', 'gzip+base64', 'gz+b64', 'gzip+b64']: return ['application/base64', 'application/x-gzip'] - # Yaml already encodes binary data as base64 if it is given to the + # Yaml already encodes binary data as base64 if it is given to the # yaml file as binary, so those will be automatically decoded for you. # But the above b64 is just for people that are more 'comfortable' # specifing it manually (which might be a possiblity) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index da4d0180..686c6a9b 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -7,6 +7,7 @@ # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> # Author: Joshua Harlow <harlowja@yahoo-inc.com> +# Author: Ben Howard <ben.howard@canonical.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -23,12 +24,17 @@ from StringIO import StringIO import abc +import grp +import os +import pwd +import re from cloudinit import importer from cloudinit import log as logging +from cloudinit import ssh_util from cloudinit import util -# TODO: Make this via config?? +# TODO(harlowja): Make this via config?? IFACE_ACTIONS = { 'up': ['ifup', '--all'], 'down': ['ifdown', '--all'], @@ -40,12 +46,32 @@ LOG = logging.getLogger(__name__) class Distro(object): __metaclass__ = abc.ABCMeta + default_user = None def __init__(self, name, cfg, paths): self._paths = paths self._cfg = cfg self.name = name + def add_default_user(self): + # Adds the distro user using the rules: + # - Password is same as username but is locked + # - nopasswd sudo access + + user = self.get_default_user() + if not user: + raise NotImplementedError("No Default user") + + self.create_user(user, + plain_text_passwd=user, + home="/home/%s" % user, + shell="/bin/bash", + lockpasswd=True, + gecos="%s%s" % (user[0:1].upper(), user[1:]), + sudo="ALL=(ALL) NOPASSWD:ALL") + + LOG.info("Added default '%s' user with passwordless sudo", user) + @abc.abstractmethod def install_packages(self, pkglist): raise NotImplementedError() @@ -75,8 +101,26 @@ class Distro(object): def update_package_sources(self): raise NotImplementedError() - def get_package_mirror(self): - return self.get_option('package_mirror') + def get_primary_arch(self): + arch = os.uname[4] + if arch in ("i386", "i486", "i586", "i686"): + return "i386" + return arch + + def _get_arch_package_mirror_info(self, arch=None): + mirror_info = self.get_option("package_mirrors", None) + if arch == None: + arch = self.get_primary_arch() + return _get_arch_package_mirror_info(mirror_info, arch) + + def get_package_mirror_info(self, arch=None, + availability_zone=None): + # this resolves the package_mirrors config option + # down to a single dict of {mirror_name: mirror_url} + arch_info = self._get_arch_package_mirror_info(arch) + + return _get_package_mirror_info(availability_zone=availability_zone, + mirror_info=arch_info) def apply_network(self, settings, bring_up=True): # Write it out @@ -150,6 +194,227 @@ class Distro(object): util.logexc(LOG, "Running interface command %s failed", cmd) return False + def isuser(self, name): + try: + if pwd.getpwnam(name): + return True + except KeyError: + return False + + def get_default_user(self): + return self.default_user + + def create_user(self, name, **kwargs): + """ + Creates users for the system using the GNU passwd tools. This + will work on an GNU system. This should be overriden on + distros where useradd is not desirable or not available. + """ + + adduser_cmd = ['useradd', name] + x_adduser_cmd = ['useradd', name] + + # Since we are creating users, we want to carefully validate the + # inputs. If something goes wrong, we can end up with a system + # that nobody can login to. + adduser_opts = { + "gecos": '--comment', + "homedir": '--home', + "primarygroup": '--gid', + "groups": '--groups', + "passwd": '--password', + "shell": '--shell', + "expiredate": '--expiredate', + "inactive": '--inactive', + } + + adduser_opts_flags = { + "nousergroup": '--no-user-group', + "system": '--system', + "nologinit": '--no-log-init', + "nocreatehome": "-M", + } + + # Now check the value and create the command + for option in kwargs: + value = kwargs[option] + if option in adduser_opts and value \ + and isinstance(value, str): + adduser_cmd.extend([adduser_opts[option], value]) + + # Redact the password field from the logs + if option != "password": + x_adduser_cmd.extend([adduser_opts[option], value]) + else: + x_adduser_cmd.extend([adduser_opts[option], 'REDACTED']) + + elif option in adduser_opts_flags and value: + adduser_cmd.append(adduser_opts_flags[option]) + x_adduser_cmd.append(adduser_opts_flags[option]) + + # Default to creating home directory unless otherwise directed + # Also, we do not create home directories for system users. + if "nocreatehome" not in kwargs and "system" not in kwargs: + adduser_cmd.append('-m') + + # Create the user + if self.isuser(name): + LOG.warn("User %s already exists, skipping." % name) + else: + LOG.debug("Creating name %s" % name) + try: + util.subp(adduser_cmd, logstring=x_adduser_cmd) + except Exception as e: + util.logexc(LOG, "Failed to create user %s due to error.", e) + raise e + + # Set password if plain-text password provided + if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: + self.set_passwd(name, kwargs['plain_text_passwd']) + + # Default locking down the account. + if ('lockpasswd' not in kwargs and + ('lockpasswd' in kwargs and kwargs['lockpasswd']) or + 'system' not in kwargs): + try: + util.subp(['passwd', '--lock', name]) + except Exception as e: + util.logexc(LOG, ("Failed to disable password logins for" + "user %s" % name), e) + raise e + + # Configure sudo access + if 'sudo' in kwargs: + self.write_sudo_rules(name, kwargs['sudo']) + + # Import SSH keys + if 'sshauthorizedkeys' in kwargs: + keys = set(kwargs['sshauthorizedkeys']) or [] + ssh_util.setup_user_keys(keys, name, None, self._paths) + + return True + + def set_passwd(self, user, passwd, hashed=False): + pass_string = '%s:%s' % (user, passwd) + cmd = ['chpasswd'] + + if hashed: + cmd.append('--encrypted') + + try: + util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) + except Exception as e: + util.logexc(LOG, "Failed to set password for %s" % user) + raise e + + return True + + def write_sudo_rules(self, + user, + rules, + sudo_file="/etc/sudoers.d/90-cloud-init-users", + ): + + content_header = "# user rules for %s" % user + content = "%s\n%s %s\n\n" % (content_header, user, rules) + + if isinstance(rules, list): + content = "%s\n" % content_header + for rule in rules: + content += "%s %s\n" % (user, rule) + content += "\n" + + if not os.path.exists(sudo_file): + util.write_file(sudo_file, content, 0644) + + else: + try: + with open(sudo_file, 'a') as f: + f.write(content) + except IOError as e: + util.logexc(LOG, "Failed to write %s" % sudo_file, e) + raise e + + def isgroup(self, name): + try: + if grp.getgrnam(name): + return True + except: + return False + + def create_group(self, name, members): + group_add_cmd = ['groupadd', name] + + # Check if group exists, and then add it doesn't + if self.isgroup(name): + LOG.warn("Skipping creation of existing group '%s'" % name) + else: + try: + util.subp(group_add_cmd) + LOG.info("Created new group %s" % name) + except Exception as e: + util.logexc("Failed to create group %s" % name, e) + + # Add members to the group, if so defined + if len(members) > 0: + for member in members: + if not self.isuser(member): + LOG.warn("Unable to add group member '%s' to group '%s'" + "; user does not exist." % (member, name)) + continue + + util.subp(['usermod', '-a', '-G', name, member]) + LOG.info("Added user '%s' to group '%s'" % (member, name)) + + +def _get_package_mirror_info(mirror_info, availability_zone=None, + mirror_filter=util.search_for_mirror): + # given a arch specific 'mirror_info' entry (from package_mirrors) + # search through the 'search' entries, and fallback appropriately + # return a dict with only {name: mirror} entries. + + ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % + "north|northeast|east|southeast|south|southwest|west|northwest") + + subst = {} + if availability_zone: + subst['availability_zone'] = availability_zone + + if availability_zone and re.match(ec2_az_re, availability_zone): + subst['ec2_region'] = "%s" % availability_zone[0:-1] + + results = {} + for (name, mirror) in mirror_info.get('failsafe', {}).iteritems(): + results[name] = mirror + + for (name, searchlist) in mirror_info.get('search', {}).iteritems(): + mirrors = [] + for tmpl in searchlist: + try: + mirrors.append(tmpl % subst) + except KeyError: + pass + + found = mirror_filter(mirrors) + if found: + results[name] = found + + LOG.debug("filtered distro mirror info: %s" % results) + + return results + + +def _get_arch_package_mirror_info(package_mirrors, arch): + # pull out the specific arch from a 'package_mirrors' config option + default = None + for item in package_mirrors: + arches = item.get("arches") + if arch in arches: + return item + if "default" in arches: + default = item + return default + def fetch(name): locs = importer.find_module(name, diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 3247d7ce..da8c1a5b 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -147,3 +147,7 @@ class Distro(distros.Distro): def update_package_sources(self): self._runner.run("update-sources", self.package_command, ["update"], freq=PER_INSTANCE) + + def get_primary_arch(self): + (arch, _err) = util.subp(['dpkg', '--print-architecture']) + return str(arch).strip() diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 7fa69f03..d81ee5fb 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -69,7 +69,7 @@ class Distro(distros.Distro): self.package_command('install', pkglist) def _write_network(self, settings): - # TODO fix this... since this is the ubuntu format + # TODO(harlowja) fix this... since this is the ubuntu format entries = translate_network(settings) LOG.debug("Translated ubuntu style network settings %s into %s", settings, entries) @@ -258,7 +258,7 @@ class QuotingConfigObj(ConfigObj): # This is a util function to translate a ubuntu /etc/network/interfaces 'blob' # to a rhel equiv. that can then be written to /etc/sysconfig/network-scripts/ -# TODO remove when we have python-netcf active... +# TODO(harlowja) remove when we have python-netcf active... def translate_network(settings): # Get the standard cmd, args from the ubuntu format entries = [] diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 77c2aff4..1f4efb59 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -7,6 +7,7 @@ # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> # Author: Joshua Harlow <harlowja@yahoo-inc.com> +# Author: Ben Howard <ben.howard@canonical.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -21,11 +22,12 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from cloudinit.distros import debian - from cloudinit import log as logging LOG = logging.getLogger(__name__) class Distro(debian.Distro): - pass + + distro_name = 'ubuntu' + default_user = 'ubuntu' diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 6d1502f4..99caed1f 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -133,7 +133,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): modfname = os.path.join(pdata['handlerdir'], "%s" % (modname)) if not modfname.endswith(".py"): modfname = "%s.py" % (modfname) - # TODO: Check if path exists?? + # TODO(harlowja): Check if path exists?? util.write_file(modfname, payload, 0600) handlers = pdata['handlers'] try: diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index a9d8e544..6c5c11ca 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -43,7 +43,7 @@ class ShellScriptPartHandler(handlers.Handler): def _handle_part(self, _data, ctype, filename, payload, _frequency): if ctype in handlers.CONTENT_SIGNALS: - # TODO: maybe delete existing things here + # TODO(harlowja): maybe delete existing things here return filename = util.clean_filename(filename) diff --git a/cloudinit/log.py b/cloudinit/log.py index 819c85b6..2333e5ee 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -21,8 +21,8 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import logging -import logging.handlers import logging.config +import logging.handlers import collections import os diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 2083cf60..8cc9e3b4 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -31,10 +31,13 @@ CFG_BUILTIN = { 'datasource_list': [ 'NoCloud', 'ConfigDrive', + 'AltCloud', 'OVF', 'MAAS', 'Ec2', - 'CloudStack' + 'CloudStack', + # At the end to act as a 'catch' when none of the above work... + 'None', ], 'def_log_file': '/var/log/cloud-init.log', 'log_cfgs': [], diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py new file mode 100644 index 00000000..69c376a5 --- /dev/null +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -0,0 +1,299 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joe VLcek <JVLcek@RedHat.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +''' +This file contains code used to gather the user data passed to an +instance on RHEVm and vSphere. +''' + +import errno +import os +import os.path + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util +from cloudinit.util import ProcessExecutionError + +LOG = logging.getLogger(__name__) + +# Needed file paths +CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info' + +# Shell command lists +CMD_DMI_SYSTEM = ['/usr/sbin/dmidecode', '--string', 'system-product-name'] +CMD_PROBE_FLOPPY = ['/sbin/modprobe', 'floppy'] +CMD_UDEVADM_SETTLE = ['/sbin/udevadm', 'settle', '--quiet', '--timeout=5'] + +META_DATA_NOT_SUPPORTED = { + 'block-device-mapping': {}, + 'instance-id': 455, + 'local-hostname': 'localhost', + 'placement': {}, + } + + +def read_user_data_callback(mount_dir): + ''' + Description: + This callback will be applied by util.mount_cb() on the mounted + file. + + Deltacloud file name contains deltacloud. Those not using + Deltacloud but instead instrumenting the injection, could + drop deltacloud from the file name. + + Input: + mount_dir - Mount directory + + Returns: + User Data + + ''' + + deltacloud_user_data_file = mount_dir + '/deltacloud-user-data.txt' + user_data_file = mount_dir + '/user-data.txt' + + # First try deltacloud_user_data_file. On failure try user_data_file. + try: + with open(deltacloud_user_data_file, 'r') as user_data_f: + user_data = user_data_f.read().strip() + except: + try: + with open(user_data_file, 'r') as user_data_f: + user_data = user_data_f.read().strip() + except: + util.logexc(LOG, ('Failed accessing user data file.')) + return None + + return user_data + + +class DataSourceAltCloud(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed = None + self.supported_seed_starts = ("/", "file://") + + def __str__(self): + mstr = "%s [seed=%s]" % (util.obj_name(self), self.seed) + return mstr + + def get_cloud_type(self): + ''' + Description: + Get the type for the cloud back end this instance is running on + by examining the string returned by: + dmidecode --string system-product-name + + On VMWare/vSphere dmidecode returns: RHEV Hypervisor + On VMWare/vSphere dmidecode returns: VMware Virtual Platform + + Input: + None + + Returns: + One of the following strings: + 'RHEV', 'VSPHERE' or 'UNKNOWN' + + ''' + + cmd = CMD_DMI_SYSTEM + try: + (cmd_out, _err) = util.subp(cmd) + except ProcessExecutionError, _err: + LOG.debug(('Failed command: %s\n%s') % \ + (' '.join(cmd), _err.message)) + return 'UNKNOWN' + except OSError, _err: + LOG.debug(('Failed command: %s\n%s') % \ + (' '.join(cmd), _err.message)) + return 'UNKNOWN' + + if cmd_out.upper().startswith('RHEV'): + return 'RHEV' + + if cmd_out.upper().startswith('VMWARE'): + return 'VSPHERE' + + return 'UNKNOWN' + + def get_data(self): + ''' + Description: + User Data is passed to the launching instance which + is used to perform instance configuration. + + Cloud providers expose the user data differently. + It is necessary to determine which cloud provider + the current instance is running on to determine + how to access the user data. Images built with + image factory will contain a CLOUD_INFO_FILE which + contains a string identifying the cloud provider. + + Images not built with Imagefactory will try to + determine what the cloud provider is based on system + information. + ''' + + LOG.debug('Invoked get_data()') + + if os.path.exists(CLOUD_INFO_FILE): + try: + cloud_info = open(CLOUD_INFO_FILE) + cloud_type = cloud_info.read().strip().upper() + cloud_info.close() + except: + util.logexc(LOG, 'Unable to access cloud info file.') + return False + else: + cloud_type = self.get_cloud_type() + + LOG.debug('cloud_type: ' + str(cloud_type)) + + if 'RHEV' in cloud_type: + if self.user_data_rhevm(): + return True + elif 'VSPHERE' in cloud_type: + if self.user_data_vsphere(): + return True + else: + # there was no recognized alternate cloud type + # indicating this handler should not be used. + return False + + # No user data found + util.logexc(LOG, ('Failed accessing user data.')) + return False + + def user_data_rhevm(self): + ''' + RHEVM specific userdata read + + If on RHEV-M the user data will be contained on the + floppy device in file <user_data_file> + To access it: + modprobe floppy + + Leverage util.mount_cb to: + mkdir <tmp mount dir> + mount /dev/fd0 <tmp mount dir> + The call back passed to util.mount_cb will do: + read <tmp mount dir>/<user_data_file> + ''' + + return_str = None + + # modprobe floppy + try: + cmd = CMD_PROBE_FLOPPY + (cmd_out, _err) = util.subp(cmd) + LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) + except ProcessExecutionError, _err: + util.logexc(LOG, (('Failed command: %s\n%s') % \ + (' '.join(cmd), _err.message))) + return False + except OSError, _err: + util.logexc(LOG, (('Failed command: %s\n%s') % \ + (' '.join(cmd), _err.message))) + return False + + floppy_dev = '/dev/fd0' + + # udevadm settle for floppy device + try: + cmd = CMD_UDEVADM_SETTLE + cmd.append('--exit-if-exists=' + floppy_dev) + (cmd_out, _err) = util.subp(cmd) + LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) + except ProcessExecutionError, _err: + util.logexc(LOG, (('Failed command: %s\n%s') % \ + (' '.join(cmd), _err.message))) + return False + except OSError, _err: + util.logexc(LOG, (('Failed command: %s\n%s') % \ + (' '.join(cmd), _err.message))) + return False + + try: + return_str = util.mount_cb(floppy_dev, read_user_data_callback) + except OSError as err: + if err.errno != errno.ENOENT: + raise + except util.MountFailedError: + util.logexc(LOG, ("Failed to mount %s" + " when looking for user data"), floppy_dev) + + self.userdata_raw = return_str + self.metadata = META_DATA_NOT_SUPPORTED + + if return_str: + return True + else: + return False + + def user_data_vsphere(self): + ''' + vSphere specific userdata read + + If on vSphere the user data will be contained on the + cdrom device in file <user_data_file> + To access it: + Leverage util.mount_cb to: + mkdir <tmp mount dir> + mount /dev/fd0 <tmp mount dir> + The call back passed to util.mount_cb will do: + read <tmp mount dir>/<user_data_file> + ''' + + return_str = None + cdrom_list = util.find_devs_with('LABEL=CDROM') + for cdrom_dev in cdrom_list: + try: + return_str = util.mount_cb(cdrom_dev, read_user_data_callback) + if return_str: + break + except OSError as err: + if err.errno != errno.ENOENT: + raise + except util.MountFailedError: + util.logexc(LOG, ("Failed to mount %s" + " when looking for user data"), cdrom_dev) + + self.userdata_raw = return_str + self.metadata = META_DATA_NOT_SUPPORTED + + if return_str: + return True + else: + return False + +# Used to match classes to dependencies +# Source DataSourceAltCloud does not really depend on networking. +# In the future 'dsmode' like behavior can be added to offer user +# the ability to run before networking. +datasources = [ + (DataSourceAltCloud, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 751bef4f..f7ffa7cb 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -49,8 +49,7 @@ class DataSourceCloudStack(sources.DataSource): self.metadata_address = "http://%s/" % (gw_addr) def get_default_gateway(self): - """ Returns the default gateway ip address in the dotted format - """ + """Returns the default gateway ip address in the dotted format.""" lines = util.load_file("/proc/net/route").splitlines() for line in lines: items = line.split("\t") @@ -132,7 +131,8 @@ class DataSourceCloudStack(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def get_availability_zone(self): + @property + def availability_zone(self): return self.metadata['availability-zone'] diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 320dd1d1..b8154367 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -30,88 +30,119 @@ LOG = logging.getLogger(__name__) # Various defaults/constants... DEFAULT_IID = "iid-dsconfigdrive" DEFAULT_MODE = 'pass' -CFG_DRIVE_FILES = [ +CFG_DRIVE_FILES_V1 = [ "etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js", ] DEFAULT_METADATA = { "instance-id": DEFAULT_IID, - "dsmode": DEFAULT_MODE, } -CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' +VALID_DSMODES = ("local", "net", "pass", "disabled") class DataSourceConfigDrive(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed = None - self.cfg = {} + self.source = None self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') + self.version = None def __str__(self): - mstr = "%s [%s]" % (util.obj_name(self), self.dsmode) - mstr += "[seed=%s]" % (self.seed) + mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode, + self.version) + mstr += "[source=%s]" % (self.source) return mstr def get_data(self): found = None md = {} - ud = "" + results = {} if os.path.isdir(self.seed_dir): try: - (md, ud) = read_config_drive_dir(self.seed_dir) + results = read_config_drive_dir(self.seed_dir) found = self.seed_dir except NonConfigDriveDir: util.logexc(LOG, "Failed reading config drive from %s", self.seed_dir) if not found: - dev = find_cfg_drive_device() - if dev: + devlist = find_candidate_devs() + for dev in devlist: try: - (md, ud) = util.mount_cb(dev, read_config_drive_dir) + results = util.mount_cb(dev, read_config_drive_dir) found = dev + break except (NonConfigDriveDir, util.MountFailedError): pass + except BrokenConfigDriveDir: + util.logexc(LOG, "broken config drive: %s", dev) if not found: return False - if 'dsconfig' in md: - self.cfg = md['dscfg'] - + md = results['metadata'] md = util.mergedict(md, DEFAULT_METADATA) - # Update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. - if 'network-interfaces' in md and self.dsmode == "local": + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("user specified invalid mode: %s" % user_dsmode) + user_dsmode = None + + dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'], + ds_cfg=self.ds_cfg.get('dsmode'), + user=user_dsmode) + + if dsmode == "disabled": + # most likely user specified + return False + + # TODO(smoser): fix this, its dirty. + # we want to do some things (writing files and network config) + # only on first boot, and even then, we want to do so in the + # local datasource (so they happen earlier) even if the configured + # dsmode is 'net' or 'pass'. To do this, we check the previous + # instance-id + prev_iid = get_previous_iid(self.paths) + cur_iid = md['instance-id'] + + if ('network_config' in results and self.dsmode == "local" and + prev_iid != cur_iid): LOG.debug("Updating network interfaces from config drive (%s)", - md['dsmode']) - self.distro.apply_network(md['network-interfaces']) + dsmode) + self.distro.apply_network(results['network_config']) - self.seed = found - self.metadata = md - self.userdata_raw = ud + # file writing occurs in local mode (to be as early as possible) + if self.dsmode == "local" and prev_iid != cur_iid and results['files']: + LOG.debug("writing injected files") + try: + write_files(results['files']) + except: + util.logexc(LOG, "Failed writing files") + + # dsmode != self.dsmode here if: + # * dsmode = "pass", pass means it should only copy files and then + # pass to another datasource + # * dsmode = "net" and self.dsmode = "local" + # so that user boothooks would be applied with network, the + # local datasource just gets out of the way, and lets the net claim + if dsmode != self.dsmode: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + return False - if md['dsmode'] == self.dsmode: - return True + self.source = found + self.metadata = md + self.userdata_raw = results.get('userdata') + self.version = results['cfgdrive_ver'] - LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) - return False + return True def get_public_ssh_keys(self): if not 'public-keys' in self.metadata: return [] return self.metadata['public-keys'] - # The data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return self.cfg - class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -123,48 +154,146 @@ class NonConfigDriveDir(Exception): pass -def find_cfg_drive_device(): - """ Get the config drive device. Return a string like '/dev/vdb' - or None (if there is no non-root device attached). This does not - check the contents, only reports that if there *were* a config_drive - attached, it would be this device. - Note: per config_drive documentation, this is - "associated as the last available disk on the instance" - """ +class BrokenConfigDriveDir(Exception): + pass - # This seems to be for debugging?? - if CFG_DRIVE_DEV_ENV in os.environ: - return os.environ[CFG_DRIVE_DEV_ENV] - # We are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it.... - letters = "abcdefghijklmnopqrstuvwxyz" - devs = util.find_devs_with("TYPE=vfat") +def find_candidate_devs(): + """Return a list of devices that may contain the config drive. - # Filter out anything not ending in a letter (ignore partitions) - devs = [f for f in devs if f[-1] in letters] + The returned list is sorted by search order where the first item has + should be searched first (highest priority) + + config drive v1: + Per documentation, this is "associated as the last available disk on the + instance", and should be VFAT. + Currently, we do not restrict search list to "last available disk" + + config drive v2: + Disk should be: + * either vfat or iso9660 formated + * labeled with 'config-2' + """ - # Sort them in reverse so "last" device is first - devs.sort(reverse=True) + by_fstype = (util.find_devs_with("TYPE=vfat") + + util.find_devs_with("TYPE=iso9660")) + by_label = util.find_devs_with("LABEL=config-2") - if devs: - return devs[0] + # give preference to "last available disk" (vdb over vda) + # note, this is not a perfect rendition of that. + by_fstype.sort(reverse=True) + by_label.sort(reverse=True) - return None + # combine list of items by putting by-label items first + # followed by fstype items, but with dupes removed + combined = (by_label + [d for d in by_fstype if d not in by_label]) + + # We are looking for block device (sda, not sda1), ignore partitions + combined = [d for d in combined if d[-1] not in "0123456789"] + + return combined def read_config_drive_dir(source_dir): + last_e = NonConfigDriveDir("Not found") + for finder in (read_config_drive_dir_v2, read_config_drive_dir_v1): + try: + data = finder(source_dir) + return data + except NonConfigDriveDir as exc: + last_e = exc + raise last_e + + +def read_config_drive_dir_v2(source_dir, version="2012-08-10"): + + if (not os.path.isdir(os.path.join(source_dir, "openstack", version)) and + os.path.isdir(os.path.join(source_dir, "openstack", "latest"))): + LOG.warn("version '%s' not available, attempting to use 'latest'" % + version) + version = "latest" + + datafiles = ( + ('metadata', + "openstack/%s/meta_data.json" % version, True, json.loads), + ('userdata', "openstack/%s/user_data" % version, False, None), + ('ec2-metadata', "ec2/latest/metadata.json", False, json.loads), + ) + + results = {'userdata': None} + for (name, path, required, process) in datafiles: + fpath = os.path.join(source_dir, path) + data = None + found = False + if os.path.isfile(fpath): + try: + with open(fpath) as fp: + data = fp.read() + except Exception as exc: + raise BrokenConfigDriveDir("failed to read: %s" % fpath) + found = True + elif required: + raise NonConfigDriveDir("missing mandatory %s" % fpath) + + if found and process: + try: + data = process(data) + except Exception as exc: + raise BrokenConfigDriveDir("failed to process: %s" % fpath) + + if found: + results[name] = data + + # instance-id is 'uuid' for openstack. just copy it to instance-id. + if 'instance-id' not in results['metadata']: + try: + results['metadata']['instance-id'] = results['metadata']['uuid'] + except KeyError: + raise BrokenConfigDriveDir("No uuid entry in metadata") + + def read_content_path(item): + # do not use os.path.join here, as content_path starts with / + cpath = os.path.sep.join((source_dir, "openstack", + "./%s" % item['content_path'])) + with open(cpath) as fp: + return(fp.read()) + + files = {} + try: + for item in results['metadata'].get('files', {}): + files[item['path']] = read_content_path(item) + + # the 'network_config' item in metadata is a content pointer + # to the network config that should be applied. + # in folsom, it is just a '/etc/network/interfaces' file. + item = results['metadata'].get("network_config", None) + if item: + results['network_config'] = read_content_path(item) + except Exception as exc: + raise BrokenConfigDriveDir("failed to read file %s: %s" % (item, exc)) + + # to openstack, user can specify meta ('nova boot --meta=key=value') and + # those will appear under metadata['meta']. + # if they specify 'dsmode' they're indicating the mode that they intend + # for this datasource to operate in. + try: + results['dsmode'] = results['metadata']['meta']['dsmode'] + except KeyError: + pass + + results['files'] = files + results['cfgdrive_ver'] = 2 + return results + + +def read_config_drive_dir_v1(source_dir): """ - read_config_drive_dir(source_dir): - read source_dir, and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a NonConfigDriveDir + read source_dir, and return a tuple with metadata dict, user-data, + files and version (1). If not a valid dir, raise a NonConfigDriveDir """ - # TODO: fix this for other operating systems... - # Ie: this is where https://fedorahosted.org/netcf/ or similar should - # be hooked in... (or could be) found = {} - for af in CFG_DRIVE_FILES: + for af in CFG_DRIVE_FILES_V1: fn = os.path.join(source_dir, af) if os.path.isfile(fn): found[af] = fn @@ -173,11 +302,10 @@ def read_config_drive_dir(source_dir): raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found")) md = {} - ud = "" keydata = "" if "etc/network/interfaces" in found: fn = found["etc/network/interfaces"] - md['network-interfaces'] = util.load_file(fn) + md['network_config'] = util.load_file(fn) if "root/.ssh/authorized_keys" in found: fn = found["root/.ssh/authorized_keys"] @@ -197,21 +325,77 @@ def read_config_drive_dir(source_dir): (source_dir, "invalid json in meta.js", e)) md['meta_js'] = content - # Key data override?? + # keydata in meta_js is preferred over "injected" keydata = meta_js.get('public-keys', keydata) if keydata: lines = keydata.splitlines() md['public-keys'] = [l for l in lines if len(l) and not l.startswith("#")] - for copy in ('dsmode', 'instance-id', 'dscfg'): - if copy in meta_js: - md[copy] = meta_js[copy] + # config-drive-v1 has no way for openstack to provide the instance-id + # so we copy that into metadata from the user input + if 'instance-id' in meta_js: + md['instance-id'] = meta_js['instance-id'] + + results = {'cfgdrive_ver': 1, 'metadata': md} + + # allow the user to specify 'dsmode' in a meta tag + if 'dsmode' in meta_js: + results['dsmode'] = meta_js['dsmode'] + + # config-drive-v1 has no way of specifying user-data, so the user has + # to cheat and stuff it in a meta tag also. + results['userdata'] = meta_js.get('user-data') - if 'user-data' in meta_js: - ud = meta_js['user-data'] + # this implementation does not support files + # (other than network/interfaces and authorized_keys) + results['files'] = [] - return (md, ud) + return results + + +def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): + """Determine what mode should be used. + valid values are 'pass', 'disabled', 'local', 'net' + """ + # user passed data trumps everything + if user is not None: + return user + + if ds_cfg is not None: + return ds_cfg + + # at config-drive version 1, the default behavior was pass. That + # meant to not use use it as primary data source, but expect a ec2 metadata + # source. for version 2, we default to 'net', which means + # the DataSourceConfigDriveNet, would be used. + # + # this could change in the future. If there was definitive metadata + # that indicated presense of an openstack metadata service, then + # we could change to 'pass' by default also. The motivation for that + # would be 'cloud-init query' as the web service could be more dynamic + if cfgdrv_ver == 1: + return "pass" + return "net" + + +def get_previous_iid(paths): + # interestingly, for this purpose the "previous" instance-id is the current + # instance-id. cloud-init hasn't moved them over yet as this datasource + # hasn't declared itself found. + fname = os.path.join(paths.get_cpath('data'), 'instance-id') + try: + with open(fname) as fp: + return fp.read() + except IOError: + return None + + +def write_files(files): + for (name, content) in files.iteritems(): + if name[0] != os.sep: + name = os.sep + name + util.write_file(name, content, mode=0660) # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index d9eb8f17..7e845571 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -40,7 +40,7 @@ DEF_MD_VERSION = '2009-04-04' # Default metadata urls that will be used if none are provided # They will be checked for 'resolveability' and some of the # following may be discarded if they do not resolve -DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"] +DEF_MD_URLS = [DEF_MD_URL, "http://instance-data.:8773"] class DataSourceEc2(sources.DataSource): @@ -83,40 +83,6 @@ class DataSourceEc2(sources.DataSource): def get_availability_zone(self): return self.metadata['placement']['availability-zone'] - def get_local_mirror(self): - return self.get_mirror_from_availability_zone() - - def get_mirror_from_availability_zone(self, availability_zone=None): - # Return type None indicates there is no cloud specific mirror - # Availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone is None: - availability_zone = self.get_availability_zone() - - if self.is_vpc(): - return None - - if not availability_zone: - return None - - mirror_tpl = self.distro.get_option('package_mirror_ec2_template', - None) - - if mirror_tpl is None: - return None - - # in EC2, the 'region' is 'us-east-1' if 'zone' is 'us-east-1a' - tpl_params = { - 'zone': availability_zone.strip(), - 'region': availability_zone[:-1] - } - mirror_url = mirror_tpl % (tpl_params) - - found = util.search_for_mirror([mirror_url]) - if found is not None: - return mirror_url - - return None - def _get_url_settings(self): mcfg = self.ds_cfg if not mcfg: @@ -255,6 +221,12 @@ class DataSourceEc2(sources.DataSource): return True return False + @property + def availability_zone(self): + try: + return self.metadata['placement']['availability-zone'] + except KeyError: + return None # Used to match classes to dependencies datasources = [ diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index f16d5c21..c568d365 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -262,3 +262,94 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ + import argparse + import pprint + + parser = argparse.ArgumentParser(description='Interact with MAAS DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.safe_load(fp) + if 'datasource' in cfg: + cfg = cfg['datasource']['MAAS'] + for key in creds.keys(): + if key in cfg and creds[key] is None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) + + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) is not None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) + + main() diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py new file mode 100644 index 00000000..c2125bee --- /dev/null +++ b/cloudinit/sources/DataSourceNone.py @@ -0,0 +1,61 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +class DataSourceNone(sources.DataSource): + def __init__(self, sys_cfg, distro, paths, ud_proc=None): + sources.DataSource.__init__(self, sys_cfg, distro, paths, ud_proc) + self.metadata = {} + self.userdata_raw = '' + + def get_data(self): + # If the datasource config has any provided 'fallback' + # userdata or metadata, use it... + if 'userdata_raw' in self.ds_cfg: + self.userdata_raw = self.ds_cfg['userdata_raw'] + if 'metadata' in self.ds_cfg: + self.metadata = self.ds_cfg['metadata'] + return True + + def get_instance_id(self): + return 'iid-datasource-none' + + def __str__(self): + return util.obj_name(self) + + @property + def is_disconnected(self): + return True + + +# Used to match classes to dependencies +datasources = [ + (DataSourceNone, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceNone, []), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index b25724a5..4719d254 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -65,6 +65,10 @@ class DataSource(object): self.userdata = self.ud_proc.process(raw_data) return self.userdata + @property + def is_disconnected(self): + return False + def get_userdata_raw(self): return self.userdata_raw @@ -113,9 +117,9 @@ class DataSource(object): def get_locale(self): return 'en_US.UTF-8' - def get_local_mirror(self): - # ?? - return None + @property + def availability_zone(self): + return self.metadata.get('availability-zone') def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: @@ -162,6 +166,10 @@ class DataSource(object): else: return hostname + def get_package_mirror_info(self): + return self.distro.get_package_mirror_info( + availability_zone=self.availability_zone) + def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index e0a2f0ca..88a11a1a 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -181,12 +181,11 @@ def parse_authorized_keys(fname): return contents -def update_authorized_keys(fname, keys): - entries = parse_authorized_keys(fname) +def update_authorized_keys(old_entries, keys): to_add = list(keys) - for i in range(0, len(entries)): - ent = entries[i] + for i in range(0, len(old_entries)): + ent = old_entries[i] if ent.empty() or not ent.base64: continue # Replace those with the same base64 @@ -199,66 +198,81 @@ def update_authorized_keys(fname, keys): # Don't add it later if k in to_add: to_add.remove(k) - entries[i] = ent + old_entries[i] = ent # Now append any entries we did not match above for key in to_add: - entries.append(key) + old_entries.append(key) # Now format them back to strings... - lines = [str(b) for b in entries] + lines = [str(b) for b in old_entries] # Ensure it ends with a newline lines.append('') return '\n'.join(lines) -def setup_user_keys(keys, user, key_prefix, paths): - # Make sure the users .ssh dir is setup accordingly - pwent = pwd.getpwnam(user) - ssh_dir = os.path.join(pwent.pw_dir, '.ssh') - ssh_dir = paths.join(False, ssh_dir) - if not os.path.exists(ssh_dir): - util.ensure_dir(ssh_dir, mode=0700) - util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid) +def users_ssh_info(username, paths): + pw_ent = pwd.getpwnam(username) + if not pw_ent: + raise RuntimeError("Unable to get ssh info for user %r" % (username)) + ssh_dir = paths.join(False, os.path.join(pw_ent.pw_dir, '.ssh')) + return (ssh_dir, pw_ent) - # Turn the keys given into actual entries - parser = AuthKeyLineParser() - key_entries = [] - for k in keys: - key_entries.append(parser.parse(str(k), def_opt=key_prefix)) +def extract_authorized_keys(username, paths): + (ssh_dir, pw_ent) = users_ssh_info(username, paths) sshd_conf_fn = paths.join(True, DEF_SSHD_CFG) + auth_key_fn = None with util.SeLinuxGuard(ssh_dir, recursive=True): try: - # AuthorizedKeysFile may contain tokens + # The 'AuthorizedKeysFile' may contain tokens # of the form %T which are substituted during connection set-up. # The following tokens are defined: %% is replaced by a literal # '%', %h is replaced by the home directory of the user being # authenticated and %u is replaced by the username of that user. ssh_cfg = parse_ssh_config_map(sshd_conf_fn) - akeys = ssh_cfg.get("authorizedkeysfile", '') - akeys = akeys.strip() - if not akeys: - akeys = "%h/.ssh/authorized_keys" - akeys = akeys.replace("%h", pwent.pw_dir) - akeys = akeys.replace("%u", user) - akeys = akeys.replace("%%", '%') - if not akeys.startswith('/'): - akeys = os.path.join(pwent.pw_dir, akeys) - authorized_keys = paths.join(False, akeys) + auth_key_fn = ssh_cfg.get("authorizedkeysfile", '').strip() + if not auth_key_fn: + auth_key_fn = "%h/.ssh/authorized_keys" + auth_key_fn = auth_key_fn.replace("%h", pw_ent.pw_dir) + auth_key_fn = auth_key_fn.replace("%u", username) + auth_key_fn = auth_key_fn.replace("%%", '%') + if not auth_key_fn.startswith('/'): + auth_key_fn = os.path.join(pw_ent.pw_dir, auth_key_fn) + auth_key_fn = paths.join(False, auth_key_fn) except (IOError, OSError): - authorized_keys = os.path.join(ssh_dir, 'authorized_keys') + # Give up and use a default key filename + auth_key_fn = os.path.join(ssh_dir, 'authorized_keys') util.logexc(LOG, ("Failed extracting 'AuthorizedKeysFile'" " in ssh config" - " from %s, using 'AuthorizedKeysFile' file" - " %s instead"), - sshd_conf_fn, authorized_keys) - - content = update_authorized_keys(authorized_keys, key_entries) - util.ensure_dir(os.path.dirname(authorized_keys), mode=0700) - util.write_file(authorized_keys, content, mode=0600) - util.chownbyid(authorized_keys, pwent.pw_uid, pwent.pw_gid) + " from %r, using 'AuthorizedKeysFile' file" + " %r instead"), + sshd_conf_fn, auth_key_fn) + auth_key_entries = parse_authorized_keys(auth_key_fn) + return (auth_key_fn, auth_key_entries) + + +def setup_user_keys(keys, username, key_prefix, paths): + # Make sure the users .ssh dir is setup accordingly + (ssh_dir, pwent) = users_ssh_info(username, paths) + if not os.path.isdir(ssh_dir): + util.ensure_dir(ssh_dir, mode=0700) + util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid) + + # Turn the 'update' keys given into actual entries + parser = AuthKeyLineParser() + key_entries = [] + for k in keys: + key_entries.append(parser.parse(str(k), def_opt=key_prefix)) + + # Extract the old and make the new + (auth_key_fn, auth_key_entries) = extract_authorized_keys(username, paths) + with util.SeLinuxGuard(ssh_dir, recursive=True): + content = update_authorized_keys(auth_key_entries, key_entries) + util.ensure_dir(os.path.dirname(auth_key_fn), mode=0700) + util.write_file(auth_key_fn, content, mode=0600) + util.chownbyid(auth_key_fn, pwent.pw_uid, pwent.pw_gid) class SshdConfigLine(object): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2f6a566c..c9634a90 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -326,7 +326,7 @@ class Init(object): 'paths': self.paths, 'datasource': self.datasource, } - # TODO Hmmm, should we dynamically import these?? + # TODO(harlowja) Hmmm, should we dynamically import these?? def_handlers = [ cc_part.CloudConfigPartHandler(**opts), ss_part.ShellScriptPartHandler(**opts), @@ -519,7 +519,7 @@ class Modules(object): " but not on %s distro. It may or may not work" " correctly."), name, worked_distros, d_name) # Use the configs logger and not our own - # TODO: possibly check the module + # TODO(harlowja): possibly check the module # for having a LOG attr and just give it back # its own logger? func_args = [name, self.cfg, diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f5d01818..af98b488 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -23,9 +23,9 @@ import os import email +from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.mime.base import MIMEBase from cloudinit import handlers from cloudinit import log as logging @@ -159,7 +159,7 @@ class UserDataProcessor(object): if isinstance(ent, (str, basestring)): ent = {'content': ent} if not isinstance(ent, (dict)): - # TODO raise? + # TODO(harlowja) raise? continue content = ent.get('content', '') diff --git a/cloudinit/util.py b/cloudinit/util.py index 6eb2a10e..6872cc31 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -24,8 +24,8 @@ from StringIO import StringIO -import copy as obj_copy import contextlib +import copy as obj_copy import errno import glob import grp @@ -55,6 +55,7 @@ from cloudinit import url_helper as uhelp from cloudinit.settings import (CFG_BUILTIN) +_DNS_REDIRECT_IP = None LOG = logging.getLogger(__name__) # Helps cleanup filenames to ensure they aren't FS incompatible @@ -316,8 +317,9 @@ def multi_log(text, console=True, stderr=True, else: log.log(log_level, text) + def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" + """determine if input string is a ipv4 address. return boolean.""" toks = instr.split('.') if len(toks) != 4: return False @@ -825,9 +827,43 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), def is_resolvable(name): - """ determine if a url is resolvable, return a boolean """ + """determine if a url is resolvable, return a boolean + This also attempts to be resilent against dns redirection. + + Note, that normal nsswitch resolution is used here. So in order + to avoid any utilization of 'search' entries in /etc/resolv.conf + we have to append '.'. + + The top level 'invalid' domain is invalid per RFC. And example.com + should also not exist. The random entry will be resolved inside + the search list. + """ + global _DNS_REDIRECT_IP # pylint: disable=W0603 + if _DNS_REDIRECT_IP is None: + badips = set() + badnames = ("does-not-exist.example.com.", "example.invalid.", + rand_str()) + badresults = {} + for iname in badnames: + try: + result = socket.getaddrinfo(iname, None, 0, 0, + socket.SOCK_STREAM, socket.AI_CANONNAME) + badresults[iname] = [] + for (_fam, _stype, _proto, cname, sockaddr) in result: + badresults[iname].append("%s: %s" % (cname, sockaddr[0])) + badips.add(sockaddr[0]) + except socket.gaierror: + pass + _DNS_REDIRECT_IP = badips + if badresults: + LOG.debug("detected dns redirection: %s" % badresults) + try: - socket.getaddrinfo(name, None) + result = socket.getaddrinfo(name, None) + # check first result's sockaddr field + addr = result[0][4][0] + if addr in _DNS_REDIRECT_IP: + return False return True except socket.gaierror: return False @@ -839,7 +875,7 @@ def get_hostname(): def is_resolvable_url(url): - """ determine if this url is resolvable (existing or ip) """ + """determine if this url is resolvable (existing or ip).""" return (is_resolvable(urlparse.urlparse(url).hostname)) @@ -1070,7 +1106,7 @@ def hash_blob(blob, routine, mlen=None): def rename(src, dest): LOG.debug("Renaming %s to %s", src, dest) - # TODO use a se guard here?? + # TODO(harlowja) use a se guard here?? os.rename(src, dest) @@ -1294,12 +1330,19 @@ def delete_dir_contents(dirname): del_file(node_fullpath) -def subp(args, data=None, rcs=None, env=None, capture=True, shell=False): +def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, + logstring=False): if rcs is None: rcs = [0] try: - LOG.debug(("Running command %s with allowed return codes %s" - " (shell=%s, capture=%s)"), args, rcs, shell, capture) + + if not logstring: + LOG.debug(("Running command %s with allowed return codes %s" + " (shell=%s, capture=%s)"), args, rcs, shell, capture) + else: + LOG.debug(("Running hidden command to protect sensitive " + "input/output logstring: %s"), logstring) + if not capture: stdout = None stderr = None |