diff options
Diffstat (limited to 'cloudinit')
33 files changed, 1511 insertions, 409 deletions
diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 896cb4d0..3ac22967 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -50,6 +50,5 @@ def handle(name, cfg, cloud, log, _args): cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) except: - util.logexc(log, - ("Failed to run bootcmd module %s"), name) + util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index b6e1fd37..4f8c8f80 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -121,15 +123,15 @@ class ResizeGrowPart(object): util.subp(["growpart", '--dry-run', diskdev, partnum]) except util.ProcessExecutionError as e: if e.exit_code != 1: - util.logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" % - (diskdev, partnum))) + util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", + diskdev, partnum) raise ResizeFailedException(e) return (before, before) try: util.subp(["growpart", diskdev, partnum]) except util.ProcessExecutionError as e: - util.logexc(LOG, "Failed: growpart %s %s" % (diskdev, partnum)) + util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) raise ResizeFailedException(e) return (before, get_size(partdev)) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index c873c8a8..2e058ccd 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -65,8 +65,8 @@ def handle(name, cfg, cloud, log, args): tries = int(tries) except: tries = 10 - util.logexc(log, ("Configuration entry 'tries'" - " is not an integer, using %s instead"), tries) + util.logexc(log, "Configuration entry 'tries' is not an integer, " + "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL @@ -85,8 +85,8 @@ def handle(name, cfg, cloud, log, args): try: all_keys[n] = util.load_file(path) except: - util.logexc(log, ("%s: failed to open, can not" - " phone home that data!"), path) + util.logexc(log, "%s: failed to open, can not phone home that " + "data!", path) submit_keys = {} for k in post_list: @@ -115,5 +115,5 @@ def handle(name, cfg, cloud, log, args): retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) except: - util.logexc(log, ("Failed to post phone home data to" - " %s in %s tries"), url, tries) + util.logexc(log, "Failed to post phone home data to %s in %s tries", + url, tries) diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 8a460f7e..879b62b1 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2013 Craig Tracey +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. # # Author: Craig Tracey <craigtracey@gmail.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -53,7 +55,7 @@ from cloudinit import util frequency = PER_INSTANCE -distros = ['fedora', 'rhel'] +distros = ['fedora', 'rhel', 'sles'] def generate_resolv_conf(cloud, log, params): diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 4bf18516..c771728d 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -64,8 +64,8 @@ def handle(name, _cfg, cloud, log, _args): " raw userdata"), name, MY_HOOKNAME) return except: - util.logexc(log, ("Failed to parse query string %s" - " into a dictionary"), ud) + util.logexc(log, "Failed to parse query string %s into a dictionary", + ud) raise wrote_fns = [] @@ -86,8 +86,8 @@ def handle(name, _cfg, cloud, log, _args): wrote_fns.append(fname) except Exception as e: captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", - MY_NAME, url, fname) + util.logexc(log, "%s failed to read %s and write %s", MY_NAME, url, + fname) if wrote_fns: log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 2b32fc94..5d7f4331 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -32,6 +32,6 @@ def handle(name, cfg, cloud, log, _args): log.debug("Setting the hostname to %s (%s)", fqdn, hostname) cloud.distro.set_hostname(hostname, fqdn) except Exception: - util.logexc(log, "Failed to set the hostname to %s (%s)", - fqdn, hostname) + util.logexc(log, "Failed to set the hostname to %s (%s)", fqdn, + hostname) raise diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index c6bf62fd..56a36906 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -75,14 +75,14 @@ def handle(_name, cfg, cloud, log, args): plist_in.append("%s:%s" % (u, p)) users.append(u) - ch_in = '\n'.join(plist_in) + ch_in = '\n'.join(plist_in) + '\n' try: log.debug("Changing password for %s:", users) util.subp(['chpasswd'], ch_in) except Exception as e: errors.append(e) - util.logexc(log, - "Failed to set passwords with chpasswd for %s", users) + util.logexc(log, "Failed to set passwords with chpasswd for %s", + users) if len(randlist): blurb = ("Set the following 'random' passwords\n", diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 7ef20d9f..64a5e3cb 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -85,8 +85,8 @@ def handle(_name, cfg, cloud, log, _args): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except: - util.logexc(log, ("Failed generated a key" - " for %s from %s"), pair[0], pair[1]) + util.logexc(log, "Failed generated a key for %s from %s", + pair[0], pair[1]) else: # if not, generate them genkeys = util.get_cfg_option_list(cfg, @@ -102,8 +102,8 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: - util.logexc(log, ("Failed generating key type" - " %s to file %s"), keytype, keyfile) + util.logexc(log, "Failed generating key type %s to " + "file %s", keytype, keyfile) try: (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 83af36e9..50d96e15 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -71,8 +71,8 @@ def handle(_name, cfg, cloud, log, args): try: import_ssh_ids(import_ids, user, log) except Exception as exc: - util.logexc(log, "ssh-import-id failed for: %s %s" % - (user, import_ids), exc) + util.logexc(log, "ssh-import-id failed for: %s %s", user, + import_ids) elist.append(exc) if len(elist): diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 52225cd8..e396ba13 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser <scott.moser@canonical.com> # Author: Juerg Haefliger <juerg.haefliger@hp.com> @@ -38,6 +38,6 @@ def handle(name, cfg, cloud, log, _args): log.debug("Updating hostname to %s (%s)", fqdn, hostname) cloud.distro.update_hostname(hostname, fqdn, prev_fn) except Exception: - util.logexc(log, "Failed to update the hostname to %s (%s)", - fqdn, hostname) + util.logexc(log, "Failed to update the hostname to %s (%s)", fqdn, + hostname) raise diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 50d52594..249e1b19 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -38,7 +38,8 @@ from cloudinit.distros.parsers import hosts OSFAMILIES = { 'debian': ['debian', 'ubuntu'], - 'redhat': ['fedora', 'rhel'] + 'redhat': ['fedora', 'rhel'], + 'suse': ['sles'] } LOG = logging.getLogger(__name__) @@ -142,8 +143,8 @@ class Distro(object): try: util.subp(['hostname', hostname]) except util.ProcessExecutionError: - util.logexc(LOG, ("Failed to non-persistently adjust" - " the system hostname to %s"), hostname) + util.logexc(LOG, "Failed to non-persistently adjust the system " + "hostname to %s", hostname) @abc.abstractmethod def _select_hostname(self, hostname, fqdn): @@ -200,8 +201,8 @@ class Distro(object): try: self._write_hostname(hostname, fn) except IOError: - util.logexc(LOG, "Failed to write hostname %s to %s", - hostname, fn) + util.logexc(LOG, "Failed to write hostname %s to %s", hostname, + fn) if (sys_hostname and prev_hostname and sys_hostname != prev_hostname): @@ -281,15 +282,16 @@ class Distro(object): def get_default_user(self): return self.get_option('default_user') - def create_user(self, name, **kwargs): + def add_user(self, name, **kwargs): """ - Creates users for the system using the GNU passwd tools. This - will work on an GNU system. This should be overriden on - distros where useradd is not desirable or not available. + Add a user to the system using standard GNU tools """ + if util.is_user(name): + LOG.info("User %s already exists, skipping." % name) + return adduser_cmd = ['useradd', name] - x_adduser_cmd = ['useradd', name] + log_adduser_cmd = ['useradd', name] # Since we are creating users, we want to carefully validate the # inputs. If something goes wrong, we can end up with a system @@ -306,63 +308,65 @@ class Distro(object): "selinux_user": '--selinux-user', } - adduser_opts_flags = { + adduser_flags = { "no_user_group": '--no-user-group', "system": '--system', "no_log_init": '--no-log-init', - "no_create_home": "-M", } - redact_fields = ['passwd'] + redact_opts = ['passwd'] + + # Check the values and create the command + for key, val in kwargs.iteritems(): + + if key in adduser_opts and val and isinstance(val, str): + adduser_cmd.extend([adduser_opts[key], val]) - # Now check the value and create the command - for option in kwargs: - value = kwargs[option] - if option in adduser_opts and value \ - and isinstance(value, str): - adduser_cmd.extend([adduser_opts[option], value]) - # Redact certain fields from the logs - if option in redact_fields: - x_adduser_cmd.extend([adduser_opts[option], 'REDACTED']) - else: - x_adduser_cmd.extend([adduser_opts[option], value]) - elif option in adduser_opts_flags and value: - adduser_cmd.append(adduser_opts_flags[option]) # Redact certain fields from the logs - if option in redact_fields: - x_adduser_cmd.append('REDACTED') + if key in redact_opts: + log_adduser_cmd.extend([adduser_opts[key], 'REDACTED']) else: - x_adduser_cmd.append(adduser_opts_flags[option]) + log_adduser_cmd.extend([adduser_opts[key], val]) - # Default to creating home directory unless otherwise directed - # Also, we do not create home directories for system users. - if "no_create_home" not in kwargs and "system" not in kwargs: - adduser_cmd.append('-m') + elif key in adduser_flags and val: + adduser_cmd.append(adduser_flags[key]) + log_adduser_cmd.append(adduser_flags[key]) - # Create the user - if util.is_user(name): - LOG.warn("User %s already exists, skipping." % name) + # Don't create the home directory if directed so or if the user is a + # system user + if 'no_create_home' in kwargs or 'system' in kwargs: + adduser_cmd.append('-M') + log_adduser_cmd.append('-M') else: - LOG.debug("Adding user named %s", name) - try: - util.subp(adduser_cmd, logstring=x_adduser_cmd) - except Exception as e: - util.logexc(LOG, "Failed to create user %s due to error.", e) - raise e + adduser_cmd.append('-m') + log_adduser_cmd.append('-m') + + # Run the command + LOG.debug("Adding user %s", name) + try: + util.subp(adduser_cmd, logstring=log_adduser_cmd) + except Exception as e: + util.logexc(LOG, "Failed to create user %s", name) + raise e - # Set password if plain-text password provided + def create_user(self, name, **kwargs): + """ + Creates users for the system using the GNU passwd tools. This + will work on an GNU system. This should be overriden on + distros where useradd is not desirable or not available. + """ + + # Add the user + self.add_user(name, **kwargs) + + # Set password if plain-text password provided and non-empty if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']: self.set_passwd(name, kwargs['plain_text_passwd']) # Default locking down the account. 'lock_passwd' defaults to True. # lock account unless lock_password is False. if kwargs.get('lock_passwd', True): - try: - util.subp(['passwd', '--lock', name]) - except Exception as e: - util.logexc(LOG, ("Failed to disable password logins for" - "user %s" % name), e) - raise e + self.lock_passwd(name) # Configure sudo access if 'sudo' in kwargs: @@ -375,17 +379,33 @@ class Distro(object): return True + def lock_passwd(self, name): + """ + Lock the password of a user, i.e., disable password logins + """ + try: + # Need to use the short option name '-l' instead of '--lock' + # (which would be more descriptive) since SLES 11 doesn't know + # about long names. + util.subp(['passwd', '-l', name]) + except Exception as e: + util.logexc(LOG, 'Failed to disable password for user %s', name) + raise e + def set_passwd(self, user, passwd, hashed=False): pass_string = '%s:%s' % (user, passwd) cmd = ['chpasswd'] if hashed: - cmd.append('--encrypted') + # Need to use the short option name '-e' instead of '--encrypted' + # (which would be more descriptive) since SLES 11 doesn't know + # about long names. + cmd.append('-e') try: util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) except Exception as e: - util.logexc(LOG, "Failed to set password for %s" % user) + util.logexc(LOG, "Failed to set password for %s", user) raise e return True @@ -427,7 +447,7 @@ class Distro(object): util.append_file(sudo_base, sudoers_contents) LOG.debug("Added '#includedir %s' to %s" % (path, sudo_base)) except IOError as e: - util.logexc(LOG, "Failed to write %s" % sudo_base, e) + util.logexc(LOG, "Failed to write %s", sudo_base) raise e util.ensure_dir(path, 0750) @@ -478,15 +498,15 @@ class Distro(object): try: util.subp(group_add_cmd) LOG.info("Created new group %s" % name) - except Exception as e: - util.logexc("Failed to create group %s" % name, e) + except Exception: + util.logexc("Failed to create group %s", name) # Add members to the group, if so defined if len(members) > 0: for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist." % (member, name)) + "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 174da3ab..a022ca60 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -23,14 +23,11 @@ import os from cloudinit import distros - -from cloudinit.distros.parsers.resolv_conf import ResolvConf -from cloudinit.distros.parsers.sys_conf import SysConf - from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit.distros import rhel_util from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -67,33 +64,9 @@ class Distro(distros.Distro): def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) - def _adjust_resolve(self, dns_servers, search_servers): - try: - r_conf = ResolvConf(util.load_file(self.resolve_conf_fn)) - r_conf.parse() - except IOError: - util.logexc(LOG, - "Failed at parsing %s reverting to an empty instance", - self.resolve_conf_fn) - r_conf = ResolvConf('') - r_conf.parse() - if dns_servers: - for s in dns_servers: - try: - r_conf.add_nameserver(s) - except ValueError: - util.logexc(LOG, "Failed at adding nameserver %s", s) - if search_servers: - for s in search_servers: - try: - r_conf.add_search_domain(s) - except ValueError: - util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(self.resolve_conf_fn, str(r_conf), 0644) - def _write_network(self, settings): # TODO(harlowja) fix this... since this is the ubuntu format - entries = translate_network(settings) + entries = rhel_util.translate_network(settings) LOG.debug("Translated ubuntu style network settings %s into %s", settings, entries) # Make the intermediate format as the rhel format... @@ -112,41 +85,21 @@ class Distro(distros.Distro): 'MACADDR': info.get('hwaddress'), 'ONBOOT': _make_sysconfig_bool(info.get('auto')), } - self._update_sysconfig_file(net_fn, net_cfg) + rhel_util.update_sysconfig_file(net_fn, net_cfg) if 'dns-nameservers' in info: nameservers.extend(info['dns-nameservers']) if 'dns-search' in info: searchservers.extend(info['dns-search']) if nameservers or searchservers: - self._adjust_resolve(nameservers, searchservers) + rhel_util.update_resolve_conf_file(self.resolve_conf_fn, + nameservers, searchservers) if dev_names: net_cfg = { 'NETWORKING': _make_sysconfig_bool(True), } - self._update_sysconfig_file(self.network_conf_fn, net_cfg) + rhel_util.update_sysconfig_file(self.network_conf_fn, net_cfg) return dev_names - def _update_sysconfig_file(self, fn, adjustments, allow_empty=False): - if not adjustments: - return - (exists, contents) = self._read_conf(fn) - updated_am = 0 - for (k, v) in adjustments.items(): - if v is None: - continue - v = str(v) - if len(v) == 0 and not allow_empty: - continue - contents[k] = v - updated_am += 1 - if updated_am: - lines = [ - str(contents), - ] - if not exists: - lines.insert(0, util.make_header()) - util.write_file(fn, "\n".join(lines) + "\n", 0644) - def _dist_uses_systemd(self): # Fedora 18 and RHEL 7 were the first adopters in their series (dist, vers) = util.system_info()['dist'][:2] @@ -165,7 +118,7 @@ class Distro(distros.Distro): locale_cfg = { 'LANG': locale, } - self._update_sysconfig_file(out_fn, locale_cfg) + rhel_util.update_sysconfig_file(out_fn, locale_cfg) def _write_hostname(self, hostname, out_fn): if self._dist_uses_systemd(): @@ -174,7 +127,7 @@ class Distro(distros.Distro): host_cfg = { 'HOSTNAME': hostname, } - self._update_sysconfig_file(out_fn, host_cfg) + rhel_util.update_sysconfig_file(out_fn, host_cfg) def _select_hostname(self, hostname, fqdn): # See: http://bit.ly/TwitgL @@ -198,22 +151,12 @@ class Distro(distros.Distro): else: return default else: - (_exists, contents) = self._read_conf(filename) + (_exists, contents) = rhel_util.read_sysconfig_file(filename) if 'HOSTNAME' in contents: return contents['HOSTNAME'] else: return default - def _read_conf(self, fn): - exists = False - try: - contents = util.load_file(fn).splitlines() - exists = True - except IOError: - contents = [] - return (exists, - SysConf(contents)) - def _bring_up_interfaces(self, device_names): if device_names and 'all' in device_names: raise RuntimeError(('Distro %s can not translate ' @@ -237,7 +180,7 @@ class Distro(distros.Distro): clock_cfg = { 'ZONE': str(tz), } - self._update_sysconfig_file(self.clock_conf_fn, clock_cfg) + rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) @@ -272,90 +215,3 @@ class Distro(distros.Distro): def update_package_sources(self): self._runner.run("update-sources", self.package_command, ["makecache"], freq=PER_INSTANCE) - - -# This is a util function to translate a ubuntu /etc/network/interfaces 'blob' -# to a rhel equiv. that can then be written to /etc/sysconfig/network-scripts/ -# TODO(harlowja) remove when we have python-netcf active... -def translate_network(settings): - # Get the standard cmd, args from the ubuntu format - entries = [] - for line in settings.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - split_up = line.split(None, 1) - if len(split_up) <= 1: - continue - entries.append(split_up) - # Figure out where each iface section is - ifaces = [] - consume = {} - for (cmd, args) in entries: - if cmd == 'iface': - if consume: - ifaces.append(consume) - consume = {} - consume[cmd] = args - else: - consume[cmd] = args - # Check if anything left over to consume - absorb = False - for (cmd, args) in consume.iteritems(): - if cmd == 'iface': - absorb = True - if absorb: - ifaces.append(consume) - # Now translate - real_ifaces = {} - for info in ifaces: - if 'iface' not in info: - continue - iface_details = info['iface'].split(None) - dev_name = None - if len(iface_details) >= 1: - dev = iface_details[0].strip().lower() - if dev: - dev_name = dev - if not dev_name: - continue - iface_info = {} - if len(iface_details) >= 3: - proto_type = iface_details[2].strip().lower() - # Seems like this can be 'loopback' which we don't - # really care about - if proto_type in ['dhcp', 'static']: - iface_info['bootproto'] = proto_type - # These can just be copied over - for k in ['netmask', 'address', 'gateway', 'broadcast']: - if k in info: - val = info[k].strip().lower() - if val: - iface_info[k] = val - # Name server info provided?? - if 'dns-nameservers' in info: - iface_info['dns-nameservers'] = info['dns-nameservers'].split() - # Name server search info provided?? - if 'dns-search' in info: - iface_info['dns-search'] = info['dns-search'].split() - # Is any mac address spoofing going on?? - if 'hwaddress' in info: - hw_info = info['hwaddress'].lower().strip() - hw_split = hw_info.split(None, 1) - if len(hw_split) == 2 and hw_split[0].startswith('ether'): - hw_addr = hw_split[1] - if hw_addr: - iface_info['hwaddress'] = hw_addr - real_ifaces[dev_name] = iface_info - # Check for those that should be started on boot via 'auto' - for (cmd, args) in entries: - if cmd == 'auto': - # Seems like auto can be like 'auto eth0 eth0:1' so just get the - # first part out as the device name - args = args.split(None) - if not args: - continue - dev_name = args[0].strip().lower() - if dev_name in real_ifaces: - real_ifaces[dev_name]['auto'] = True - return real_ifaces diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py new file mode 100644 index 00000000..1aba58b8 --- /dev/null +++ b/cloudinit/distros/rhel_util.py @@ -0,0 +1,177 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser <scott.moser@canonical.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# + +from cloudinit.distros.parsers.resolv_conf import ResolvConf +from cloudinit.distros.parsers.sys_conf import SysConf + +from cloudinit import log as logging +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +# This is a util function to translate Debian based distro interface blobs as +# given in /etc/network/interfaces to an equivalent format for distributions +# that use ifcfg-* style (Red Hat and SUSE). +# TODO(harlowja) remove when we have python-netcf active... +def translate_network(settings): + # Get the standard cmd, args from the ubuntu format + entries = [] + for line in settings.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + split_up = line.split(None, 1) + if len(split_up) <= 1: + continue + entries.append(split_up) + # Figure out where each iface section is + ifaces = [] + consume = {} + for (cmd, args) in entries: + if cmd == 'iface': + if consume: + ifaces.append(consume) + consume = {} + consume[cmd] = args + else: + consume[cmd] = args + # Check if anything left over to consume + absorb = False + for (cmd, args) in consume.iteritems(): + if cmd == 'iface': + absorb = True + if absorb: + ifaces.append(consume) + # Now translate + real_ifaces = {} + for info in ifaces: + if 'iface' not in info: + continue + iface_details = info['iface'].split(None) + dev_name = None + if len(iface_details) >= 1: + dev = iface_details[0].strip().lower() + if dev: + dev_name = dev + if not dev_name: + continue + iface_info = {} + if len(iface_details) >= 3: + proto_type = iface_details[2].strip().lower() + # Seems like this can be 'loopback' which we don't + # really care about + if proto_type in ['dhcp', 'static']: + iface_info['bootproto'] = proto_type + # These can just be copied over + for k in ['netmask', 'address', 'gateway', 'broadcast']: + if k in info: + val = info[k].strip().lower() + if val: + iface_info[k] = val + # Name server info provided?? + if 'dns-nameservers' in info: + iface_info['dns-nameservers'] = info['dns-nameservers'].split() + # Name server search info provided?? + if 'dns-search' in info: + iface_info['dns-search'] = info['dns-search'].split() + # Is any mac address spoofing going on?? + if 'hwaddress' in info: + hw_info = info['hwaddress'].lower().strip() + hw_split = hw_info.split(None, 1) + if len(hw_split) == 2 and hw_split[0].startswith('ether'): + hw_addr = hw_split[1] + if hw_addr: + iface_info['hwaddress'] = hw_addr + real_ifaces[dev_name] = iface_info + # Check for those that should be started on boot via 'auto' + for (cmd, args) in entries: + if cmd == 'auto': + # Seems like auto can be like 'auto eth0 eth0:1' so just get the + # first part out as the device name + args = args.split(None) + if not args: + continue + dev_name = args[0].strip().lower() + if dev_name in real_ifaces: + real_ifaces[dev_name]['auto'] = True + return real_ifaces + + +# Helper function to update a RHEL/SUSE /etc/sysconfig/* file +def update_sysconfig_file(fn, adjustments, allow_empty=False): + if not adjustments: + return + (exists, contents) = read_sysconfig_file(fn) + updated_am = 0 + for (k, v) in adjustments.items(): + if v is None: + continue + v = str(v) + if len(v) == 0 and not allow_empty: + continue + contents[k] = v + updated_am += 1 + if updated_am: + lines = [ + str(contents), + ] + if not exists: + lines.insert(0, util.make_header()) + util.write_file(fn, "\n".join(lines) + "\n", 0644) + + +# Helper function to read a RHEL/SUSE /etc/sysconfig/* file +def read_sysconfig_file(fn): + exists = False + try: + contents = util.load_file(fn).splitlines() + exists = True + except IOError: + contents = [] + return (exists, SysConf(contents)) + + +# Helper function to update RHEL/SUSE /etc/resolv.conf +def update_resolve_conf_file(fn, dns_servers, search_servers): + try: + r_conf = ResolvConf(util.load_file(fn)) + r_conf.parse() + except IOError: + util.logexc(LOG, "Failed at parsing %s reverting to an empty " + "instance", fn) + r_conf = ResolvConf('') + r_conf.parse() + if dns_servers: + for s in dns_servers: + try: + r_conf.add_nameserver(s) + except ValueError: + util.logexc(LOG, "Failed at adding nameserver %s", s) + if search_servers: + for s in search_servers: + try: + r_conf.add_search_domain(s) + except ValueError: + util.logexc(LOG, "Failed at adding search domain %s", s) + util.write_file(fn, str(r_conf), 0644) diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py new file mode 100644 index 00000000..904e931a --- /dev/null +++ b/cloudinit/distros/sles.py @@ -0,0 +1,193 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Author: Juerg Haefliger <juerg.haefliger@hp.com> +# +# Leaning very heavily on the RHEL and Debian implementation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +from cloudinit import distros + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.distros import rhel_util +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + clock_conf_fn = '/etc/sysconfig/clock' + locale_conf_fn = '/etc/sysconfig/language' + network_conf_fn = '/etc/sysconfig/network' + hostname_conf_fn = '/etc/HOSTNAME' + network_script_tpl = '/etc/sysconfig/network/ifcfg-%s' + resolve_conf_fn = '/etc/resolv.conf' + tz_local_fn = '/etc/localtime' + tz_zone_dir = '/usr/share/zoneinfo' + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + self.osfamily = 'suse' + + def install_packages(self, pkglist): + self.package_command('install', args='-l', pkgs=pkglist) + + def _write_network(self, settings): + # Convert debian settings to ifcfg format + entries = rhel_util.translate_network(settings) + LOG.debug("Translated ubuntu style network settings %s into %s", + settings, entries) + # Make the intermediate format as the suse format... + nameservers = [] + searchservers = [] + dev_names = entries.keys() + for (dev, info) in entries.iteritems(): + net_fn = self.network_script_tpl % (dev) + mode = info.get('auto') + if mode and mode.lower() == 'true': + mode = 'auto' + else: + mode = 'manual' + net_cfg = { + 'BOOTPROTO': info.get('bootproto'), + 'BROADCAST': info.get('broadcast'), + 'GATEWAY': info.get('gateway'), + 'IPADDR': info.get('address'), + 'LLADDR': info.get('hwaddress'), + 'NETMASK': info.get('netmask'), + 'STARTMODE': mode, + 'USERCONTROL': 'no' + } + if dev != 'lo': + net_cfg['ETHERDEVICE'] = dev + net_cfg['ETHTOOL_OPTIONS'] = '' + else: + net_cfg['FIREWALL'] = 'no' + rhel_util.update_sysconfig_file(net_fn, net_cfg, True) + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + if 'dns-search' in info: + searchservers.extend(info['dns-search']) + if nameservers or searchservers: + rhel_util.update_resolve_conf_file(self.resolve_conf_fn, + nameservers, searchservers) + return dev_names + + def apply_locale(self, locale, out_fn=None): + if not out_fn: + out_fn = self.locale_conf_fn + locale_cfg = { + 'RC_LANG': locale, + } + rhel_util.update_sysconfig_file(out_fn, locale_cfg) + + def _write_hostname(self, hostname, out_fn): + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(hostname) + util.write_file(out_fn, str(conf), 0644) + + def _select_hostname(self, hostname, fqdn): + # Prefer the short hostname over the long + # fully qualified domain name + if not hostname: + return fqdn + return hostname + + def _read_system_hostname(self): + host_fn = self.hostname_conf_fn + return (host_fn, self._read_hostname(host_fn)) + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_hostname(self, filename, default=None): + hostname = None + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def _bring_up_interfaces(self, device_names): + if device_names and 'all' in device_names: + raise RuntimeError(('Distro %s can not translate ' + 'the device name "all"') % (self.name)) + return distros.Distro._bring_up_interfaces(self, device_names) + + def set_timezone(self, tz): + # TODO(harlowja): move this code into + # the parent distro... + tz_file = os.path.join(self.tz_zone_dir, str(tz)) + if not os.path.isfile(tz_file): + raise RuntimeError(("Invalid timezone %s," + " no file found at %s") % (tz, tz_file)) + # Adjust the sysconfig clock zone setting + clock_cfg = { + 'TIMEZONE': str(tz), + } + rhel_util.update_sysconfig_file(self.clock_conf_fn, clock_cfg) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['zypper'] + # No user interaction possible, enable non-interactive mode + cmd.append('--non-interactive') + + # Comand is the operation, such as install + cmd.append(command) + + # args are the arguments to the command, not global options + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, capture=False) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ['refresh'], freq=PER_INSTANCE) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 924463ce..2ddc75f4 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -62,6 +62,7 @@ INCLUSION_TYPES_MAP = { '#part-handler': 'text/part-handler', '#cloud-boothook': 'text/cloud-boothook', '#cloud-config-archive': 'text/cloud-config-archive', + '#cloud-config-jsonp': 'text/cloud-config-jsonp', } # Sorted longest first @@ -117,10 +118,9 @@ def run_part(mod, data, filename, payload, frequency, headers): else: raise ValueError("Unknown module version %s" % (mod_ver)) except: - util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" - " with frequency %s"), - mod, content_type, filename, - mod_ver, frequency) + util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " + "frequency %s", mod, content_type, filename, mod_ver, + frequency) def call_begin(mod, data, frequency): @@ -152,14 +152,13 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): try: mod = fixup_handler(importer.import_module(modname)) call_begin(mod, pdata['data'], frequency) - # Only register and increment - # after the above have worked (so we don't if it - # fails) - handlers.register(mod) + # Only register and increment after the above have worked, so we don't + # register if it fails starting. + handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 except: - util.logexc(LOG, ("Failed at registering python file: %s" - " (part handler %s)"), modfname, curcount) + util.logexc(LOG, "Failed at registering python file: %s (part " + "handler %s)", modfname, curcount) def _extract_first_or_bytes(blob, size): diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index bf2899ab..1848ce2c 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +BOOTHOOK_PREFIX = "#cloud-boothook" class BootHookPartHandler(handlers.Handler): @@ -41,19 +42,15 @@ class BootHookPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#cloud-boothook"), + handlers.type_from_starts_with(BOOTHOOK_PREFIX), ] def _write_part(self, payload, filename): filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 filepath = os.path.join(self.boothook_dir, filename) - contents = payload[start:] - util.write_file(filepath, contents, 0700) + contents = util.strip_prefix_suffix(util.dos2unix(payload), + prefix=BOOTHOOK_PREFIX) + util.write_file(filepath, contents.lstrip(), 0700) return filepath def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 @@ -70,5 +67,5 @@ class BootHookPartHandler(handlers.Handler): except util.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: - util.logexc(LOG, ("Boothooks unknown " - "error when running %s"), filepath) + util.logexc(LOG, "Boothooks unknown error when running %s", + filepath) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index c97ca3e8..34a73115 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import jsonpatch + from cloudinit import handlers from cloudinit import log as logging from cloudinit import mergers @@ -49,6 +51,14 @@ MERGE_HEADER = 'Merge-Type' # # This gets loaded into yaml with final result {'a': 22} DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') +CLOUD_PREFIX = "#cloud-config" +JSONP_PREFIX = "#cloud-config-jsonp" + +# The file header -> content types this module will handle. +CC_TYPES = { + JSONP_PREFIX: handlers.type_from_starts_with(JSONP_PREFIX), + CLOUD_PREFIX: handlers.type_from_starts_with(CLOUD_PREFIX), +} class CloudConfigPartHandler(handlers.Handler): @@ -59,9 +69,7 @@ class CloudConfigPartHandler(handlers.Handler): self.file_names = [] def list_types(self): - return [ - handlers.type_from_starts_with("#cloud-config"), - ] + return list(CC_TYPES.values()) def _write_cloud_config(self): if not self.cloud_fn: @@ -78,7 +86,7 @@ class CloudConfigPartHandler(handlers.Handler): if self.cloud_buf is not None: # Something was actually gathered.... lines = [ - "#cloud-config", + CLOUD_PREFIX, '', ] lines.extend(file_lines) @@ -107,13 +115,21 @@ class CloudConfigPartHandler(handlers.Handler): all_mergers = DEF_MERGERS return (payload_yaml, all_mergers) + def _merge_patch(self, payload): + # JSON doesn't handle comments in this manner, so ensure that + # if we started with this 'type' that we remove it before + # attempting to load it as json (which the jsonpatch library will + # attempt to do). + payload = payload.lstrip() + payload = util.strip_prefix_suffix(payload, prefix=JSONP_PREFIX) + patch = jsonpatch.JsonPatch.from_string(payload) + LOG.debug("Merging by applying json patch %s", patch) + self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) + def _merge_part(self, payload, headers): (payload_yaml, my_mergers) = self._extract_mergers(payload, headers) LOG.debug("Merging by applying %s", my_mergers) merger = mergers.construct(my_mergers) - if self.cloud_buf is None: - # First time through, merge with an empty dict... - self.cloud_buf = {} self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml) def _reset(self): @@ -130,7 +146,13 @@ class CloudConfigPartHandler(handlers.Handler): self._reset() return try: - self._merge_part(payload, headers) + # First time through, merge with an empty dict... + if self.cloud_buf is None or not self.file_names: + self.cloud_buf = {} + if ctype == CC_TYPES[JSONP_PREFIX]: + self._merge_patch(payload) + else: + self._merge_part(payload, headers) # Ensure filename is ok to store for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index b185c374..62289d98 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +SHELL_PREFIX = "#!" class ShellScriptPartHandler(handlers.Handler): @@ -38,7 +39,7 @@ class ShellScriptPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#!"), + handlers.type_from_starts_with(SHELL_PREFIX), ] def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index edd56527..bac4cad2 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -22,6 +22,7 @@ import os +import re from cloudinit import handlers from cloudinit import log as logging @@ -30,6 +31,7 @@ from cloudinit import util from cloudinit.settings import (PER_INSTANCE) LOG = logging.getLogger(__name__) +UPSTART_PREFIX = "#upstart-job" class UpstartJobPartHandler(handlers.Handler): @@ -39,7 +41,7 @@ class UpstartJobPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#upstart-job"), + handlers.type_from_starts_with(UPSTART_PREFIX), ] def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 @@ -66,14 +68,53 @@ class UpstartJobPartHandler(handlers.Handler): path = os.path.join(self.upstart_dir, filename) util.write_file(path, payload, 0644) - # FIXME LATER (LP: #1124384) - # a bug in upstart means that invoking reload-configuration - # at this stage in boot causes havoc. So, until that is fixed - # we will not do that. However, I'd like to be able to easily - # test to see if this bug is still present in an image with - # a newer upstart. So, a boot hook could easiliy write this file. - if os.path.exists("/run/cloud-init-upstart-reload"): - # if inotify support is not present in the root filesystem - # (overlayroot) then we need to tell upstart to re-read /etc - + if SUITABLE_UPSTART: util.subp(["initctl", "reload-configuration"], capture=False) + + +def _has_suitable_upstart(): + # (LP: #1124384) + # a bug in upstart means that invoking reload-configuration + # at this stage in boot causes havoc. So, try to determine if upstart + # is installed, and reloading configuration is OK. + if not os.path.exists("/sbin/initctl"): + return False + try: + (version_out, _err) = util.subp(["initctl", "version"]) + except: + util.logexc(LOG, "initctl version failed") + return False + + # expecting 'initctl version' to output something like: init (upstart X.Y) + if re.match("upstart 1.[0-7][)]", version_out): + return False + if "upstart 0." in version_out: + return False + elif "upstart 1.8" in version_out: + if not os.path.exists("/usr/bin/dpkg-query"): + return False + try: + (dpkg_ver, _err) = util.subp(["dpkg-query", + "--showformat=${Version}", + "--show", "upstart"], rcs=[0, 1]) + except Exception: + util.logexc(LOG, "dpkg-query failed") + return False + + try: + good = "1.8-0ubuntu1.2" + util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + return True + except util.ProcessExecutionError as e: + if e.exit_code is 1: + pass + else: + util.logexc(LOG, "dpkg --compare-versions failed [%s]", + e.exit_code) + except Exception as e: + util.logexc(LOG, "dpkg --compare-versions failed") + return False + else: + return True + +SUITABLE_UPSTART = _has_suitable_upstart() diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index a4e6fb03..1c46efde 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -216,8 +216,8 @@ class ConfigMerger(object): if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) except: - util.logexc(LOG, ("Failed loading of datasource" - " config object from %s"), self._ds) + util.logexc(LOG, "Failed loading of datasource config object " + "from %s", self._ds) return d_cfgs def _get_env_configs(self): @@ -227,8 +227,8 @@ class ConfigMerger(object): try: e_cfgs.append(util.read_conf(e_fn)) except: - util.logexc(LOG, ('Failed loading of env. config' - ' from %s'), e_fn) + util.logexc(LOG, 'Failed loading of env. config from %s', + e_fn) return e_cfgs def _get_instance_configs(self): @@ -242,8 +242,8 @@ class ConfigMerger(object): try: i_cfgs.append(util.read_conf(cc_fn)) except: - util.logexc(LOG, ('Failed loading of cloud-config' - ' from %s'), cc_fn) + util.logexc(LOG, 'Failed loading of cloud-config from %s', + cc_fn) return i_cfgs def _read_cfg(self): @@ -259,8 +259,8 @@ class ConfigMerger(object): try: cfgs.append(util.read_conf(c_fn)) except: - util.logexc(LOG, ("Failed loading of configuration" - " from %s"), c_fn) + util.logexc(LOG, "Failed loading of configuration from %s", + c_fn) cfgs.extend(self._get_env_configs()) cfgs.extend(self._get_instance_configs()) @@ -281,6 +281,7 @@ class ContentHandlers(object): def __init__(self): self.registered = {} + self.initialized = [] def __contains__(self, item): return self.is_registered(item) @@ -291,11 +292,13 @@ class ContentHandlers(object): def is_registered(self, content_type): return content_type in self.registered - def register(self, mod): + def register(self, mod, initialized=False): types = set() for t in mod.list_types(): self.registered[t] = mod types.add(t) + if initialized and mod not in self.initialized: + self.initialized.append(mod) return types def _get_handler(self, content_type): diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 76591bea..62999b4e 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -19,6 +19,7 @@ DEF_MERGE_TYPE = 'replace' MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE, 'no_replace') + def _has_any(what, *keys): for k in keys: if k in what: diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 8cc9e3b4..9f6badae 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -31,11 +31,13 @@ CFG_BUILTIN = { 'datasource_list': [ 'NoCloud', 'ConfigDrive', + 'Azure', 'AltCloud', 'OVF', 'MAAS', 'Ec2', 'CloudStack', + 'SmartOS', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 64548d43..a834f8eb 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -1,10 +1,11 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Joe VLcek <JVLcek@RedHat.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -79,7 +80,7 @@ def read_user_data_callback(mount_dir): try: user_data = util.load_file(user_data_file).strip() except IOError: - util.logexc(LOG, ('Failed accessing user data file.')) + util.logexc(LOG, 'Failed accessing user data file.') return None return user_data @@ -178,7 +179,7 @@ class DataSourceAltCloud(sources.DataSource): return False # No user data found - util.logexc(LOG, ('Failed accessing user data.')) + util.logexc(LOG, 'Failed accessing user data.') return False def user_data_rhevm(self): @@ -205,12 +206,12 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False except OSError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False floppy_dev = '/dev/fd0' @@ -222,12 +223,12 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False except OSError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False try: @@ -236,8 +237,8 @@ class DataSourceAltCloud(sources.DataSource): if err.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for user data"), floppy_dev) + util.logexc(LOG, "Failed to mount %s when looking for user data", + floppy_dev) self.userdata_raw = return_str self.metadata = META_DATA_NOT_SUPPORTED @@ -272,8 +273,8 @@ class DataSourceAltCloud(sources.DataSource): if err.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for user data"), cdrom_dev) + util.logexc(LOG, "Failed to mount %s when looking for user " + "data", cdrom_dev) self.userdata_raw = return_str self.metadata = META_DATA_NOT_SUPPORTED diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py new file mode 100644 index 00000000..d4863429 --- /dev/null +++ b/cloudinit/sources/DataSourceAzure.py @@ -0,0 +1,485 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import base64 +import os +import os.path +import time +from xml.dom import minidom + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DS_NAME = 'Azure' +DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} +AGENT_START = ['service', 'walinuxagent', 'start'] +BOUNCE_COMMAND = ("i=$interface; x=0; ifdown $i || x=$?; " + "ifup $i || x=$?; exit $x") +BUILTIN_DS_CONFIG = { + 'agent_command': AGENT_START, + 'data_dir': "/var/lib/waagent", + 'set_hostname': True, + 'hostname_bounce': { + 'interface': 'eth0', + 'policy': True, + 'command': BOUNCE_COMMAND, + 'hostname_command': 'hostname', + } +} +DS_CFG_PATH = ['datasource', DS_NAME] + + +class DataSourceAzureNet(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'azure') + self.cfg = {} + self.seed = None + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), + BUILTIN_DS_CONFIG]) + + def __str__(self): + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) + + def get_data(self): + # azure removes/ejects the cdrom containing the ovf-env.xml + # file on reboot. So, in order to successfully reboot we + # need to look in the datadir and consider that valid + ddir = self.ds_cfg['data_dir'] + + candidates = [self.seed_dir] + candidates.extend(list_possible_azure_ds_devs()) + if ddir: + candidates.append(ddir) + + found = None + + for cdev in candidates: + try: + if cdev.startswith("/dev/"): + ret = util.mount_cb(cdev, load_azure_ds_dir) + else: + ret = load_azure_ds_dir(cdev) + + except NonAzureDataSource: + continue + except BrokenAzureDataSource as exc: + raise exc + except util.MountFailedError: + LOG.warn("%s was not mountable" % cdev) + continue + + (md, self.userdata_raw, cfg, files) = ret + self.seed = cdev + self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) + self.cfg = cfg + found = cdev + + LOG.debug("found datasource in %s", cdev) + break + + if not found: + return False + + if found == ddir: + LOG.debug("using files cached in %s", ddir) + + # now update ds_cfg to reflect contents pass in config + usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) + self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) + mycfg = self.ds_cfg + + # walinux agent writes files world readable, but expects + # the directory to be protected. + write_files(mycfg['data_dir'], files, dirmode=0700) + + # handle the hostname 'publishing' + try: + handle_set_hostname(mycfg.get('set_hostname'), + self.metadata.get('local-hostname'), + mycfg['hostname_bounce']) + except Exception as e: + LOG.warn("Failed publishing hostname: %s" % e) + util.logexc(LOG, "handling set_hostname failed") + + try: + invoke_agent(mycfg['agent_command']) + except util.ProcessExecutionError: + # claim the datasource even if the command failed + util.logexc(LOG, "agent command '%s' failed.", + mycfg['agent_command']) + + shcfgxml = os.path.join(mycfg['data_dir'], "SharedConfig.xml") + wait_for = [shcfgxml] + + fp_files = [] + for pk in self.cfg.get('_pubkeys', []): + bname = pk['fingerprint'] + ".crt" + fp_files += [os.path.join(mycfg['data_dir'], bname)] + + start = time.time() + missing = wait_for_files(wait_for + fp_files) + if len(missing): + LOG.warn("Did not find files, but going on: %s", missing) + else: + LOG.debug("waited %.3f seconds for %d files to appear", + time.time() - start, len(wait_for)) + + if shcfgxml in missing: + LOG.warn("SharedConfig.xml missing, using static instance-id") + else: + try: + self.metadata['instance-id'] = iid_from_shared_config(shcfgxml) + except ValueError as e: + LOG.warn("failed to get instance id in %s: %s" % (shcfgxml, e)) + + pubkeys = pubkeys_from_crt_files(fp_files) + + self.metadata['public-keys'] = pubkeys + + return True + + def get_config_obj(self): + return self.cfg + + +def handle_set_hostname(enabled, hostname, cfg): + if not util.is_true(enabled): + return + + if not hostname: + LOG.warn("set_hostname was true but no local-hostname") + return + + apply_hostname_bounce(hostname=hostname, policy=cfg['policy'], + interface=cfg['interface'], + command=cfg['command'], + hostname_command=cfg['hostname_command']) + + +def apply_hostname_bounce(hostname, policy, interface, command, + hostname_command="hostname"): + # set the hostname to 'hostname' if it is not already set to that. + # then, if policy is not off, bounce the interface using command + prev_hostname = util.subp(hostname_command, capture=True)[0].strip() + + util.subp([hostname_command, hostname]) + + if util.is_false(policy): + return + + if prev_hostname == hostname and policy != "force": + return + + env = os.environ.copy() + env['interface'] = interface + + if command == "builtin": + command = BOUNCE_COMMAND + + util.subp(command, shell=(not isinstance(command, list)), capture=True) + + +def crtfile_to_pubkey(fname): + pipeline = ('openssl x509 -noout -pubkey < "$0" |' + 'ssh-keygen -i -m PKCS8 -f /dev/stdin') + (out, _err) = util.subp(['sh', '-c', pipeline, fname], capture=True) + return out.rstrip() + + +def pubkeys_from_crt_files(flist): + pubkeys = [] + errors = [] + for fname in flist: + try: + pubkeys.append(crtfile_to_pubkey(fname)) + except util.ProcessExecutionError: + errors.extend(fname) + + if errors: + LOG.warn("failed to convert the crt files to pubkey: %s" % errors) + + return pubkeys + + +def wait_for_files(flist, maxwait=60, naplen=.5): + need = set(flist) + waited = 0 + while waited < maxwait: + need -= set([f for f in need if os.path.exists(f)]) + if len(need) == 0: + return [] + time.sleep(naplen) + waited += naplen + return need + + +def write_files(datadir, files, dirmode=None): + if not datadir: + return + if not files: + files = {} + util.ensure_dir(datadir, dirmode) + for (name, content) in files.items(): + util.write_file(filename=os.path.join(datadir, name), + content=content, mode=0600) + + +def invoke_agent(cmd): + # this is a function itself to simplify patching it for test + if cmd: + LOG.debug("invoking agent: %s" % cmd) + util.subp(cmd, shell=(not isinstance(cmd, list))) + else: + LOG.debug("not invoking agent") + + +def find_child(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return ret + + +def load_azure_ovf_pubkeys(sshnode): + # This parses a 'SSH' node formatted like below, and returns + # an array of dicts. + # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', + # 'path': 'where/to/go'}] + # + # <SSH><PublicKeys> + # <PublicKey><Fingerprint>ABC</FingerPrint><Path>/ABC</Path> + # ... + # </PublicKeys></SSH> + results = find_child(sshnode, lambda n: n.localName == "PublicKeys") + if len(results) == 0: + return [] + if len(results) > 1: + raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" % + len(results)) + + pubkeys_node = results[0] + pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey") + + if len(pubkeys) == 0: + return [] + + found = [] + text_node = minidom.Document.TEXT_NODE + + for pk_node in pubkeys: + if not pk_node.hasChildNodes(): + continue + cur = {'fingerprint': "", 'path': ""} + for child in pk_node.childNodes: + if (child.nodeType == text_node or not child.localName): + continue + + name = child.localName.lower() + + if name not in cur.keys(): + continue + + if (len(child.childNodes) != 1 or + child.childNodes[0].nodeType != text_node): + continue + + cur[name] = child.childNodes[0].wholeText.strip() + found.append(cur) + + return found + + +def single_node_at_path(node, pathlist): + curnode = node + for tok in pathlist: + results = find_child(curnode, lambda n: n.localName == tok) + if len(results) == 0: + raise ValueError("missing %s token in %s" % (tok, str(pathlist))) + if len(results) > 1: + raise ValueError("found %s nodes of type %s looking for %s" % + (len(results), tok, str(pathlist))) + curnode = results[0] + + return curnode + + +def read_azure_ovf(contents): + try: + dom = minidom.parseString(contents) + except Exception as e: + raise NonAzureDataSource("invalid xml: %s" % e) + + results = find_child(dom.documentElement, + lambda n: n.localName == "ProvisioningSection") + + if len(results) == 0: + raise NonAzureDataSource("No ProvisioningSection") + if len(results) > 1: + raise BrokenAzureDataSource("found '%d' ProvisioningSection items" % + len(results)) + provSection = results[0] + + lpcs_nodes = find_child(provSection, + lambda n: n.localName == "LinuxProvisioningConfigurationSet") + + if len(results) == 0: + raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") + if len(results) > 1: + raise BrokenAzureDataSource("found '%d' %ss" % + ("LinuxProvisioningConfigurationSet", + len(results))) + lpcs = lpcs_nodes[0] + + if not lpcs.hasChildNodes(): + raise BrokenAzureDataSource("no child nodes of configuration set") + + md_props = 'seedfrom' + md = {'azure_data': {}} + cfg = {} + ud = "" + password = None + username = None + + for child in lpcs.childNodes: + if child.nodeType == dom.TEXT_NODE or not child.localName: + continue + + name = child.localName.lower() + + simple = False + value = "" + if (len(child.childNodes) == 1 and + child.childNodes[0].nodeType == dom.TEXT_NODE): + simple = True + value = child.childNodes[0].wholeText + + attrs = {k: v for k, v in child.attributes.items()} + + # we accept either UserData or CustomData. If both are present + # then behavior is undefined. + if (name == "userdata" or name == "customdata"): + if attrs.get('encoding') in (None, "base64"): + ud = base64.b64decode(''.join(value.split())) + else: + ud = value + elif name == "username": + username = value + elif name == "userpassword": + password = value + elif name == "hostname": + md['local-hostname'] = value + elif name == "dscfg": + if attrs.get('encoding') in (None, "base64"): + dscfg = base64.b64decode(''.join(value.split())) + else: + dscfg = value + cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})} + elif name == "ssh": + cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) + elif name == "disablesshpasswordauthentication": + cfg['ssh_pwauth'] = util.is_false(value) + elif simple: + if name in md_props: + md[name] = value + else: + md['azure_data'][name] = value + + defuser = {} + if username: + defuser['name'] = username + if password: + defuser['password'] = password + defuser['lock_passwd'] = False + + if defuser: + cfg['system_info'] = {'default_user': defuser} + + if 'ssh_pwauth' not in cfg and password: + cfg['ssh_pwauth'] = True + + return (md, ud, cfg) + + +def list_possible_azure_ds_devs(): + # return a sorted list of devices that might have a azure datasource + devlist = [] + for fstype in ("iso9660", "udf"): + devlist.extend(util.find_devs_with("TYPE=%s" % fstype)) + + devlist.sort(reverse=True) + return devlist + + +def load_azure_ds_dir(source_dir): + ovf_file = os.path.join(source_dir, "ovf-env.xml") + + if not os.path.isfile(ovf_file): + raise NonAzureDataSource("No ovf-env file found") + + with open(ovf_file, "r") as fp: + contents = fp.read() + + md, ud, cfg = read_azure_ovf(contents) + return (md, ud, cfg, {'ovf-env.xml': contents}) + + +def iid_from_shared_config(path): + with open(path, "rb") as fp: + content = fp.read() + return iid_from_shared_config_content(content) + + +def iid_from_shared_config_content(content): + """ + find INSTANCE_ID in: + <?xml version="1.0" encoding="utf-8"?> + <SharedConfig version="1.0.0.0" goalStateIncarnation="1"> + <Deployment name="INSTANCE_ID" guid="{...}" incarnation="0"> + <Service name="..." guid="{00000000-0000-0000-0000-000000000000}" /> + """ + dom = minidom.parseString(content) + depnode = single_node_at_path(dom, ["SharedConfig", "Deployment"]) + return depnode.attributes.get('name').value + + +class BrokenAzureDataSource(Exception): + pass + + +class NonAzureDataSource(Exception): + pass + + +# Used to match classes to dependencies +datasources = [ + (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 81c8cda9..08f661e4 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -4,11 +4,13 @@ # Copyright (C) 2012 Cosmin Luta # Copyright (C) 2012 Yahoo! Inc. # Copyright (C) 2012 Gerard Dethier +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. # # Author: Cosmin Luta <q4break@gmail.com> # Author: Scott Moser <scott.moser@canonical.com> # Author: Joshua Harlow <harlowja@yahoo-inc.com> # Author: Gerard Dethier <g.dethier@gmail.com> +# Author: Juerg Haefliger <juerg.haefliger@hp.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -109,8 +111,8 @@ class DataSourceCloudStack(sources.DataSource): int(time.time() - start_time)) return True except Exception: - util.logexc(LOG, ('Failed fetching from metadata ' - 'service %s'), self.metadata_address) + util.logexc(LOG, 'Failed fetching from metadata service %s', + self.metadata_address) return False def get_instance_id(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 084abca7..4ef92a56 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -119,8 +119,8 @@ class DataSourceNoCloud(sources.DataSource): if e.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for data"), dev) + util.logexc(LOG, "Failed to mount %s when looking for " + "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py new file mode 100644 index 00000000..1ce20c10 --- /dev/null +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -0,0 +1,195 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Canonical Ltd. +# +# Author: Ben Howard <ben.howard@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +# +# Datasource for provisioning on SmartOS. This works on Joyent +# and public/private Clouds using SmartOS. +# +# SmartOS hosts use a serial console (/dev/ttyS1) on Linux Guests. +# The meta-data is transmitted via key/value pairs made by +# requests on the console. For example, to get the hostname, you +# would send "GET hostname" on /dev/ttyS1. +# + + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util +import os +import os.path +import serial + + +DEF_TTY_LOC = '/dev/ttyS1' +DEF_TTY_TIMEOUT = 60 +LOG = logging.getLogger(__name__) + +SMARTOS_ATTRIB_MAP = { + #Cloud-init Key : (SmartOS Key, Strip line endings) + 'local-hostname': ('hostname', True), + 'public-keys': ('root_authorized_keys', True), + 'user-script': ('user-script', False), + 'user-data': ('user-data', False), + 'iptables_disable': ('iptables_disable', True), + 'motd_sys_info': ('motd_sys_info', True), +} + + +class DataSourceSmartOS(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'sdc') + self.is_smartdc = None + self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) + self.seed_timeout = self.sys_cfg.get("serial_timeout", + DEF_TTY_TIMEOUT) + + def __str__(self): + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) + + def get_data(self): + md = {} + ud = "" + + if not os.path.exists(self.seed): + LOG.debug("Host does not appear to be on SmartOS") + return False + self.seed = self.seed + + dmi_info = dmi_data() + if dmi_info is False: + LOG.debug("No dmidata utility found") + return False + + system_uuid, system_type = dmi_info + if 'smartdc' not in system_type.lower(): + LOG.debug("Host is not on SmartOS") + return False + self.is_smartdc = True + md['instance-id'] = system_uuid + + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): + smartos_noun, strip = attribute + md[ci_noun] = query_data(smartos_noun, self.seed, + self.seed_timeout, strip=strip) + + if not md['local-hostname']: + md['local-hostname'] = system_uuid + + if md['user-data']: + ud = md['user-data'] + else: + ud = md['user-script'] + + self.metadata = md + self.userdata_raw = ud + return True + + def get_instance_id(self): + return self.metadata['instance-id'] + + +def get_serial(seed_device, seed_timeout): + """This is replaced in unit testing, allowing us to replace + serial.Serial with a mocked class + + The timeout value of 60 seconds should never be hit. The value + is taken from SmartOS own provisioning tools. Since we are reading + each line individually up until the single ".", the transfer is + usually very fast (i.e. microseconds) to get the response. + """ + if not seed_device: + raise AttributeError("seed_device value is not set") + + ser = serial.Serial(seed_device, timeout=seed_timeout) + if not ser.isOpen(): + raise SystemError("Unable to open %s" % seed_device) + + return ser + + +def query_data(noun, seed_device, seed_timeout, strip=False): + """Makes a request to via the serial console via "GET <NOUN>" + + In the response, the first line is the status, while subsequent lines + are is the value. A blank line with a "." is used to indicate end of + response. + """ + + if not noun: + return False + + ser = get_serial(seed_device, seed_timeout) + ser.write("GET %s\n" % noun.rstrip()) + status = str(ser.readline()).rstrip() + response = [] + eom_found = False + + if 'SUCCESS' not in status: + ser.close() + return None + + while not eom_found: + m = ser.readline() + if m.rstrip() == ".": + eom_found = True + else: + response.append(m) + + ser.close() + if not strip: + return "".join(response) + else: + return "".join(response).rstrip() + + return None + + +def dmi_data(): + sys_uuid, sys_type = None, None + dmidecode_path = util.which('dmidecode') + if not dmidecode_path: + return False + + sys_uuid_cmd = [dmidecode_path, "-s", "system-uuid"] + try: + LOG.debug("Getting hostname from dmidecode") + (sys_uuid, _err) = util.subp(sys_uuid_cmd) + except Exception as e: + util.logexc(LOG, "Failed to get system UUID", e) + + sys_type_cmd = [dmidecode_path, "-s", "system-product-name"] + try: + LOG.debug("Determining hypervisor product name via dmidecode") + (sys_type, _err) = util.subp(sys_type_cmd) + except Exception as e: + util.logexc(LOG, "Failed to get system UUID", e) + + return sys_uuid.lower(), sys_type + + +# Used to match classes to dependencies +datasources = [ + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index d8fbacdd..974c0407 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -135,7 +135,8 @@ class DataSource(object): @property def availability_zone(self): - return self.metadata.get('availability-zone') + return self.metadata.get('availability-zone', + self.metadata.get('availability_zone')) def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 95133236..70a577bc 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -229,11 +229,9 @@ def extract_authorized_keys(username): except (IOError, OSError): # Give up and use a default key filename auth_key_fn = os.path.join(ssh_dir, 'authorized_keys') - util.logexc(LOG, ("Failed extracting 'AuthorizedKeysFile'" - " in ssh config" - " from %r, using 'AuthorizedKeysFile' file" - " %r instead"), - DEF_SSHD_CFG, auth_key_fn) + util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in ssh " + "config from %r, using 'AuthorizedKeysFile' file " + "%r instead", DEF_SSHD_CFG, auth_key_fn) return (auth_key_fn, parse_authorized_keys(auth_key_fn)) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 543d247f..3e49e8c5 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -154,9 +154,8 @@ class Init(object): try: util.chownbyname(log_file, u, g) except OSError: - util.logexc(LOG, ("Unable to change the ownership" - " of %s to user %s, group %s"), - log_file, u, g) + util.logexc(LOG, "Unable to change the ownership of %s to " + "user %s, group %s", log_file, u, g) def read_cfg(self, extra_fns=None): # None check so that we don't keep on re-loading if empty @@ -345,12 +344,13 @@ class Init(object): cdir = self.paths.get_cpath("handlers") idir = self._get_ipath("handlers") - # Add the path to the plugins dir to the top of our list for import - # instance dir should be read before cloud-dir - if cdir and cdir not in sys.path: - sys.path.insert(0, cdir) - if idir and idir not in sys.path: - sys.path.insert(0, idir) + # Add the path to the plugins dir to the top of our list for importing + # new handlers. + # + # Note(harlowja): instance dir should be read before cloud-dir + for d in [cdir, idir]: + if d and d not in sys.path: + sys.path.insert(0, d) # Ensure datasource fetched before activation (just incase) user_data_msg = self.datasource.get_userdata(True) @@ -358,24 +358,34 @@ class Init(object): # This keeps track of all the active handlers c_handlers = helpers.ContentHandlers() - # Add handlers in cdir - potential_handlers = util.find_modules(cdir) - for (fname, mod_name) in potential_handlers.iteritems(): - try: - mod_locs = importer.find_module(mod_name, [''], - ['list_types', - 'handle_part']) - if not mod_locs: - LOG.warn(("Could not find a valid user-data handler" - " named %s in file %s"), mod_name, fname) - continue - mod = importer.import_module(mod_locs[0]) - mod = handlers.fixup_handler(mod) - types = c_handlers.register(mod) - LOG.debug("Added handler for %s from %s", types, fname) - except: - util.logexc(LOG, "Failed to register handler from %s", fname) - + def register_handlers_in_dir(path): + # Attempts to register any handler modules under the given path. + if not path or not os.path.isdir(path): + return + potential_handlers = util.find_modules(path) + for (fname, mod_name) in potential_handlers.iteritems(): + try: + mod_locs = importer.find_module(mod_name, [''], + ['list_types', + 'handle_part']) + if not mod_locs: + LOG.warn(("Could not find a valid user-data handler" + " named %s in file %s"), mod_name, fname) + continue + mod = importer.import_module(mod_locs[0]) + mod = handlers.fixup_handler(mod) + types = c_handlers.register(mod) + LOG.debug("Added handler for %s from %s", types, fname) + except Exception: + util.logexc(LOG, "Failed to register handler from %s", + fname) + + # Add any handlers in the cloud-dir + register_handlers_in_dir(cdir) + + # Register any other handlers that come from the default set. This + # is done after the cloud-dir handlers so that the cdir modules can + # take over the default user-data handler content-types. def_handlers = self._default_userdata_handlers() applied_def_handlers = c_handlers.register_defaults(def_handlers) if applied_def_handlers: @@ -384,36 +394,51 @@ class Init(object): # Form our cloud interface data = self.cloudify() - # Init the handlers first - called = [] - for (_ctype, mod) in c_handlers.iteritems(): - if mod in called: - continue - handlers.call_begin(mod, data, frequency) - called.append(mod) - - # Walk the user data - part_data = { - 'handlers': c_handlers, - # Any new handlers that are encountered get writen here - 'handlerdir': idir, - 'data': data, - # The default frequency if handlers don't have one - 'frequency': frequency, - # This will be used when new handlers are found - # to help write there contents to files with numbered - # names... - 'handlercount': 0, - } - handlers.walk(user_data_msg, handlers.walker_callback, data=part_data) + def init_handlers(): + # Init the handlers first + for (_ctype, mod) in c_handlers.iteritems(): + if mod in c_handlers.initialized: + # Avoid initing the same module twice (if said module + # is registered to more than one content-type). + continue + handlers.call_begin(mod, data, frequency) + c_handlers.initialized.append(mod) + + def walk_handlers(): + # Walk the user data + part_data = { + 'handlers': c_handlers, + # Any new handlers that are encountered get writen here + 'handlerdir': idir, + 'data': data, + # The default frequency if handlers don't have one + 'frequency': frequency, + # This will be used when new handlers are found + # to help write there contents to files with numbered + # names... + 'handlercount': 0, + } + handlers.walk(user_data_msg, handlers.walker_callback, + data=part_data) + + def finalize_handlers(): + # Give callbacks opportunity to finalize + for (_ctype, mod) in c_handlers.iteritems(): + if mod not in c_handlers.initialized: + # Said module was never inited in the first place, so lets + # not attempt to finalize those that never got called. + continue + c_handlers.initialized.remove(mod) + try: + handlers.call_end(mod, data, frequency) + except: + util.logexc(LOG, "Failed to finalize handler: %s", mod) - # Give callbacks opportunity to finalize - called = [] - for (_ctype, mod) in c_handlers.iteritems(): - if mod in called: - continue - handlers.call_end(mod, data, frequency) - called.append(mod) + try: + init_handlers() + walk_handlers() + finally: + finalize_handlers() # Perform post-consumption adjustments so that # modules that run during the init stage reflect diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index df069ff8..d49ea094 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -23,8 +23,10 @@ import os import email + from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from cloudinit import handlers @@ -48,6 +50,18 @@ ARCHIVE_TYPES = ["text/cloud-config-archive"] UNDEF_TYPE = "text/plain" ARCHIVE_UNDEF_TYPE = "text/cloud-config" +# This seems to hit most of the gzip possible content types. +DECOMP_TYPES = [ + 'application/gzip', + 'application/gzip-compressed', + 'application/gzipped', + 'application/x-compress', + 'application/x-compressed', + 'application/x-gunzip', + 'application/x-gzip', + 'application/x-gzip-compressed', +] + # Msg header used to track attachments ATTACHMENT_FIELD = 'Number-Attachments' @@ -56,6 +70,17 @@ ATTACHMENT_FIELD = 'Number-Attachments' EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"] +def _replace_header(msg, key, value): + del msg[key] + msg[key] = value + + +def _set_filename(msg, filename): + del msg['Content-Disposition'] + msg.add_header('Content-Disposition', + 'attachment', filename=str(filename)) + + class UserDataProcessor(object): def __init__(self, paths): self.paths = paths @@ -67,6 +92,10 @@ class UserDataProcessor(object): return accumulating_msg def _process_msg(self, base_msg, append_msg): + + def find_ctype(payload): + return handlers.type_from_starts_with(payload) + for part in base_msg.walk(): if is_skippable(part): continue @@ -74,21 +103,51 @@ class UserDataProcessor(object): ctype = None ctype_orig = part.get_content_type() payload = part.get_payload(decode=True) + was_compressed = False + + # When the message states it is of a gzipped content type ensure + # that we attempt to decode said payload so that the decompressed + # data can be examined (instead of the compressed data). + if ctype_orig in DECOMP_TYPES: + try: + payload = util.decomp_gzip(payload, quiet=False) + # At this point we don't know what the content-type is + # since we just decompressed it. + ctype_orig = None + was_compressed = True + except util.DecompressionError as e: + LOG.warn("Failed decompressing payload from %s of length" + " %s due to: %s", ctype_orig, len(payload), e) + continue + # Attempt to figure out the payloads content-type if not ctype_orig: ctype_orig = UNDEF_TYPE - if ctype_orig in TYPE_NEEDED: - ctype = handlers.type_from_starts_with(payload) - + ctype = find_ctype(payload) if ctype is None: ctype = ctype_orig + # In the case where the data was compressed, we want to make sure + # that we create a new message that contains the found content + # type with the uncompressed content since later traversals of the + # messages will expect a part not compressed. + if was_compressed: + maintype, subtype = ctype.split("/", 1) + n_part = MIMENonMultipart(maintype, subtype) + n_part.set_payload(payload) + # Copy various headers from the old part to the new one, + # but don't include all the headers since some are not useful + # after decoding and decompression. + if part.get_filename(): + _set_filename(n_part, part.get_filename()) + for h in ('Launch-Index',): + if h in part: + _replace_header(n_part, h, str(part[h])) + part = n_part + if ctype != ctype_orig: - if CONTENT_TYPE in part: - part.replace_header(CONTENT_TYPE, ctype) - else: - part[CONTENT_TYPE] = ctype + _replace_header(part, CONTENT_TYPE, ctype) if ctype in INCLUDE_TYPES: self._do_include(payload, append_msg) @@ -98,12 +157,9 @@ class UserDataProcessor(object): self._explode_archive(payload, append_msg) continue - # Should this be happening, shouldn't + # TODO(harlowja): Should this be happening, shouldn't # the part header be modified and not the base? - if CONTENT_TYPE in base_msg: - base_msg.replace_header(CONTENT_TYPE, ctype) - else: - base_msg[CONTENT_TYPE] = ctype + _replace_header(base_msg, CONTENT_TYPE, ctype) self._attach_part(append_msg, part) @@ -138,8 +194,7 @@ class UserDataProcessor(object): def _process_before_attach(self, msg, attached_id): if not msg.get_filename(): - msg.add_header('Content-Disposition', - 'attachment', filename=PART_FN_TPL % (attached_id)) + _set_filename(msg, PART_FN_TPL % (attached_id)) self._attach_launch_index(msg) def _do_include(self, content, append_msg): @@ -217,13 +272,15 @@ class UserDataProcessor(object): msg.set_payload(content) if 'filename' in ent: - msg.add_header('Content-Disposition', - 'attachment', filename=ent['filename']) + _set_filename(msg, ent['filename']) if 'launch-index' in ent: msg.add_header('Launch-Index', str(ent['launch-index'])) for header in list(ent.keys()): - if header in ('content', 'filename', 'type', 'launch-index'): + if header.lower() in ('content', 'filename', 'type', + 'launch-index', 'content-disposition', + ATTACHMENT_FIELD.lower(), + CONTENT_TYPE.lower()): continue msg.add_header(header, ent[header]) @@ -238,13 +295,13 @@ class UserDataProcessor(object): outer_msg[ATTACHMENT_FIELD] = '0' if new_count is not None: - outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) + _replace_header(outer_msg, ATTACHMENT_FIELD, str(new_count)) fetched_count = 0 try: fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) except (ValueError, TypeError): - outer_msg.replace_header(ATTACHMENT_FIELD, str(fetched_count)) + _replace_header(outer_msg, ATTACHMENT_FIELD, str(fetched_count)) return fetched_count def _attach_part(self, outer_msg, part): @@ -276,10 +333,7 @@ def convert_string(raw_data, headers=None): if "mime-version:" in data[0:4096].lower(): msg = email.message_from_string(data) for (key, val) in headers.iteritems(): - if key in msg: - msg.replace_header(key, val) - else: - msg[key] = val + _replace_header(msg, key, val) else: mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE) maintype, subtype = mtype.split("/", 1) diff --git a/cloudinit/util.py b/cloudinit/util.py index b27b3567..8542fe27 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -219,8 +219,7 @@ def fork_cb(child_cb, *args): child_cb(*args) os._exit(0) # pylint: disable=W0212 except: - logexc(LOG, ("Failed forking and" - " calling callback %s"), + logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) # pylint: disable=W0212 else: @@ -1531,6 +1530,14 @@ def shellify(cmdlist, add_header=True): return content +def strip_prefix_suffix(line, prefix=None, suffix=None): + if prefix and line.startswith(prefix): + line = line[len(prefix):] + if suffix and line.endswith(suffix): + line = line[:-len(suffix)] + return line + + def is_container(): """ Checks to see if this code running in a container of some sort @@ -1744,3 +1751,22 @@ def get_mount_info(path, log=LOG): mountinfo_path = '/proc/%s/mountinfo' % os.getpid() lines = load_file(mountinfo_path).splitlines() return parse_mount_info(path, lines, log) + + +def which(program): + # Return path of program for execution if found in path + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + _fpath, _ = os.path.split(program) + if _fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None diff --git a/cloudinit/version.py b/cloudinit/version.py index 024d5118..4b29a587 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -20,7 +20,7 @@ from distutils import version as vr def version(): - return vr.StrictVersion("0.7.2") + return vr.StrictVersion("0.7.3") def version_string(): |