From 8900f9cba622eeaf3810003c5a6ff7522312277b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:42:38 -0700 Subject: 1. Adding some new helper files that split off file inclusion, templating, importing, constant usage. 1. Move all datasources to a new sources directory 1. Rename some files to be more consistent with python file/module naming. --- cloudinit/CloudConfig/__init__.py | 274 ------------------ cloudinit/CloudConfig/cc_apt_pipelining.py | 53 ---- cloudinit/CloudConfig/cc_apt_update_upgrade.py | 241 ---------------- cloudinit/CloudConfig/cc_bootcmd.py | 48 ---- cloudinit/CloudConfig/cc_byobu.py | 77 ----- cloudinit/CloudConfig/cc_ca_certs.py | 90 ------ cloudinit/CloudConfig/cc_chef.py | 119 -------- cloudinit/CloudConfig/cc_disable_ec2_metadata.py | 30 -- cloudinit/CloudConfig/cc_final_message.py | 58 ---- cloudinit/CloudConfig/cc_foo.py | 29 -- cloudinit/CloudConfig/cc_grub_dpkg.py | 64 ----- cloudinit/CloudConfig/cc_keys_to_console.py | 42 --- cloudinit/CloudConfig/cc_landscape.py | 75 ----- cloudinit/CloudConfig/cc_locale.py | 54 ---- cloudinit/CloudConfig/cc_mcollective.py | 99 ------- cloudinit/CloudConfig/cc_mounts.py | 179 ------------ cloudinit/CloudConfig/cc_phone_home.py | 106 ------- cloudinit/CloudConfig/cc_puppet.py | 108 ------- cloudinit/CloudConfig/cc_resizefs.py | 108 ------- cloudinit/CloudConfig/cc_rightscale_userdata.py | 78 ----- cloudinit/CloudConfig/cc_rsyslog.py | 101 ------- cloudinit/CloudConfig/cc_runcmd.py | 32 --- cloudinit/CloudConfig/cc_salt_minion.py | 56 ---- cloudinit/CloudConfig/cc_scripts_per_boot.py | 34 --- cloudinit/CloudConfig/cc_scripts_per_instance.py | 34 --- cloudinit/CloudConfig/cc_scripts_per_once.py | 34 --- cloudinit/CloudConfig/cc_scripts_user.py | 34 --- cloudinit/CloudConfig/cc_set_hostname.py | 42 --- cloudinit/CloudConfig/cc_set_passwords.py | 129 --------- cloudinit/CloudConfig/cc_ssh.py | 106 ------- cloudinit/CloudConfig/cc_ssh_import_id.py | 50 ---- cloudinit/CloudConfig/cc_timezone.py | 67 ----- cloudinit/CloudConfig/cc_update_etc_hosts.py | 87 ------ cloudinit/CloudConfig/cc_update_hostname.py | 101 ------- cloudinit/DataSource.py | 214 -------------- cloudinit/DataSourceCloudStack.py | 92 ------ cloudinit/DataSourceConfigDrive.py | 231 --------------- cloudinit/DataSourceEc2.py | 217 -------------- cloudinit/DataSourceMAAS.py | 345 ----------------------- cloudinit/DataSourceNoCloud.py | 232 --------------- cloudinit/DataSourceOVF.py | 332 ---------------------- cloudinit/SshUtil.py | 227 --------------- cloudinit/UserDataHandler.py | 262 ----------------- cloudinit/constants.py | 37 +++ cloudinit/handlers/DataSource.py | 214 ++++++++++++++ cloudinit/handlers/DataSourceCloudStack.py | 92 ++++++ cloudinit/handlers/DataSourceConfigDrive.py | 231 +++++++++++++++ cloudinit/handlers/DataSourceEc2.py | 217 ++++++++++++++ cloudinit/handlers/DataSourceMAAS.py | 345 +++++++++++++++++++++++ cloudinit/handlers/DataSourceNoCloud.py | 232 +++++++++++++++ cloudinit/handlers/DataSourceOVF.py | 332 ++++++++++++++++++++++ cloudinit/handlers/__init__.py | 274 ++++++++++++++++++ cloudinit/handlers/cc_apt_pipelining.py | 53 ++++ cloudinit/handlers/cc_apt_update_upgrade.py | 241 ++++++++++++++++ cloudinit/handlers/cc_bootcmd.py | 48 ++++ cloudinit/handlers/cc_byobu.py | 77 +++++ cloudinit/handlers/cc_ca_certs.py | 90 ++++++ cloudinit/handlers/cc_chef.py | 119 ++++++++ cloudinit/handlers/cc_disable_ec2_metadata.py | 30 ++ cloudinit/handlers/cc_final_message.py | 58 ++++ cloudinit/handlers/cc_foo.py | 29 ++ cloudinit/handlers/cc_grub_dpkg.py | 64 +++++ cloudinit/handlers/cc_keys_to_console.py | 42 +++ cloudinit/handlers/cc_landscape.py | 75 +++++ cloudinit/handlers/cc_locale.py | 54 ++++ cloudinit/handlers/cc_mcollective.py | 99 +++++++ cloudinit/handlers/cc_mounts.py | 179 ++++++++++++ cloudinit/handlers/cc_phone_home.py | 106 +++++++ cloudinit/handlers/cc_puppet.py | 108 +++++++ cloudinit/handlers/cc_resizefs.py | 108 +++++++ cloudinit/handlers/cc_rightscale_userdata.py | 78 +++++ cloudinit/handlers/cc_rsyslog.py | 101 +++++++ cloudinit/handlers/cc_runcmd.py | 32 +++ cloudinit/handlers/cc_salt_minion.py | 56 ++++ cloudinit/handlers/cc_scripts_per_boot.py | 34 +++ cloudinit/handlers/cc_scripts_per_instance.py | 34 +++ cloudinit/handlers/cc_scripts_per_once.py | 34 +++ cloudinit/handlers/cc_scripts_user.py | 34 +++ cloudinit/handlers/cc_set_hostname.py | 42 +++ cloudinit/handlers/cc_set_passwords.py | 129 +++++++++ cloudinit/handlers/cc_ssh.py | 106 +++++++ cloudinit/handlers/cc_ssh_import_id.py | 50 ++++ cloudinit/handlers/cc_timezone.py | 67 +++++ cloudinit/handlers/cc_update_etc_hosts.py | 87 ++++++ cloudinit/handlers/cc_update_hostname.py | 101 +++++++ cloudinit/importer.py | 11 + cloudinit/includer.py | 65 +++++ cloudinit/log.py | 94 ++++++ cloudinit/ssh_util.py | 227 +++++++++++++++ cloudinit/templater.py | 17 ++ cloudinit/user_data.py | 262 +++++++++++++++++ 91 files changed, 5215 insertions(+), 4991 deletions(-) delete mode 100644 cloudinit/CloudConfig/__init__.py delete mode 100644 cloudinit/CloudConfig/cc_apt_pipelining.py delete mode 100644 cloudinit/CloudConfig/cc_apt_update_upgrade.py delete mode 100644 cloudinit/CloudConfig/cc_bootcmd.py delete mode 100644 cloudinit/CloudConfig/cc_byobu.py delete mode 100644 cloudinit/CloudConfig/cc_ca_certs.py delete mode 100644 cloudinit/CloudConfig/cc_chef.py delete mode 100644 cloudinit/CloudConfig/cc_disable_ec2_metadata.py delete mode 100644 cloudinit/CloudConfig/cc_final_message.py delete mode 100644 cloudinit/CloudConfig/cc_foo.py delete mode 100644 cloudinit/CloudConfig/cc_grub_dpkg.py delete mode 100644 cloudinit/CloudConfig/cc_keys_to_console.py delete mode 100644 cloudinit/CloudConfig/cc_landscape.py delete mode 100644 cloudinit/CloudConfig/cc_locale.py delete mode 100644 cloudinit/CloudConfig/cc_mcollective.py delete mode 100644 cloudinit/CloudConfig/cc_mounts.py delete mode 100644 cloudinit/CloudConfig/cc_phone_home.py delete mode 100644 cloudinit/CloudConfig/cc_puppet.py delete mode 100644 cloudinit/CloudConfig/cc_resizefs.py delete mode 100644 cloudinit/CloudConfig/cc_rightscale_userdata.py delete mode 100644 cloudinit/CloudConfig/cc_rsyslog.py delete mode 100644 cloudinit/CloudConfig/cc_runcmd.py delete mode 100644 cloudinit/CloudConfig/cc_salt_minion.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_per_boot.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_per_instance.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_per_once.py delete mode 100644 cloudinit/CloudConfig/cc_scripts_user.py delete mode 100644 cloudinit/CloudConfig/cc_set_hostname.py delete mode 100644 cloudinit/CloudConfig/cc_set_passwords.py delete mode 100644 cloudinit/CloudConfig/cc_ssh.py delete mode 100644 cloudinit/CloudConfig/cc_ssh_import_id.py delete mode 100644 cloudinit/CloudConfig/cc_timezone.py delete mode 100644 cloudinit/CloudConfig/cc_update_etc_hosts.py delete mode 100644 cloudinit/CloudConfig/cc_update_hostname.py delete mode 100644 cloudinit/DataSource.py delete mode 100644 cloudinit/DataSourceCloudStack.py delete mode 100644 cloudinit/DataSourceConfigDrive.py delete mode 100644 cloudinit/DataSourceEc2.py delete mode 100644 cloudinit/DataSourceMAAS.py delete mode 100644 cloudinit/DataSourceNoCloud.py delete mode 100644 cloudinit/DataSourceOVF.py delete mode 100644 cloudinit/SshUtil.py delete mode 100644 cloudinit/UserDataHandler.py create mode 100644 cloudinit/constants.py create mode 100644 cloudinit/handlers/DataSource.py create mode 100644 cloudinit/handlers/DataSourceCloudStack.py create mode 100644 cloudinit/handlers/DataSourceConfigDrive.py create mode 100644 cloudinit/handlers/DataSourceEc2.py create mode 100644 cloudinit/handlers/DataSourceMAAS.py create mode 100644 cloudinit/handlers/DataSourceNoCloud.py create mode 100644 cloudinit/handlers/DataSourceOVF.py create mode 100644 cloudinit/handlers/__init__.py create mode 100644 cloudinit/handlers/cc_apt_pipelining.py create mode 100644 cloudinit/handlers/cc_apt_update_upgrade.py create mode 100644 cloudinit/handlers/cc_bootcmd.py create mode 100644 cloudinit/handlers/cc_byobu.py create mode 100644 cloudinit/handlers/cc_ca_certs.py create mode 100644 cloudinit/handlers/cc_chef.py create mode 100644 cloudinit/handlers/cc_disable_ec2_metadata.py create mode 100644 cloudinit/handlers/cc_final_message.py create mode 100644 cloudinit/handlers/cc_foo.py create mode 100644 cloudinit/handlers/cc_grub_dpkg.py create mode 100644 cloudinit/handlers/cc_keys_to_console.py create mode 100644 cloudinit/handlers/cc_landscape.py create mode 100644 cloudinit/handlers/cc_locale.py create mode 100644 cloudinit/handlers/cc_mcollective.py create mode 100644 cloudinit/handlers/cc_mounts.py create mode 100644 cloudinit/handlers/cc_phone_home.py create mode 100644 cloudinit/handlers/cc_puppet.py create mode 100644 cloudinit/handlers/cc_resizefs.py create mode 100644 cloudinit/handlers/cc_rightscale_userdata.py create mode 100644 cloudinit/handlers/cc_rsyslog.py create mode 100644 cloudinit/handlers/cc_runcmd.py create mode 100644 cloudinit/handlers/cc_salt_minion.py create mode 100644 cloudinit/handlers/cc_scripts_per_boot.py create mode 100644 cloudinit/handlers/cc_scripts_per_instance.py create mode 100644 cloudinit/handlers/cc_scripts_per_once.py create mode 100644 cloudinit/handlers/cc_scripts_user.py create mode 100644 cloudinit/handlers/cc_set_hostname.py create mode 100644 cloudinit/handlers/cc_set_passwords.py create mode 100644 cloudinit/handlers/cc_ssh.py create mode 100644 cloudinit/handlers/cc_ssh_import_id.py create mode 100644 cloudinit/handlers/cc_timezone.py create mode 100644 cloudinit/handlers/cc_update_etc_hosts.py create mode 100644 cloudinit/handlers/cc_update_hostname.py create mode 100644 cloudinit/importer.py create mode 100644 cloudinit/includer.py create mode 100644 cloudinit/log.py create mode 100644 cloudinit/ssh_util.py create mode 100644 cloudinit/templater.py create mode 100644 cloudinit/user_data.py diff --git a/cloudinit/CloudConfig/__init__.py b/cloudinit/CloudConfig/__init__.py deleted file mode 100644 index a16bdde6..00000000 --- a/cloudinit/CloudConfig/__init__.py +++ /dev/null @@ -1,274 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2008-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Chuck Short -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import yaml -import cloudinit -import cloudinit.util as util -import sys -import traceback -import os -import subprocess -import time - -per_instance = cloudinit.per_instance -per_always = cloudinit.per_always -per_once = cloudinit.per_once - - -class CloudConfig(): - cfgfile = None - cfg = None - - def __init__(self, cfgfile, cloud=None, ds_deps=None): - if cloud == None: - self.cloud = cloudinit.CloudInit(ds_deps) - self.cloud.get_data_source() - else: - self.cloud = cloud - self.cfg = self.get_config_obj(cfgfile) - - def get_config_obj(self, cfgfile): - try: - cfg = util.read_conf(cfgfile) - except: - # TODO: this 'log' could/should be passed in - cloudinit.log.critical("Failed loading of cloud config '%s'. " - "Continuing with empty config\n" % cfgfile) - cloudinit.log.debug(traceback.format_exc() + "\n") - cfg = None - if cfg is None: - cfg = {} - - try: - ds_cfg = self.cloud.datasource.get_config_obj() - except: - ds_cfg = {} - - cfg = util.mergedict(cfg, ds_cfg) - return(util.mergedict(cfg, self.cloud.cfg)) - - def handle(self, name, args, freq=None): - try: - mod = __import__("cc_" + name.replace("-", "_"), globals()) - def_freq = getattr(mod, "frequency", per_instance) - handler = getattr(mod, "handle") - - if not freq: - freq = def_freq - - self.cloud.sem_and_run("config-" + name, freq, handler, - [name, self.cfg, self.cloud, cloudinit.log, args]) - except: - raise - - -# reads a cloudconfig module list, returns -# a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg, name): - if name not in cfg: - return([]) - module_list = [] - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - for item in cfg[name]: - if isinstance(item, str): - module_list.append((item,)) - elif isinstance(item, list): - module_list.append(item) - else: - raise TypeError("failed to read '%s' item in config") - return(module_list) - - -def run_cc_modules(cc, module_list, log): - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args)) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - log.error("config handling of %s, %s, %s failed\n" % - (name, freq, run_args)) - failures.append(name) - - return(failures) - - -# always returns well formated values -# cfg is expected to have an entry 'output' in it, which is a dictionary -# that includes entries for 'init', 'config', 'final' or 'all' -# init: /var/log/cloud.out -# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] -# final: -# output: "| logger -p" -# error: "> /dev/null" -# this returns the specific 'mode' entry, cleanly formatted, with value -# None if if none is given -def get_output_cfg(cfg, mode="init"): - ret = [None, None] - if not 'output' in cfg: - return ret - - outcfg = cfg['output'] - if mode in outcfg: - modecfg = outcfg[mode] - else: - if 'all' not in outcfg: - return ret - # if there is a 'all' item in the output list - # then it applies to all users of this (init, config, final) - modecfg = outcfg['all'] - - # if value is a string, it specifies stdout and stderr - if isinstance(modecfg, str): - ret = [modecfg, modecfg] - - # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg, list): - if len(modecfg) > 0: - ret[0] = modecfg[0] - if len(modecfg) > 1: - ret[1] = modecfg[1] - - # if it is a dictionary, expect 'out' and 'error' - # items, which indicate out and error - if isinstance(modecfg, dict): - if 'output' in modecfg: - ret[0] = modecfg['output'] - if 'error' in modecfg: - ret[1] = modecfg['error'] - - # if err's entry == "&1", then make it same as stdout - # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": - ret[1] = ret[0] - - swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: - continue - val = ret[i].lstrip() - found = False - for s in swlist: - if val.startswith(s): - val = "%s %s" % (s, val[len(s):].strip()) - found = True - break - if not found: - # default behavior is append - val = "%s %s" % (">>", val.strip()) - ret[i] = val - - return(ret) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): - if outfmt: - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - if errfmt == outfmt: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - -def run_per_instance(name, func, args, clear_on_fail=False): - semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) - if os.path.exists(semfile): - return - - util.write_file(semfile, str(time.time())) - try: - func(*args) - except: - if clear_on_fail: - os.unlink(semfile) - raise - - -# apt_get top level command (install, update...), and args to pass it -def apt_get(tlc, args=None): - if args is None: - args = [] - e = os.environ.copy() - e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', tlc] - cmd.extend(args) - subprocess.check_call(cmd, env=e) - - -def update_package_sources(): - run_per_instance("update-sources", apt_get, ("update",)) - - -def install_packages(pkglist): - update_package_sources() - apt_get("install", pkglist) diff --git a/cloudinit/CloudConfig/cc_apt_pipelining.py b/cloudinit/CloudConfig/cc_apt_pipelining.py deleted file mode 100644 index 0286a9ae..00000000 --- a/cloudinit/CloudConfig/cc_apt_pipelining.py +++ /dev/null @@ -1,53 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance -default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" - - -def handle(_name, cfg, _cloud, log, _args): - - apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) - apt_pipe_value = str(apt_pipe_value).lower() - - if apt_pipe_value == "false": - write_apt_snippet("0", log) - - elif apt_pipe_value in ("none", "unchanged", "os"): - return - - elif apt_pipe_value in str(range(0, 6)): - write_apt_snippet(apt_pipe_value, log) - - else: - log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) - - -def write_apt_snippet(setting, log, f_name=default_file): - """ Writes f_name with apt pipeline depth 'setting' """ - - acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' - file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" - + (acquire_pipeline_depth % setting)) - - util.write_file(f_name, file_contents) - - log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/CloudConfig/cc_apt_update_upgrade.py b/cloudinit/CloudConfig/cc_apt_update_upgrade.py deleted file mode 100644 index a7049bce..00000000 --- a/cloudinit/CloudConfig/cc_apt_update_upgrade.py +++ /dev/null @@ -1,241 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback -import os -import glob -import cloudinit.CloudConfig as cc - - -def handle(_name, cfg, cloud, log, _args): - update = util.get_cfg_option_bool(cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) - - release = get_release() - - mirror = find_apt_mirror(cloud, cfg) - - log.debug("selected mirror at: %s" % mirror) - - if not util.get_cfg_option_bool(cfg, \ - 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") - rename_apt_lists(old_mir, mirror) - - # set up proxy - proxy = cfg.get("apt_proxy", None) - proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" - if proxy: - try: - contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename, "w") as fp: - fp.write(contents % proxy) - except Exception as e: - log.warn("Failed to write proxy to %s" % proxy_filename) - elif os.path.isfile(proxy_filename): - os.unlink(proxy_filename) - - # process 'apt_sources' - if 'apt_sources' in cfg: - errors = add_sources(cfg['apt_sources'], - {'MIRROR': mirror, 'RELEASE': release}) - for e in errors: - log.warn("Source Error: %s\n" % ':'.join(e)) - - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections") - log.debug(traceback.format_exc()) - - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) - - errors = [] - if update or len(pkglist) or upgrade: - try: - cc.update_package_sources() - except subprocess.CalledProcessError as e: - log.warn("apt-get update failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if upgrade: - try: - cc.apt_get("upgrade") - except subprocess.CalledProcessError as e: - log.warn("apt upgrade failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if len(pkglist): - try: - cc.install_packages(pkglist) - except subprocess.CalledProcessError as e: - log.warn("Failed to install packages: %s " % pkglist) - log.debug(traceback.format_exc()) - errors.append(e) - - if len(errors): - raise errors[0] - - return(True) - - -def mirror2lists_fileprefix(mirror): - string = mirror - # take of http:// or ftp:// - if string.endswith("/"): - string = string[0:-1] - pos = string.find("://") - if pos >= 0: - string = string[pos + 3:] - string = string.replace("/", "_") - return string - - -def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): - oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) - nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) - if(oprefix == nprefix): - return - olen = len(oprefix) - for filename in glob.glob("%s_*" % oprefix): - os.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], - stdout=subprocess.PIPE).communicate() - return(str(stdout).strip()) - - -def generate_sources_list(codename, mirror): - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - {'mirror': mirror, 'codename': codename}) - - -def add_sources(srclist, searchList=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList - """ - if searchList is None: - searchList = {} - elst = [] - - for ent in srclist: - if 'source' not in ent: - elst.append(["", "missing source"]) - continue - - source = ent['source'] - if source.startswith("ppa:"): - try: - util.subp(["add-apt-repository", source]) - except: - elst.append([source, "add-apt-repository failed"]) - continue - - source = util.render_string(source, searchList) - - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) - - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = util.getkeybyid(ent['keyid'], ks) - except: - elst.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - elst.append([source, "failed add key"]) - - try: - util.write_file(ent['filename'], source + "\n", omode="ab") - except: - elst.append([source, "failed write to file %s" % ent['filename']]) - - return(elst) - - -def find_apt_mirror(cloud, cfg): - """ find an apt_mirror given the cloud and cfg provided """ - - # TODO: distro and defaults should be configurable - distro = "ubuntu" - defaults = { - 'ubuntu': "http://archive.ubuntu.com/ubuntu", - 'debian': "http://archive.debian.org/debian", - } - mirror = None - - cfg_mirror = cfg.get("apt_mirror", None) - if cfg_mirror: - mirror = cfg["apt_mirror"] - elif "apt_mirror_search" in cfg: - mirror = util.search_for_mirror(cfg['apt_mirror_search']) - else: - if cloud: - mirror = cloud.get_mirror() - - mydom = "" - - doms = [] - - if not mirror and cloud: - # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - mydom = ".".join(fqdn.split(".")[1:]) - if mydom: - doms.append(".%s" % mydom) - - if not mirror: - doms.extend((".localdomain", "",)) - - mirror_list = [] - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) - for post in doms: - mirror_list.append(mirrorfmt % post) - - mirror = util.search_for_mirror(mirror_list) - - if not mirror: - mirror = defaults[distro] - - return mirror diff --git a/cloudinit/CloudConfig/cc_bootcmd.py b/cloudinit/CloudConfig/cc_bootcmd.py deleted file mode 100644 index f584da02..00000000 --- a/cloudinit/CloudConfig/cc_bootcmd.py +++ /dev/null @@ -1,48 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -import tempfile -import os -from cloudinit.CloudConfig import per_always -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if "bootcmd" not in cfg: - return - - try: - content = util.shellify(cfg["bootcmd"]) - tmpf = tempfile.TemporaryFile() - tmpf.write(content) - tmpf.seek(0) - except: - log.warn("failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - env['INSTANCE_ID'] = cloud.get_instance_id() - subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) - tmpf.close() - except: - log.warn("failed to run commands from bootcmd") - raise diff --git a/cloudinit/CloudConfig/cc_byobu.py b/cloudinit/CloudConfig/cc_byobu.py deleted file mode 100644 index e821b261..00000000 --- a/cloudinit/CloudConfig/cc_byobu.py +++ /dev/null @@ -1,77 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - value = args[0] - else: - value = util.get_cfg_option_str(cfg, "byobu_by_default", "") - - if not value: - return - - if value == "user" or value == "system": - value = "enable-%s" % value - - valid = ("enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable") - if not value in valid: - log.warn("Unknown value %s for byobu_by_default" % value) - - mod_user = value.endswith("-user") - mod_sys = value.endswith("-system") - if value.startswith("enable"): - bl_inst = "install" - dc_val = "byobu byobu/launch-by-default boolean true" - mod_sys = True - else: - if value == "disable": - mod_user = True - mod_sys = True - bl_inst = "uninstall" - dc_val = "byobu byobu/launch-by-default boolean false" - - shcmd = "" - if mod_user: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) - shcmd += " || X=$(($X+1)); " - if mod_sys: - shcmd += "echo \"%s\" | debconf-set-selections" % dc_val - shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" - shcmd += " || X=$(($X+1)); " - - cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - - log.debug("setting byobu to %s" % value) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/CloudConfig/cc_ca_certs.py b/cloudinit/CloudConfig/cc_ca_certs.py deleted file mode 100644 index 3af6238a..00000000 --- a/cloudinit/CloudConfig/cc_ca_certs.py +++ /dev/null @@ -1,90 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Mike Milner -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import os -from subprocess import check_call -from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents, subp) - -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" - - -def update_ca_certs(): - """ - Updates the CA certificate cache on the current machine. - """ - check_call(["update-ca-certificates"]) - - -def add_ca_certs(certs): - """ - Adds certificates to the system. To actually apply the new certificates - you must also call L{update_ca_certs}. - - @param certs: A list of certificate strings. - """ - if certs: - cert_file_contents = "\n".join(certs) - cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - write_file(cert_file_fullpath, cert_file_contents, mode=0644) - # Append cert filename to CA_CERT_CONFIG file. - write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") - - -def remove_default_ca_certs(): - """ - Removes all default trusted CA certificates from the system. To actually - apply the change you must also call L{update_ca_certs}. - """ - delete_dir_contents(CA_CERT_PATH) - delete_dir_contents(CA_CERT_SYSTEM_PATH) - write_file(CA_CERT_CONFIG, "", mode=0644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp(('debconf-set-selections', '-'), debconf_sel) - - -def handle(_name, cfg, _cloud, log, _args): - """ - Call to handle ca-cert sections in cloud-config file. - - @param name: The module name "ca-cert" from cloud.cfg - @param cfg: A nested dict containing the entire cloud config contents. - @param cloud: The L{CloudInit} object in use. - @param log: Pre-initialized Python logger object to use for logging. - @param args: Any module arguments from cloud.cfg - """ - # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: - return - ca_cert_cfg = cfg['ca-certs'] - - # If there is a remove-defaults option set to true, remove the system - # default trusted CA certs first. - if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") - remove_default_ca_certs() - - # If we are given any new trusted CA certs to add, add them. - if "trusted" in ca_cert_cfg: - trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") - if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) - - # Update the system with the new cert configuration. - update_ca_certs() diff --git a/cloudinit/CloudConfig/cc_chef.py b/cloudinit/CloudConfig/cc_chef.py deleted file mode 100644 index 941e04fe..00000000 --- a/cloudinit/CloudConfig/cc_chef.py +++ /dev/null @@ -1,119 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Avishai Ish-Shalom -# Author: Mike Moulton -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import json -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -ruby_version_default = "1.8" - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a chef key in the configuration don't do anything - if 'chef' not in cfg: - return - chef_cfg = cfg['chef'] - - # ensure the chef directories we use exist - mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', - '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) - - # set the validation key based on the presence of either 'validation_key' - # or 'validation_cert'. In the case where both exist, 'validation_key' - # takes precedence - for key in ('validation_key', 'validation_cert'): - if key in chef_cfg and chef_cfg[key]: - with open('/etc/chef/validation.pem', 'w') as validation_key_fh: - validation_key_fh.write(chef_cfg[key]) - break - - # create the chef config from template - util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) - - # set the firstboot json - with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): - initial_json[k] = initial_attributes[k] - firstboot_json_fh.write(json.dumps(initial_json)) - - # If chef is not installed, we install chef based on 'install_type' - if not os.path.isfile('/usr/bin/chef-client'): - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - ruby_version_default) - install_chef_from_gems(ruby_version, chef_version) - # and finally, run chef-client - log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', - '-s', '20']) - else: - # this will install and run the chef-client from packages - cc.install_packages(('chef',)) - - -def get_ruby_packages(version): - # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] - if version == "1.8": - pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return(pkgs) - - -def install_chef_from_gems(ruby_version, chef_version=None): - cc.install_packages(get_ruby_packages(ruby_version)) - if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') - if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') - if chef_version: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q']) - else: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q']) - - -def ensure_dir(d): - if not os.path.exists(d): - os.makedirs(d) - - -def mkdirs(dirs): - for d in dirs: - ensure_dir(d) diff --git a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py b/cloudinit/CloudConfig/cc_disable_ec2_metadata.py deleted file mode 100644 index 6b31ea8e..00000000 --- a/cloudinit/CloudConfig/cc_disable_ec2_metadata.py +++ /dev/null @@ -1,30 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall = "route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) diff --git a/cloudinit/CloudConfig/cc_final_message.py b/cloudinit/CloudConfig/cc_final_message.py deleted file mode 100644 index abb4ca32..00000000 --- a/cloudinit/CloudConfig/cc_final_message.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_always -import sys -from cloudinit import util, boot_finished -import time - -frequency = per_always - -final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - msg_in = args[0] - else: - msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) - - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - log.warn("unable to open /proc/uptime\n") - uptime = "na" - - try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: - ts = "na" - - try: - subs = {'UPTIME': uptime, 'TIMESTAMP': ts} - sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) - except Exception as e: - log.warn("failed to render string to stdout: %s" % e) - - fp = open(boot_finished, "wb") - fp.write(uptime + "\n") - fp.close() diff --git a/cloudinit/CloudConfig/cc_foo.py b/cloudinit/CloudConfig/cc_foo.py deleted file mode 100644 index 35ec3fa7..00000000 --- a/cloudinit/CloudConfig/cc_foo.py +++ /dev/null @@ -1,29 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#import cloudinit -#import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance - - -def handle(_name, _cfg, _cloud, _log, _args): - print "hi" diff --git a/cloudinit/CloudConfig/cc_grub_dpkg.py b/cloudinit/CloudConfig/cc_grub_dpkg.py deleted file mode 100644 index 9f3a7eaf..00000000 --- a/cloudinit/CloudConfig/cc_grub_dpkg.py +++ /dev/null @@ -1,64 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import traceback -import os - - -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None - - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) - - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs == None: - idevs = "" - if idevs_empty == None: - idevs_empty = "true" - else: - if idevs_empty == None: - idevs_empty = "false" - if idevs == None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): - if os.path.exists(dev): - idevs = dev - break - - # now idevs and idevs_empty are set to determined values - # or, those set by user - - dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ - "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty - log.debug("setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) - - try: - util.subp(('debconf-set-selections'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections for grub-dpkg") - log.debug(traceback.format_exc()) diff --git a/cloudinit/CloudConfig/cc_keys_to_console.py b/cloudinit/CloudConfig/cc_keys_to_console.py deleted file mode 100644 index 73a477c0..00000000 --- a/cloudinit/CloudConfig/cc_keys_to_console.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -import subprocess - -frequency = per_instance - - -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_key_console_blacklist", ["ssh-dss"]) - try: - confp = open('/dev/console', "wb") - cmd.append(','.join(fp_blacklist)) - cmd.append(','.join(key_blacklist)) - subprocess.call(cmd, stdout=confp) - confp.close() - except: - log.warn("writing keys to console value") - raise diff --git a/cloudinit/CloudConfig/cc_landscape.py b/cloudinit/CloudConfig/cc_landscape.py deleted file mode 100644 index a4113cbe..00000000 --- a/cloudinit/CloudConfig/cc_landscape.py +++ /dev/null @@ -1,75 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -from cloudinit.CloudConfig import per_instance -from configobj import ConfigObj - -frequency = per_instance - -lsc_client_cfg_file = "/etc/landscape/client.conf" - -# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 -lsc_builtincfg = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", - } -} - - -def handle(_name, cfg, _cloud, log, _args): - """ - Basically turn a top level 'landscape' entry with a 'client' dict - and render it to ConfigObj format under '[client]' section in - /etc/landscape/client.conf - """ - - ls_cloudcfg = cfg.get("landscape", {}) - - if not isinstance(ls_cloudcfg, dict): - raise(Exception("'landscape' existed in config, but not a dict")) - - merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) - - if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): - os.makedirs(os.path.dirname(lsc_client_cfg_file)) - - with open(lsc_client_cfg_file, "w") as fp: - merged.write(fp) - - log.debug("updated %s" % lsc_client_cfg_file) - - -def mergeTogether(objs): - """ - merge together ConfigObj objects or things that ConfigObj() will take in - later entries override earlier - """ - cfg = ConfigObj({}) - for obj in objs: - if isinstance(obj, ConfigObj): - cfg.merge(obj) - else: - cfg.merge(ConfigObj(obj)) - return cfg diff --git a/cloudinit/CloudConfig/cc_locale.py b/cloudinit/CloudConfig/cc_locale.py deleted file mode 100644 index 2bb22fdb..00000000 --- a/cloudinit/CloudConfig/cc_locale.py +++ /dev/null @@ -1,54 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os.path -import subprocess -import traceback - - -def apply_locale(locale, cfgfile): - if os.path.exists('/usr/sbin/locale-gen'): - subprocess.Popen(['locale-gen', locale]).communicate() - if os.path.exists('/usr/sbin/update-locale'): - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', cfgfile, {'locale': locale}) - - -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - locale = args[0] - else: - locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - - locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", - "/etc/default/locale") - - if not locale: - return - - log.debug("setting locale to %s" % locale) - - try: - apply_locale(locale, locale_cfgfile) - except Exception as e: - log.debug(traceback.format_exc(e)) - raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/CloudConfig/cc_mcollective.py b/cloudinit/CloudConfig/cc_mcollective.py deleted file mode 100644 index a2a6230c..00000000 --- a/cloudinit/CloudConfig/cc_mcollective.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Marc Cluet -# Based on code by Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -pubcert_file = "/etc/mcollective/ssl/server-public.pem" -pricert_file = "/etc/mcollective/ssl/server-private.pem" - - -# Our fake header section -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[nullsection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a mcollective key in the configuration don't do anything - if 'mcollective' not in cfg: - return - mcollective_cfg = cfg['mcollective'] - # Start by installing the mcollective package ... - cc.install_packages(("mcollective",)) - - # ... and then update the mcollective configuration - if 'conf' in mcollective_cfg: - # Create object for reading server.cfg values - mcollective_config = ConfigParser.ConfigParser() - # Read server.cfg values from original file in order to be able to mix - # the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' - 'server.cfg'))) - for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): - if cfg_name == 'public-cert': - util.write_file(pubcert_file, cfg, mode=0644) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_public', pubcert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - elif cfg_name == 'private-cert': - util.write_file(pricert_file, cfg, mode=0600) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_private', pricert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - else: - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - mcollective_config.set(cfg_name, o, v) - # We got all our config as wanted we'll rename - # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg', - '/etc/mcollective/server.cfg.old') - outputfile = StringIO.StringIO() - mcollective_config.write(outputfile) - # Now we got the whole file, write to disk except first line - # Note below, that we've just used ConfigParser because it generally - # works. Below, we remove the initial 'nullsection' header - # and then change 'key = value' to 'key: value'. The global - # search and replace of '=' with ':' could be problematic though. - # this most likely needs fixing. - util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', - ':'), - mode=0644) - - # Start mcollective - subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/CloudConfig/cc_mounts.py b/cloudinit/CloudConfig/cc_mounts.py deleted file mode 100644 index 6cdd74e8..00000000 --- a/cloudinit/CloudConfig/cc_mounts.py +++ /dev/null @@ -1,179 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os -import re -from string import whitespace # pylint: disable=W0402 - - -def is_mdname(name): - # return true if this is a metadata service name - if name in ["ami", "root", "swap"]: - return True - # names 'ephemeral0' or 'ephemeral1' - # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' - for enumname in ("ephemeral", "ebs"): - if name.startswith(enumname) and name.find(":") == -1: - return True - return False - - -def handle(_name, cfg, cloud, log, _args): - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] - defvals = cfg.get("mount_default_fields", defvals) - - # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], - ["swap", "none", "swap", "sw", "0", "0"]] - - cfgmnt = [] - if "mounts" in cfg: - cfgmnt = cfg["mounts"] - - # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 - shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" - shortname = re.compile(shortname_filter) - - for i in range(len(cfgmnt)): - # skip something that wasn't a list - if not isinstance(cfgmnt[i], list): - continue - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - - if is_mdname(cfgmnt[i][0]): - newname = cloud.device_name_to_device(cfgmnt[i][0]) - if not newname: - log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) - cfgmnt[i][1] = None - else: - if newname.startswith("/"): - cfgmnt[i][0] = newname - else: - cfgmnt[i][0] = "/dev/%s" % newname - else: - if shortname.match(cfgmnt[i][0]): - cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] - - # in case the user did not quote a field (likely fs-freq, fs_passno) - # but do not convert None to 'None' (LP: #898365) - for j in range(len(cfgmnt[i])): - if isinstance(cfgmnt[i][j], int): - cfgmnt[i][j] = str(cfgmnt[i][j]) - - for i in range(len(cfgmnt)): - # fill in values with defaults from defvals above - for j in range(len(defvals)): - if len(cfgmnt[i]) <= j: - cfgmnt[i].append(defvals[j]) - elif cfgmnt[i][j] is None: - cfgmnt[i][j] = defvals[j] - - # if the second entry in the list is 'None' this - # clears all previous entries of that same 'fs_spec' - # (fs_spec is the first field in /etc/fstab, ie, that device) - if cfgmnt[i][1] is None: - for j in range(i): - if cfgmnt[j][0] == cfgmnt[i][0]: - cfgmnt[j][1] = None - - # for each of the "default" mounts, add them only if no other - # entry has the same device name - for defmnt in defmnts: - devname = cloud.device_name_to_device(defmnt[0]) - if devname is None: - continue - if devname.startswith("/"): - defmnt[0] = devname - else: - defmnt[0] = "/dev/%s" % devname - - cfgmnt_has = False - for cfgm in cfgmnt: - if cfgm[0] == defmnt[0]: - cfgmnt_has = True - break - - if cfgmnt_has: - continue - cfgmnt.append(defmnt) - - # now, each entry in the cfgmnt list has all fstab values - # if the second field is None (not the string, the value) we skip it - actlist = [x for x in cfgmnt if x[1] is not None] - - if len(actlist) == 0: - return - - comment = "comment=cloudconfig" - cc_lines = [] - needswap = False - dirs = [] - for line in actlist: - # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] - if line[2] == "swap": - needswap = True - if line[1].startswith("/"): - dirs.append(line[1]) - cc_lines.append('\t'.join(line)) - - fstab_lines = [] - fstab = open("/etc/fstab", "r+") - ws = re.compile("[%s]+" % whitespace) - for line in fstab.read().splitlines(): - try: - toks = ws.split(line) - if toks[3].find(comment) != -1: - continue - except: - pass - fstab_lines.append(line) - - fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() - - if needswap: - try: - util.subp(("swapon", "-a")) - except: - log.warn("Failed to enable swap") - - for d in dirs: - if os.path.exists(d): - continue - try: - os.makedirs(d) - except: - log.warn("Failed to make '%s' config-mount\n", d) - - try: - util.subp(("mount", "-a")) - except: - log.warn("'mount -a' failed") diff --git a/cloudinit/CloudConfig/cc_phone_home.py b/cloudinit/CloudConfig/cc_phone_home.py deleted file mode 100644 index a7ff74e1..00000000 --- a/cloudinit/CloudConfig/cc_phone_home.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -from time import sleep - -frequency = per_instance -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', - 'hostname'] - - -# phone_home: -# url: http://my.foo.bar/$INSTANCE/ -# post: all -# tries: 10 -# -# phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ -# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id -# -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - ph_cfg = util.read_conf(args[0]) - else: - if not 'phone_home' in cfg: - return - ph_cfg = cfg['phone_home'] - - if 'url' not in ph_cfg: - log.warn("no 'url' token in phone_home") - return - - url = ph_cfg['url'] - post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries', 10) - try: - tries = int(tries) - except: - log.warn("tries is not an integer. using 10") - tries = 10 - - if post_list == "all": - post_list = post_list_all - - all_keys = {} - all_keys['instance_id'] = cloud.get_instance_id() - all_keys['hostname'] = cloud.get_hostname() - - pubkeys = { - 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', - 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', - 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', - } - - for n, path in pubkeys.iteritems(): - try: - fp = open(path, "rb") - all_keys[n] = fp.read() - fp.close() - except: - log.warn("%s: failed to open in phone_home" % path) - - submit_keys = {} - for k in post_list: - if k in all_keys: - submit_keys[k] = all_keys[k] - else: - submit_keys[k] = "N/A" - log.warn("requested key %s from 'post' list not available") - - url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - - null_exc = object() - last_e = null_exc - for i in range(0, tries): - try: - util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i + 1)) - return - except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i + 1)) - last_e = e - sleep(3) - - log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e is not null_exc: - raise(last_e) - - return diff --git a/cloudinit/CloudConfig/cc_puppet.py b/cloudinit/CloudConfig/cc_puppet.py deleted file mode 100644 index 6fc475f6..00000000 --- a/cloudinit/CloudConfig/cc_puppet.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import pwd -import socket -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a puppet key in the configuration don't do anything - if 'puppet' not in cfg: - return - puppet_cfg = cfg['puppet'] - # Start by installing the puppet package ... - cc.install_packages(("puppet",)) - - # ... and then update the puppet configuration - if 'conf' in puppet_cfg: - # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') - # Create object for reading puppet.conf values - puppet_config = ConfigParser.ConfigParser() - # Read puppet.conf values from original file in order to be able to - # mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in - puppet_conf_fh.readlines()))) - # Close original file, no longer needed - puppet_conf_fh.close() - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): - # ca_cert configuration is a special case - # Dump the puppetmaster ca certificate in the correct place - if cfg_name == 'ca_cert': - # Puppet ssl sub-directory isn't created yet - # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - util.restorecon_if_possible('/var/lib/puppet', recursive=True) - else: - #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to - # write it again - if puppet_config.has_section(cfg_name) == False: - puppet_config.add_section(cfg_name) - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - if o == 'certname': - # Expand %f as the fqdn - v = v.replace("%f", socket.getfqdn()) - # Expand %i as the instance id - v = v.replace("%i", - cloud.datasource.get_instance_id()) - # certname needs to be downcase - v = v.lower() - puppet_config.set(cfg_name, o, v) - #puppet_conf_fh.write("%s=%s\n" % (o, v)) - # We got all our config as wanted we'll rename - # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') - with open('/etc/puppet/puppet.conf', 'wb') as configfile: - puppet_config.write(configfile) - util.restorecon_if_possible('/etc/puppet/puppet.conf') - # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) - elif os.path.exists('/bin/systemctl'): - subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) - elif os.path.exists('/sbin/chkconfig'): - subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) - else: - log.warn("Do not know how to enable puppet service on this system") - # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/CloudConfig/cc_resizefs.py b/cloudinit/CloudConfig/cc_resizefs.py deleted file mode 100644 index 2dc66def..00000000 --- a/cloudinit/CloudConfig/cc_resizefs.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import os -import stat -import sys -import time -import tempfile -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = False - if str(args[0]).lower() in ['true', '1', 'on', 'yes']: - resize_root = True - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if str(resize_root).lower() in ['false', '0']: - return - - # we use mktemp rather than mkstemp because early in boot nothing - # else should be able to race us for this, and we need to mknod. - devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") - - try: - st_dev = os.stat("/").st_dev - dev = os.makedev(os.major(st_dev), os.minor(st_dev)) - os.mknod(devpth, 0400 | stat.S_IFBLK, dev) - except: - if util.is_container(): - log.debug("inside container, ignoring mknod failure in resizefs") - return - log.warn("Failed to make device node to resize /") - raise - - cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] - try: - (fstype, _err) = util.subp(cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % - (os.major(st_dev), os.minor(st_dev), cmd)) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - if str(fstype).startswith("ext"): - resize_cmd = ['resize2fs', devpth] - elif fstype == "xfs": - resize_cmd = ['xfs_growfs', devpth] - else: - os.unlink(devpth) - log.debug("not resizing unknown filesystem %s" % fstype) - return - - if resize_root == "noblock": - fid = os.fork() - if fid == 0: - try: - do_resize(resize_cmd, devpth, log) - os._exit(0) # pylint: disable=W0212 - except Exception as exc: - sys.stderr.write("Failed: %s" % exc) - os._exit(1) # pylint: disable=W0212 - else: - do_resize(resize_cmd, devpth, log) - - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % - (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), - resize_root)) - - return - - -def do_resize(resize_cmd, devpth, log): - try: - start = time.time() - util.subp(resize_cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to resize filesystem (%s)" % resize_cmd) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - os.unlink(devpth) - log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/CloudConfig/cc_rightscale_userdata.py b/cloudinit/CloudConfig/cc_rightscale_userdata.py deleted file mode 100644 index 5ed0848f..00000000 --- a/cloudinit/CloudConfig/cc_rightscale_userdata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur -from urlparse import parse_qs - -frequency = per_instance -my_name = "cc_rightscale_userdata" -my_hookname = 'CLOUD_INIT_REMOTE_HOOK' - - -def handle(_name, _cfg, cloud, log, _args): - try: - ud = cloud.get_userdata_raw() - except: - log.warn("failed to get raw userdata in %s" % my_name) - return - - try: - mdict = parse_qs(ud) - if not my_hookname in mdict: - return - except: - log.warn("failed to urlparse.parse_qa(userdata_raw())") - raise - - scripts_d = get_ipath_cur('scripts') - i = 0 - first_e = None - for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d, i) - i = i + 1 - try: - content = util.readurl(url) - util.write_file(fname, content, mode=0700) - except Exception as e: - if not first_e: - first_e = None - log.warn("%s failed to read %s: %s" % (my_name, url, e)) - - if first_e: - raise(e) diff --git a/cloudinit/CloudConfig/cc_rsyslog.py b/cloudinit/CloudConfig/cc_rsyslog.py deleted file mode 100644 index ac7f2c74..00000000 --- a/cloudinit/CloudConfig/cc_rsyslog.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab syntax=python -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit -import logging -import cloudinit.util as util -import traceback - -DEF_FILENAME = "20-cloud-config.conf" -DEF_DIR = "/etc/rsyslog.d" - - -def handle(_name, cfg, _cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - - # process 'rsyslog' - if not 'rsyslog' in cfg: - return - - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) - - files = [] - elst = [] - for ent in cfg['rsyslog']: - if isinstance(ent, dict): - if not "content" in ent: - elst.append((ent, "no 'content' entry")) - continue - content = ent['content'] - filename = ent.get("filename", def_fname) - else: - content = ent - filename = def_fname - - if not filename.startswith("/"): - filename = "%s/%s" % (def_dir, filename) - - omode = "ab" - # truncate filename first time you see it - if filename not in files: - omode = "wb" - files.append(filename) - - try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: - log.debug(traceback.format_exc(e)) - elst.append((content, "failed to write to %s" % filename)) - - # need to restart syslogd - restarted = False - try: - # if this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set - log.debug("restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - - except Exception as e: - elst.append(("restart", str(e))) - - if restarted: - # this only needs to run if we *actually* restarted - # syslog above. - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.debug("rsyslog configured %s" % files) - - for e in elst: - log.warn("rsyslog error: %s\n" % ':'.join(e)) - - return diff --git a/cloudinit/CloudConfig/cc_runcmd.py b/cloudinit/CloudConfig/cc_runcmd.py deleted file mode 100644 index f7e8c671..00000000 --- a/cloudinit/CloudConfig/cc_runcmd.py +++ /dev/null @@ -1,32 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if "runcmd" not in cfg: - return - outfile = "%s/runcmd" % cloud.get_ipath('scripts') - try: - content = util.shellify(cfg["runcmd"]) - util.write_file(outfile, content, 0700) - except: - log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/CloudConfig/cc_salt_minion.py b/cloudinit/CloudConfig/cc_salt_minion.py deleted file mode 100644 index 1a3b5039..00000000 --- a/cloudinit/CloudConfig/cc_salt_minion.py +++ /dev/null @@ -1,56 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Jeff Bauer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import subprocess -import cloudinit.CloudConfig as cc -import yaml - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a salt key in the configuration don't do anything - if 'salt_minion' not in cfg: - return - salt_cfg = cfg['salt_minion'] - # Start by installing the salt package ... - cc.install_packages(("salt",)) - config_dir = '/etc/salt' - if not os.path.isdir(config_dir): - os.makedirs(config_dir) - # ... and then update the salt configuration - if 'conf' in salt_cfg: - # Add all sections from the conf object to /etc/salt/minion - minion_config = os.path.join(config_dir, 'minion') - yaml.dump(salt_cfg['conf'], - file(minion_config, 'w'), - default_flow_style=False) - # ... copy the key pair if specified - if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = '/etc/salt/pki' - cumask = os.umask(077) - if not os.path.isdir(pki_dir): - os.makedirs(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - with open(pub_name, 'w') as f: - f.write(salt_cfg['public_key']) - with open(pem_name, 'w') as f: - f.write(salt_cfg['private_key']) - os.umask(cumask) - - # Start salt-minion - subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/CloudConfig/cc_scripts_per_boot.py b/cloudinit/CloudConfig/cc_scripts_per_boot.py deleted file mode 100644 index 41a74754..00000000 --- a/cloudinit/CloudConfig/cc_scripts_per_boot.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -from cloudinit import get_cpath - -frequency = per_always -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_instance.py b/cloudinit/CloudConfig/cc_scripts_per_instance.py deleted file mode 100644 index a2981eab..00000000 --- a/cloudinit/CloudConfig/cc_scripts_per_instance.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_cpath - -frequency = per_instance -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_scripts_per_once.py b/cloudinit/CloudConfig/cc_scripts_per_once.py deleted file mode 100644 index a69151da..00000000 --- a/cloudinit/CloudConfig/cc_scripts_per_once.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_once -from cloudinit import get_cpath - -frequency = per_once -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_scripts_user.py b/cloudinit/CloudConfig/cc_scripts_user.py deleted file mode 100644 index 933aa4e0..00000000 --- a/cloudinit/CloudConfig/cc_scripts_user.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur - -frequency = per_instance -runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/CloudConfig/cc_set_hostname.py b/cloudinit/CloudConfig/cc_set_hostname.py deleted file mode 100644 index acea74d9..00000000 --- a/cloudinit/CloudConfig/cc_set_hostname.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not setting hostname") - return(True) - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - set_hostname(hostname, log) - except Exception: - util.logexc(log) - log.warn("failed to set hostname to %s\n", hostname) - - return(True) - - -def set_hostname(hostname, log): - util.subp(['hostname', hostname]) - util.write_file("/etc/hostname", "%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/CloudConfig/cc_set_passwords.py b/cloudinit/CloudConfig/cc_set_passwords.py deleted file mode 100644 index 9d0bbdb8..00000000 --- a/cloudinit/CloudConfig/cc_set_passwords.py +++ /dev/null @@ -1,129 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import sys -import random -from string import letters, digits # pylint: disable=W0402 - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - # if run from command line, and give args, wipe the chpasswd['list'] - password = args[0] - if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: - del cfg['chpasswd']['list'] - else: - password = util.get_cfg_option_str(cfg, "password", None) - - expire = True - pw_auth = "no" - change_pwauth = False - plist = None - - if 'chpasswd' in cfg: - chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg, 'list', plist) - expire = util.get_cfg_option_bool(chfg, 'expire', expire) - - if not plist and password: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - plist = "%s:%s" % (user, password) - - errors = [] - if plist: - plist_in = [] - randlist = [] - users = [] - for line in plist.splitlines(): - u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) - - ch_in = '\n'.join(plist_in) - try: - util.subp(['chpasswd'], ch_in) - log.debug("changed password for %s:" % users) - except Exception as e: - errors.append(e) - log.warn("failed to set passwords with chpasswd: %s" % e) - - if len(randlist): - sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", - '\n'.join(randlist))) - - if expire: - enum = len(errors) - for u in users: - try: - util.subp(['passwd', '--expire', u]) - except Exception as e: - errors.append(e) - log.warn("failed to expire account for %s" % u) - if enum == len(errors): - log.debug("expired passwords for: %s" % u) - - if 'ssh_pwauth' in cfg: - val = str(cfg['ssh_pwauth']).lower() - if val in ("true", "1", "yes"): - pw_auth = "yes" - change_pwauth = True - elif val in ("false", "0", "no"): - pw_auth = "no" - change_pwauth = True - else: - change_pwauth = False - - if change_pwauth: - pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" - msg = "set PasswordAuthentication to '%s'" % pw_auth - try: - cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config'] - util.subp(cmd) - log.debug(msg) - except Exception as e: - log.warn("failed %s" % msg) - errors.append(e) - - try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), - 'restart']) - log.debug("restarted sshd") - except: - log.warn("restart of ssh failed") - - if len(errors): - raise(errors[0]) - - return - - -def rand_str(strlen=32, select_from=letters + digits): - return("".join([random.choice(select_from) for _x in range(0, strlen)])) - - -def rand_user_password(pwlen=9): - selfrom = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) - return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/CloudConfig/cc_ssh.py b/cloudinit/CloudConfig/cc_ssh.py deleted file mode 100644 index 48eb58bc..00000000 --- a/cloudinit/CloudConfig/cc_ssh.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import cloudinit.SshUtil as sshutil -import os -import glob -import subprocess - -DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ -"rather than the user \\\"root\\\".\';echo;sleep 10\"" - - -def handle(_name, cfg, cloud, log, _args): - - # remove the static keys from the pristine image - if cfg.get("ssh_deletekeys", True): - for f in glob.glob("/etc/ssh/ssh_host_*key*"): - try: - os.unlink(f) - except: - pass - - if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them - key2file = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), - } - - for key, val in cfg["ssh_keys"].items(): - if key in key2file: - util.write_file(key2file[key][0], val, key2file[key][1]) - - priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', } - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' - for priv, pub in priv2pub.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: - continue - pair = (key2file[priv][0], key2file[pub][0]) - subprocess.call(('sh', '-xc', cmd % pair)) - log.debug("generated %s from %s" % pair) - else: - # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype - if not os.path.exists(keyfile): - subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', - '-f', keyfile]) - - util.restorecon_if_possible('/etc/ssh', recursive=True) - - try: - user = util.get_cfg_option_str(cfg, 'user') - disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) - disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) - keys = cloud.get_public_ssh_keys() - - if "ssh_authorized_keys" in cfg: - cfgkeys = cfg["ssh_authorized_keys"] - keys.extend(cfgkeys) - - apply_credentials(keys, user, disable_root, disable_root_opts, log) - except: - util.logexc(log) - log.warn("applying credentials failed!\n") - - -def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): - keys = set(keys) - if user: - sshutil.setup_user_keys(keys, user, '', log) - - if disable_root: - key_prefix = disable_root_opts.replace('$USER', user) - else: - key_prefix = '' - - sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/CloudConfig/cc_ssh_import_id.py b/cloudinit/CloudConfig/cc_ssh_import_id.py deleted file mode 100644 index bbf5bd83..00000000 --- a/cloudinit/CloudConfig/cc_ssh_import_id.py +++ /dev/null @@ -1,50 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - user = args[0] - ids = [] - if len(args) > 1: - ids = args[1:] - else: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) - - if len(ids) == 0: - return - - cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - - log.debug("importing ssh ids. cmd = %s" % cmd) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/CloudConfig/cc_timezone.py b/cloudinit/CloudConfig/cc_timezone.py deleted file mode 100644 index e5c9901b..00000000 --- a/cloudinit/CloudConfig/cc_timezone.py +++ /dev/null @@ -1,67 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -from cloudinit import util -import os.path -import shutil - -frequency = per_instance -tz_base = "/usr/share/zoneinfo" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - timezone = args[0] - else: - timezone = util.get_cfg_option_str(cfg, "timezone", False) - - if not timezone: - return - - tz_file = "%s/%s" % (tz_base, timezone) - - if not os.path.isfile(tz_file): - log.debug("Invalid timezone %s" % tz_file) - raise Exception("Invalid timezone %s" % tz_file) - - try: - fp = open("/etc/timezone", "wb") - fp.write("%s\n" % timezone) - fp.close() - except: - log.debug("failed to write to /etc/timezone") - raise - if os.path.exists("/etc/sysconfig/clock"): - try: - with open("/etc/sysconfig/clock", "w") as fp: - fp.write('ZONE="%s"\n' % timezone) - except: - log.debug("failed to write to /etc/sysconfig/clock") - raise - - try: - shutil.copy(tz_file, "/etc/localtime") - except: - log.debug("failed to copy %s to /etc/localtime" % tz_file) - raise - - log.debug("set timezone to %s" % timezone) - return diff --git a/cloudinit/CloudConfig/cc_update_etc_hosts.py b/cloudinit/CloudConfig/cc_update_etc_hosts.py deleted file mode 100644 index 6ad2fca8..00000000 --- a/cloudinit/CloudConfig/cc_update_etc_hosts.py +++ /dev/null @@ -1,87 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -import StringIO - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - - manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if manage_hosts in ("True", "true", True, "template"): - # render from template file - try: - if not hostname: - log.info("manage_etc_hosts was set, but no hostname found") - return - - util.render_to_file('hosts', '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) - except Exception: - log.warn("failed to update /etc/hosts") - raise - elif manage_hosts == "localhost": - log.debug("managing 127.0.1.1 in /etc/hosts") - update_etc_hosts(hostname, fqdn, log) - return - else: - if manage_hosts not in ("False", False): - log.warn("Unknown value for manage_etc_hosts. Assuming False") - else: - log.debug("not managing /etc/hosts") - - -def update_etc_hosts(hostname, fqdn, _log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: - new_etchosts.write(line) - continue - if line == header: - continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open('/etc/hosts', 'wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return diff --git a/cloudinit/CloudConfig/cc_update_hostname.py b/cloudinit/CloudConfig/cc_update_hostname.py deleted file mode 100644 index b9d1919a..00000000 --- a/cloudinit/CloudConfig/cc_update_hostname.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import errno -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not updating hostname") - return - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") - update_hostname(hostname, prev, log) - except Exception: - log.warn("failed to set hostname\n") - raise - - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename, "r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError as e: - if e.errno != errno.ENOENT: - raise - return default - - -def update_hostname(hostname, prev_file, log): - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None - - try: - hostname_prev = read_hostname(prev_file) - except Exception as e: - log.warn("Failed to open %s: %s" % (prev_file, e)) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname, "%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname, fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file, etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/DataSource.py b/cloudinit/DataSource.py deleted file mode 100644 index e2a9150d..00000000 --- a/cloudinit/DataSource.py +++ /dev/null @@ -1,214 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" - -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket - - -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - - self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) - - def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) - return self.userdata - - def get_userdata_raw(self): - return(self.userdata_raw) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return({}) - - def get_public_ssh_keys(self): - keys = [] - if 'public-keys' not in self.metadata: - return([]) - - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) - - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) - - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return(keys) - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return(None) - - def get_locale(self): - return('en_US.UTF-8') - - def get_local_mirror(self): - return None - - def get_instance_id(self): - if 'instance-id' not in self.metadata: - return "iid-datasource" - return(self.metadata['instance-id']) - - def get_hostname(self, fqdn=False): - defdomain = "localdomain" - defhost = "localhost" - - domain = defdomain - if not 'local-hostname' in self.metadata: - - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - - hostname = socket.gethostname() - - fqdn = util.get_fqdn_from_hosts(hostname) - - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".", "-") - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = '.'.join(toks[1:]) - else: - hostname = toks[0] - - if fqdn: - return "%s.%s" % (hostname, domain) - else: - return hostname - - -# return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSourceCollections" modules -# and calling their "get_datasource_list". -# return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] - for ds_coll in cfg_list: - for pkg in pkglist: - if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) - - -# depends is a list of dependencies (DEP_FILESYSTEM) -# dslist is a list of 2 item lists -# dslist = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# it returns a list of 'class' that matched these deps exactly -# it is a helper function for DataSourceCollections -def list_from_depends(depends, dslist): - retlist = [] - depset = set(depends) - for elem in dslist: - (cls, deps) = elem - if depset == set(deps): - retlist.append(cls) - return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) diff --git a/cloudinit/DataSourceCloudStack.py b/cloudinit/DataSourceCloudStack.py deleted file mode 100644 index 5afdf7b6..00000000 --- a/cloudinit/DataSourceCloudStack.py +++ /dev/null @@ -1,92 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Cosmin Luta -# -# Author: Cosmin Luta -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -from socket import inet_ntoa -import time -import boto.utils as boto_utils -from struct import pack - - -class DataSourceCloudStack(DataSource.DataSource): - api_ver = 'latest' - seeddir = base_seeddir + '/cs' - metadata_address = None - - def __init__(self, sys_cfg=None): - DataSource.DataSource.__init__(self, sys_cfg) - # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ - self.metadata_address = "http://%s/" % self.get_default_gateway() - - def get_default_gateway(self): - """ Returns the default gateway ip address in the dotted format - """ - with open("/proc/net/route", "r") as f: - for line in f.readlines(): - items = line.split("\t") - if items[1] == "00000000": - # found the default route, get the gateway - gw = inet_ntoa(pack(" -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -import json -import subprocess - -DEFAULT_IID = "iid-dsconfigdrive" - - -class DataSourceConfigDrive(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/config_drive' - cfg = {} - userdata_raw = None - metadata = None - dsmode = "local" - - def __str__(self): - mstr = "DataSourceConfigDrive[%s]" % self.dsmode - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = None - md = {} - ud = "" - - defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} - - if os.path.isdir(self.seeddir): - try: - (md, ud) = read_config_drive_dir(self.seeddir) - found = self.seeddir - except nonConfigDriveDir: - pass - - if not found: - dev = cfg_drive_device() - if dev: - try: - (md, ud) = util.mount_callback_umount(dev, - read_config_drive_dir) - found = dev - except (nonConfigDriveDir, util.mountFailedError): - pass - - if not found: - return False - - if 'dsconfig' in md: - self.cfg = md['dscfg'] - - md = util.mergedict(md, defaults) - - # update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. - if 'network-interfaces' in md and self.dsmode == "local": - if md['dsmode'] == "pass": - log.info("updating network interfaces from configdrive") - else: - log.debug("updating network interfaces from configdrive") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = found - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return(self.metadata['public-keys']) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceConfigDriveNet(DataSourceConfigDrive): - dsmode = "net" - - -class nonConfigDriveDir(Exception): - pass - - -def cfg_drive_device(): - """ get the config drive device. return a string like '/dev/vdb' - or None (if there is no non-root device attached). This does not - check the contents, only reports that if there *were* a config_drive - attached, it would be this device. - per config_drive documentation, this is - "associated as the last available disk on the instance" - """ - - if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: - return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) - - # we are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it. - - letters = "abcdefghijklmnopqrstuvwxyz" - devs = util.find_devs_with("TYPE=vfat") - - # filter out anything not ending in a letter (ignore partitions) - devs = [f for f in devs if f[-1] in letters] - - # sort them in reverse so "last" device is first - devs.sort(reverse=True) - - if len(devs): - return(devs[0]) - - return(None) - - -def read_config_drive_dir(source_dir): - """ - read_config_drive_dir(source_dir): - read source_dir, and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a nonConfigDriveDir - """ - md = {} - ud = "" - - flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") - found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] - keydata = "" - - if len(found) == 0: - raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) - - if "etc/network/interfaces" in found: - with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: - md['network-interfaces'] = fp.read() - - if "root/.ssh/authorized_keys" in found: - with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: - keydata = fp.read() - - meta_js = {} - - if "meta.js" in found: - content = '' - with open("%s/%s" % (source_dir, "meta.js")) as fp: - content = fp.read() - md['meta_js'] = content - try: - meta_js = json.loads(content) - except ValueError: - raise nonConfigDriveDir("%s: %s" % - (source_dir, "invalid json in meta.js")) - - keydata = meta_js.get('public-keys', keydata) - - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - for copy in ('dsmode', 'instance-id', 'dscfg'): - if copy in meta_js: - md[copy] = meta_js[copy] - - if 'user-data' in meta_js: - ud = meta_js['user-data'] - - return(md, ud) - -datasources = ( - (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - -if __name__ == "__main__": - def main(): - import sys - import pprint - print cfg_drive_device() - (md, ud) = read_config_drive_dir(sys.argv[1]) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - print(ud) - - main() - -# vi: ts=4 expandtab diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py deleted file mode 100644 index 7051ecda..00000000 --- a/cloudinit/DataSourceEc2.py +++ /dev/null @@ -1,217 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import socket -import time -import boto.utils as boto_utils -import os.path - - -class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = base_seeddir + '/ec2' - metadata_address = "http://169.254.169.254" - - def __str__(self): - return("DataSourceEc2") - - def get_data(self): - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - self.userdata_raw = seedret['user-data'] - self.metadata = seedret['meta-data'] - log.debug("using seeded ec2 data in %s" % self.seeddir) - return True - - try: - if not self.wait_for_metadata_service(): - return False - start = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time() - - start)) - return True - except Exception as e: - print e - return False - - def get_instance_id(self): - return(self.metadata['instance-id']) - - def get_availability_zone(self): - return(self.metadata['placement']['availability-zone']) - - def get_local_mirror(self): - return(self.get_mirror_from_availability_zone()) - - def get_mirror_from_availability_zone(self, availability_zone=None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: - availability_zone = self.get_availability_zone() - - fallback = None - - if self.is_vpc(): - return fallback - - try: - host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return fallback - - def wait_for_metadata_service(self): - mcfg = self.ds_cfg - - if not hasattr(mcfg, "get"): - mcfg = {} - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] - mdurls = mcfg.get("metadata_urls", def_mdurls) - - # Remove addresses from the list that wont resolve. - filtered = [x for x in mdurls if util.is_resolvable_url(x)] - - if set(filtered) != set(mdurls): - log.debug("removed the following from metadata urls: %s" % - list((set(mdurls) - set(filtered)))) - - if len(filtered): - mdurls = filtered - else: - log.warn("Empty metadata url list! using default list") - mdurls = def_mdurls - - urls = [] - url2base = {False: False} - for url in mdurls: - cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) - urls.append(cur) - url2base[cur] = url - - starttime = time.time() - url = util.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) - - if url: - log.debug("Using metadata source: '%s'" % url2base[url]) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - self.metadata_address = url2base[url] - return (bool(url)) - - def device_name_to_device(self, name): - # consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if 'block-device-mapping' not in self.metadata: - return(None) - - found = None - for entname, device in self.metadata['block-device-mapping'].items(): - if entname == name: - found = device - break - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - found = device - if found == None: - log.debug("unable to convert %s to a device" % name) - return None - - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} - ofound = found - short = os.path.basename(found) - - if not found.startswith("/"): - found = "/dev/%s" % found - - if os.path.exists(found): - return(found) - - for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short[len(nfrom):]) - if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found, cand)) - return(cand) - - # on t1.micro, ephemeral0 will appear in block-device-mapping from - # metadata, but it will not exist on disk (and never will) - # at this pint, we've verified that the path did not exist - # in the special case of 'ephemeral0' return None to avoid bogus - # fstab entry (LP: #744019) - if name == "ephemeral0": - return None - return ofound - - def is_vpc(self): - # per comment in LP: #615545 - ph = "public-hostname" - p4 = "public-ipv4" - if ((ph not in self.metadata or self.metadata[ph] == "") and - (p4 not in self.metadata or self.metadata[p4] == "")): - return True - return False - - -datasources = [ - (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/DataSourceMAAS.py b/cloudinit/DataSourceMAAS.py deleted file mode 100644 index 61a0038f..00000000 --- a/cloudinit/DataSourceMAAS.py +++ /dev/null @@ -1,345 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import oauth.oauth as oauth -import os.path -import urllib2 -import time - - -MD_VERSION = "2012-03-01" - - -class DataSourceMAAS(DataSource.DataSource): - """ - DataSourceMAAS reads instance information from MAAS. - Given a config metadata_url, and oauth tokens, it expects to find - files under the root named: - instance-id - user-data - hostname - """ - seeddir = base_seeddir + '/maas' - baseurl = None - - def __str__(self): - return("DataSourceMAAS[%s]" % self.baseurl) - - def get_data(self): - mcfg = self.ds_cfg - - try: - (userdata, metadata) = read_maas_seed_dir(self.seeddir) - self.userdata_raw = userdata - self.metadata = metadata - self.baseurl = self.seeddir - return True - except MAASSeedDirNone: - pass - except MAASSeedDirMalformed as exc: - log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) - raise - - try: - # if there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if url == None: - return False - - if not self.wait_for_metadata_service(url): - return False - - self.baseurl = url - - (userdata, metadata) = read_maas_seed_url(self.baseurl, - self.md_headers) - self.userdata_raw = userdata - self.metadata = metadata - return True - except Exception: - util.logexc(log) - return False - - def md_headers(self, url): - mcfg = self.ds_cfg - - # if we are missing token_key, token_secret or consumer_key - # then just do non-authed requests - for required in ('token_key', 'token_secret', 'consumer_key'): - if required not in mcfg: - return({}) - - consumer_secret = mcfg.get('consumer_secret', "") - - return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret)) - - def wait_for_metadata_service(self, url): - mcfg = self.ds_cfg - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - starttime = time.time() - check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=log.warn, - headers_cb=self.md_headers) - - if url: - log.debug("Using metadata source: '%s'" % url) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - return (bool(url)) - - -def read_maas_seed_dir(seed_d): - """ - Return user-data and metadata for a maas seed dir in seed_d. - Expected format of seed_d are the following files: - * instance-id - * local-hostname - * user-data - """ - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - - if not os.path.isdir(seed_d): - raise MAASSeedDirNone("%s: not a directory") - - for fname in files: - try: - with open(os.path.join(seed_d, fname)) as fp: - md[fname] = fp.read() - fp.close() - except IOError as e: - if e.errno != errno.ENOENT: - raise - - return(check_seed_contents(md, seed_d)) - - -def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): - """ - Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() - - Expected format of seed_url is are the following files: - * //meta-data/instance-id - * //meta-data/local-hostname - * //user-data - """ - files = ('meta-data/local-hostname', - 'meta-data/instance-id', - 'meta-data/public-keys', - 'user-data') - - base_url = "%s/%s" % (seed_url, version) - md = {} - for fname in files: - url = "%s/%s" % (base_url, fname) - if header_cb: - headers = header_cb(url) - else: - headers = {} - - try: - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - md[os.path.basename(fname)] = resp.read() - except urllib2.HTTPError as e: - if e.code != 404: - raise - - return(check_seed_contents(md, seed_url)) - - -def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a - return for a datasource. - Either return a (userdata, metadata) tuple or - Raise MAASSeedDirMalformed or MAASSeedDirNone - """ - md_required = ('instance-id', 'local-hostname') - found = content.keys() - - if len(content) == 0: - raise MAASSeedDirNone("%s: no data files found" % seed) - - missing = [k for k in md_required if k not in found] - if len(missing): - raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - - userdata = content.get('user-data', "") - md = {} - for (key, val) in content.iteritems(): - if key == 'user-data': - continue - md[key] = val - - return(userdata, md) - - -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): - consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) - token = oauth.OAuthToken(token_key, token_secret) - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': token.key, - 'oauth_consumer_key': consumer.key, - } - req = oauth.OAuthRequest(http_url=url, parameters=params) - req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return(req.to_header()) - - -class MAASSeedDirNone(Exception): - pass - - -class MAASSeedDirMalformed(Exception): - pass - - -datasources = [ - (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.load(fp) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] == None: - creds[key] = cfg[key] - - def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) - - def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) - - def crawl(url, headers_cb=None): - if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) - else: - printurl("%s%s" % (url, line), headers_cb) - else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) != None: - headers = oauth_headers(url, **creds) - return headers - - if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url, my_headers) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url, my_headers) - - main() diff --git a/cloudinit/DataSourceNoCloud.py b/cloudinit/DataSourceNoCloud.py deleted file mode 100644 index e8c56b8f..00000000 --- a/cloudinit/DataSourceNoCloud.py +++ /dev/null @@ -1,232 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import subprocess - - -class DataSourceNoCloud(DataSource.DataSource): - metadata = None - userdata = None - userdata_raw = None - supported_seed_starts = ("/", "file://") - dsmode = "local" - seed = None - cmdline_id = "ds=nocloud" - seeddir = base_seeddir + '/nocloud' - - def __str__(self): - mstr = "DataSourceNoCloud" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - defaults = { - "instance-id": "nocloud", "dsmode": self.dsmode - } - - found = [] - md = {} - ud = "" - - try: - # parse the kernel command line, getting data passed in - if parse_cmdline_data(self.cmdline_id, md): - found.append("cmdline") - except: - util.logexc(log) - return False - - # check to see if the seeddir has data. - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - md = util.mergedict(md, seedret['meta-data']) - ud = seedret['user-data'] - found.append(self.seeddir) - log.debug("using seeded cache data in %s" % self.seeddir) - - # if the datasource config had a 'seedfrom' entry, then that takes - # precedence over a 'seedfrom' that was found in a filesystem - # but not over external medi - if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: - found.append("ds_config") - md["seedfrom"] = self.ds_cfg['seedfrom'] - - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - (newmd, newud) = util.mount_callback_umount(dev, - util.read_seeded) - md = util.mergedict(newmd, md) - ud = newud - - # for seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - log.debug("using data from %s" % dev) - found.append(dev) - break - except OSError, e: - if e.errno != errno.ENOENT: - raise - except util.mountFailedError: - log.warn("Failed to mount %s when looking for seed" % dev) - - # there was no indication on kernel cmdline or data - # in the seeddir suggesting this handler should be used. - if len(found) == 0: - return False - - seeded_interfaces = None - - # the special argument "seedfrom" indicates we should - # attempt to seed the userdata / metadata from its value - # its primarily value is in allowing the user to type less - # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in md: - seedfrom = md["seedfrom"] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - if 'network-interfaces' in md: - seeded_interfaces = self.dsmode - - # this could throw errors, but the user told us to do it - # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - # values in the command line override those from the seed - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - - # update the network-interfaces if metadata had 'network-interfaces' - # entry and this is the local datasource, or 'seedfrom' was used - # and the source of the seed was self.dsmode - # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in md and - (self.dsmode in ("local", seeded_interfaces))): - log.info("updating network interfaces from nocloud") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - -# returns true or false indicating if cmdline indicated -# that this module should be used -# example cmdline: -# root=LABEL=uec-rootfs ro ds=nocloud -def parse_cmdline_data(ds_id, fill, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - cmdline = " %s " % cmdline - - if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): - return False - - argline = "" - # cmdline can contain: - # ds=nocloud[;key=val;key=val] - for tok in cmdline.split(): - if tok.startswith(ds_id): - argline = tok.split("=", 1) - - # argline array is now 'nocloud' followed optionally by - # a ';' and then key=value pairs also terminated with ';' - tmp = argline[1].split(";") - if len(tmp) > 1: - kvpairs = tmp[1:] - else: - kvpairs = () - - # short2long mapping to save cmdline typing - s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} - for item in kvpairs: - try: - (k, v) = item.split("=", 1) - except: - k = item - v = None - if k in s2l: - k = s2l[k] - fill[k] = v - - return(True) - - -class DataSourceNoCloudNet(DataSourceNoCloud): - cmdline_id = "ds=nocloud-net" - supported_seed_starts = ("http://", "https://", "ftp://") - seeddir = base_seeddir + '/nocloud-net' - dsmode = "net" - - -datasources = ( - (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/DataSourceOVF.py b/cloudinit/DataSourceOVF.py deleted file mode 100644 index a0b1b518..00000000 --- a/cloudinit/DataSourceOVF.py +++ /dev/null @@ -1,332 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -from xml.dom import minidom -import base64 -import re -import tempfile -import subprocess - - -class DataSourceOVF(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/ovf' - environment = None - cfg = {} - userdata_raw = None - metadata = None - supported_seed_starts = ("/", "file://") - - def __str__(self): - mstr = "DataSourceOVF" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = [] - md = {} - ud = "" - - defaults = { - "instance-id": "iid-dsovf" - } - - (seedfile, contents) = get_ovf_env(base_seeddir) - if seedfile: - # found a seed dir - seed = "%s/%s" % (base_seeddir, seedfile) - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - - found.append(seed) - else: - np = {'iso': transport_iso9660, - 'vmware-guestd': transport_vmware_guestd, } - name = None - for name, transfunc in np.iteritems(): - (contents, _dev, _fname) = transfunc() - if contents: - break - - if contents: - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - found.append(name) - - # There was no OVF transports found - if len(found) == 0: - return False - - if 'seedfrom' in md and md['seedfrom']: - seedfrom = md['seedfrom'] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - self.cfg = cfg - return True - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return([self.metadata['public-keys'], ]) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceOVFNet(DataSourceOVF): - seeddir = base_seeddir + '/ovf-net' - supported_seed_starts = ("http://", "https://", "ftp://") - - -# this will return a dict with some content -# meta-data, user-data -def read_ovf_environment(contents): - props = getProperties(contents) - md = {} - cfg = {} - ud = "" - cfg_props = ['password', ] - md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for prop, val in props.iteritems(): - if prop == 'hostname': - prop = "local-hostname" - if prop in md_props: - md[prop] = val - elif prop in cfg_props: - cfg[prop] = val - elif prop == "user-data": - try: - ud = base64.decodestring(val) - except: - ud = val - return(md, ud, cfg) - - -# returns tuple of filename (in 'dirname', and the contents of the file) -# on "not found", returns 'None' for filename and False for contents -def get_ovf_env(dirname): - env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") - for fname in env_names: - if os.path.isfile("%s/%s" % (dirname, fname)): - fp = open("%s/%s" % (dirname, fname)) - contents = fp.read() - fp.close() - return(fname, contents) - return(None, False) - - -# transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): - - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" - - devname_regex = os.environ.get(envname, default_regex) - cdmatch = re.compile(devname_regex) - - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mounted[dev] = (dev, fstype, mp, False) - mp = mp.replace("\\040", " ") - if fstype != "iso9660" and require_iso: - continue - - if cdmatch.match(dev[5:]) == None: # take off '/dev/' - continue - - (fname, contents) = get_ovf_env(mp) - if contents is not False: - return(contents, dev, fname) - - tmpd = None - dvnull = None - - devs = os.listdir("/dev/") - devs.sort() - - for dev in devs: - fullp = "/dev/%s" % dev - - if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): - continue - - fp = None - try: - fp = open(fullp, "rb") - fp.read(512) - fp.close() - except: - if fp: - fp.close() - continue - - if tmpd is None: - tmpd = tempfile.mkdtemp() - if dvnull is None: - try: - dvnull = open("/dev/null") - except: - pass - - cmd = ["mount", "-o", "ro", fullp, tmpd] - if require_iso: - cmd.extend(('-t', 'iso9660')) - - rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) - if rc: - continue - - (fname, contents) = get_ovf_env(tmpd) - - subprocess.call(["umount", tmpd]) - - if contents is not False: - os.rmdir(tmpd) - return(contents, fullp, fname) - - if tmpd: - os.rmdir(tmpd) - - if dvnull: - dvnull.close() - - return(False, None, None) - - -def transport_vmware_guestd(): - # http://blogs.vmware.com/vapp/2009/07/ \ - # selfconfiguration-and-the-ovf-environment.html - # try: - # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] - # (out, err) = subp(cmd) - # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') - # except: - # # would need to error check here and see why this failed - # # to know if log/error should be raised - # return(False, None, None) - return(False, None, None) - - -def findChild(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return(ret) - - -def getProperties(environString): - dom = minidom.parseString(environString) - if dom.documentElement.localName != "Environment": - raise Exception("No Environment Node") - - if not dom.documentElement.hasChildNodes(): - raise Exception("No Child Nodes") - - envNsURI = "http://schemas.dmtf.org/ovf/environment/1" - - # could also check here that elem.namespaceURI == - # "http://schemas.dmtf.org/ovf/environment/1" - propSections = findChild(dom.documentElement, - lambda n: n.localName == "PropertySection") - - if len(propSections) == 0: - raise Exception("No 'PropertySection's") - - props = {} - propElems = findChild(propSections[0], lambda n: n.localName == "Property") - - for elem in propElems: - key = elem.attributes.getNamedItemNS(envNsURI, "key").value - val = elem.attributes.getNamedItemNS(envNsURI, "value").value - props[key] = val - - return(props) - - -datasources = ( - (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), - (DataSourceOVFNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) - - main() diff --git a/cloudinit/SshUtil.py b/cloudinit/SshUtil.py deleted file mode 100644 index a081fbe8..00000000 --- a/cloudinit/SshUtil.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import cloudinit.util as util - - -class AuthKeyEntry(): - # lines are options, keytype, base64-encoded key, comment - # man page says the following which I did not understand: - # The options field is optional; its presence is determined by whether - # the line starts with a number or not (the options field never starts - # with a number) - options = None - keytype = None - base64 = None - comment = None - is_comment = False - line_in = "" - - def __init__(self, line, def_opt=None): - line = line.rstrip("\n\r") - self.line_in = line - if line.startswith("#") or line.strip() == "": - self.is_comment = True - else: - ent = line.strip() - toks = ent.split(None, 3) - if len(toks) == 1: - self.base64 = toks[0] - elif len(toks) == 2: - (self.base64, self.comment) = toks - elif len(toks) == 3: - (self.keytype, self.base64, self.comment) = toks - elif len(toks) == 4: - i = 0 - ent = line.strip() - quoted = False - # taken from auth_rsa_key_allowed in auth-rsa.c - try: - while (i < len(ent) and - ((quoted) or (ent[i] not in (" ", "\t")))): - curc = ent[i] - nextc = ent[i + 1] - if curc == "\\" and nextc == '"': - i = i + 1 - elif curc == '"': - quoted = not quoted - i = i + 1 - except IndexError: - self.is_comment = True - return - - try: - self.options = ent[0:i] - (self.keytype, self.base64, self.comment) = \ - ent[i + 1:].split(None, 3) - except ValueError: - # we did not understand this line - self.is_comment = True - - if self.options == None and def_opt: - self.options = def_opt - - return - - def debug(self): - print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\n" - "comment=%s\n" % (self.line_in, self.is_comment, self.options, - self.keytype, self.base64, self.comment)), - - def __repr__(self): - if self.is_comment: - return(self.line_in) - else: - toks = [] - for e in (self.options, self.keytype, self.base64, self.comment): - if e: - toks.append(e) - - return(' '.join(toks)) - - -def update_authorized_keys(fname, keys): - # keys is a list of AuthKeyEntries - # key_prefix is the prefix (options) to prepend - try: - fp = open(fname, "r") - lines = fp.readlines() # lines have carriage return - fp.close() - except IOError: - lines = [] - - ka_stats = {} # keys_added status - for k in keys: - ka_stats[k] = False - - to_add = [] - for key in keys: - to_add.append(key) - - for i in range(0, len(lines)): - ent = AuthKeyEntry(lines[i]) - for k in keys: - if k.base64 == ent.base64 and not k.is_comment: - ent = k - try: - to_add.remove(k) - except ValueError: - pass - lines[i] = str(ent) - - # now append any entries we did not match above - for key in to_add: - lines.append(str(key)) - - if len(lines) == 0: - return("") - else: - return('\n'.join(lines) + "\n") - - -def setup_user_keys(keys, user, key_prefix, log=None): - import pwd - saved_umask = os.umask(077) - - pwent = pwd.getpwnam(user) - - ssh_dir = '%s/.ssh' % pwent.pw_dir - if not os.path.exists(ssh_dir): - os.mkdir(ssh_dir) - os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) - - try: - ssh_cfg = parse_ssh_config() - akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") - akeys = akeys.replace("%h", pwent.pw_dir) - akeys = akeys.replace("%u", user) - if not akeys.startswith('/'): - akeys = os.path.join(pwent.pw_dir, akeys) - authorized_keys = akeys - except Exception: - authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir - if log: - util.logexc(log) - - key_entries = [] - for k in keys: - ke = AuthKeyEntry(k, def_opt=key_prefix) - key_entries.append(ke) - - content = update_authorized_keys(authorized_keys, key_entries) - util.write_file(authorized_keys, content, 0600) - - os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) - util.restorecon_if_possible(ssh_dir, recursive=True) - - os.umask(saved_umask) - - -def parse_ssh_config(fname="/etc/ssh/sshd_config"): - ret = {} - fp = open(fname) - for l in fp.readlines(): - l = l.strip() - if not l or l.startswith("#"): - continue - key, val = l.split(None, 1) - ret[key] = val - fp.close() - return(ret) - -if __name__ == "__main__": - def main(): - import sys - # usage: orig_file, new_keys, [key_prefix] - # prints out merged, where 'new_keys' will trump old - ## example - ## ### begin auth_keys ### - # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= smoser-work - # ssh-rsa AAAAB3NzaC1xxxxxxxxxCmXp5Kt5/82cD/VN3NtHw== smoser@brickies - # ### end authorized_keys ### - # - # ### begin new_keys ### - # ssh-rsa nonmatch smoser@newhost - # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= new_comment - # ### end new_keys ### - # - # Then run as: - # program auth_keys new_keys \ - # 'no-port-forwarding,command=\"echo hi world;\"' - def_prefix = None - orig_key_file = sys.argv[1] - new_key_file = sys.argv[2] - if len(sys.argv) > 3: - def_prefix = sys.argv[3] - fp = open(new_key_file) - - newkeys = [] - for line in fp.readlines(): - newkeys.append(AuthKeyEntry(line, def_prefix)) - - fp.close() - print update_authorized_keys(orig_key_file, newkeys) - - main() - -# vi: ts=4 expandtab diff --git a/cloudinit/UserDataHandler.py b/cloudinit/UserDataHandler.py deleted file mode 100644 index ec914480..00000000 --- a/cloudinit/UserDataHandler.py +++ /dev/null @@ -1,262 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import email - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.base import MIMEBase -import yaml -import cloudinit -import cloudinit.util as util -import hashlib -import urllib - - -starts_with_mappings = { - '#include': 'text/x-include-url', - '#include-once': 'text/x-include-once-url', - '#!': 'text/x-shellscript', - '#cloud-config': 'text/cloud-config', - '#upstart-job': 'text/upstart-job', - '#part-handler': 'text/part-handler', - '#cloud-boothook': 'text/cloud-boothook', - '#cloud-config-archive': 'text/cloud-config-archive', -} - - -# if 'string' is compressed return decompressed otherwise return it -def decomp_str(string): - import StringIO - import gzip - try: - uncomp = gzip.GzipFile(None, "rb", 1, StringIO.StringIO(string)).read() - return(uncomp) - except: - return(string) - - -def do_include(content, appendmsg): - import os - # is just a list of urls, one per line - # also support '#include ' - includeonce = False - for line in content.splitlines(): - if line == "#include": - continue - if line == "#include-once": - includeonce = True - continue - if line.startswith("#include-once"): - line = line[len("#include-once"):].lstrip() - includeonce = True - elif line.startswith("#include"): - line = line[len("#include"):].lstrip() - if line.startswith("#"): - continue - if line.strip() == "": - continue - - # urls cannot not have leading or trailing white space - msum = hashlib.md5() # pylint: disable=E1101 - msum.update(line.strip()) - includeonce_filename = "%s/urlcache/%s" % ( - cloudinit.get_ipath_cur("data"), msum.hexdigest()) - try: - if includeonce and os.path.isfile(includeonce_filename): - with open(includeonce_filename, "r") as fp: - content = fp.read() - else: - content = urllib.urlopen(line).read() - if includeonce: - util.write_file(includeonce_filename, content, mode=0600) - except Exception: - raise - - process_includes(message_from_string(decomp_str(content)), appendmsg) - - -def explode_cc_archive(archive, appendmsg): - for ent in yaml.load(archive): - # ent can be one of: - # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } - # filename and type not be present - # or - # scalar(payload) - - def_type = "text/cloud-config" - if isinstance(ent, str): - ent = {'content': ent} - - content = ent.get('content', '') - mtype = ent.get('type', None) - if mtype == None: - mtype = type_from_startswith(content, def_type) - - maintype, subtype = mtype.split('/', 1) - if maintype == "text": - msg = MIMEText(content, _subtype=subtype) - else: - msg = MIMEBase(maintype, subtype) - msg.set_payload(content) - - if 'filename' in ent: - msg.add_header('Content-Disposition', 'attachment', - filename=ent['filename']) - - for header in ent.keys(): - if header in ('content', 'filename', 'type'): - continue - msg.add_header(header, ent['header']) - - _attach_part(appendmsg, msg) - - -def multi_part_count(outermsg, newcount=None): - """ - Return the number of attachments to this MIMEMultipart by looking - at its 'Number-Attachments' header. - """ - nfield = 'Number-Attachments' - if nfield not in outermsg: - outermsg[nfield] = "0" - - if newcount != None: - outermsg.replace_header(nfield, str(newcount)) - - return(int(outermsg.get('Number-Attachments', 0))) - - -def _attach_part(outermsg, part): - """ - Attach an part to an outer message. outermsg must be a MIMEMultipart. - Modifies a header in outermsg to keep track of number of attachments. - """ - cur = multi_part_count(outermsg) - if not part.get_filename(None): - part.add_header('Content-Disposition', 'attachment', - filename='part-%03d' % (cur + 1)) - outermsg.attach(part) - multi_part_count(outermsg, cur + 1) - - -def type_from_startswith(payload, default=None): - # slist is sorted longest first - slist = sorted(starts_with_mappings.keys(), key=lambda e: 0 - len(e)) - for sstr in slist: - if payload.startswith(sstr): - return(starts_with_mappings[sstr]) - return default - - -def process_includes(msg, appendmsg=None): - if appendmsg == None: - appendmsg = MIMEMultipart() - - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = None - ctype_orig = part.get_content_type() - - payload = part.get_payload(decode=True) - - if ctype_orig in ("text/plain", "text/x-not-multipart"): - ctype = type_from_startswith(payload) - - if ctype is None: - ctype = ctype_orig - - if ctype in ('text/x-include-url', 'text/x-include-once-url'): - do_include(payload, appendmsg) - continue - - if ctype == "text/cloud-config-archive": - explode_cc_archive(payload, appendmsg) - continue - - if 'Content-Type' in msg: - msg.replace_header('Content-Type', ctype) - else: - msg['Content-Type'] = ctype - - _attach_part(appendmsg, part) - - -def message_from_string(data, headers=None): - if headers is None: - headers = {} - if "mime-version:" in data[0:4096].lower(): - msg = email.message_from_string(data) - for (key, val) in headers.items(): - if key in msg: - msg.replace_header(key, val) - else: - msg[key] = val - else: - mtype = headers.get("Content-Type", "text/x-not-multipart") - maintype, subtype = mtype.split("/", 1) - msg = MIMEBase(maintype, subtype, *headers) - msg.set_payload(data) - - return(msg) - - -# this is heavily wasteful, reads through userdata string input -def preprocess_userdata(data): - newmsg = MIMEMultipart() - process_includes(message_from_string(decomp_str(data)), newmsg) - return(newmsg.as_string()) - - -# callback is a function that will be called with (data, content_type, -# filename, payload) -def walk_userdata(istr, callback, data=None): - partnum = 0 - for part in message_from_string(istr).walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - - ctype = part.get_content_type() - if ctype is None: - ctype = 'application/octet-stream' - - filename = part.get_filename() - if not filename: - filename = 'part-%03d' % partnum - - callback(data, ctype, filename, part.get_payload(decode=True)) - - partnum = partnum + 1 - - -if __name__ == "__main__": - def main(): - import sys - data = decomp_str(file(sys.argv[1]).read()) - newmsg = MIMEMultipart() - process_includes(message_from_string(data), newmsg) - print newmsg - print "#found %s parts" % multi_part_count(newmsg) - - main() diff --git a/cloudinit/constants.py b/cloudinit/constants.py new file mode 100644 index 00000000..7bc90f27 --- /dev/null +++ b/cloudinit/constants.py @@ -0,0 +1,37 @@ +import os + +VAR_LIB_DIR = '/var/lib/cloud' +CUR_INSTANCE_LINK = os.path.join(VAR_LIB_DIR, "instance") +BOOT_FINISHED = os.path.join(CUR_INSTANCE_LINK, "boot-finished") +SEED_DIR = os.path.join(VAR_LIB_DIR, "seed") + +CFG_ENV_NAME = "CLOUD_CFG" +CLOUD_CONFIG = '/etc/cloud/cloud.cfg' + +CFG_BUILTIN = { + 'datasource_list': ['NoCloud', + 'ConfigDrive', + 'OVF', + 'MAAS', + 'Ec2', + 'CloudStack'], + 'def_log_file': '/var/log/cloud-init.log', + 'log_cfgs': [], + 'syslog_fix_perms': 'syslog:adm' +} + +PATH_MAP = { + "handlers": "handlers", + "scripts": "scripts", + "sem": "sem", + "boothooks": "boothooks", + "userdata_raw": "user-data.txt", + "userdata": "user-data.txt.i", + "obj_pkl": "obj.pkl", + "cloud_config": "cloud-config.txt", + "data": "data", +} + +PER_INSTANCE = "once-per-instance" +PER_ALWAYS = "always" +PER_ONCE = "once" diff --git a/cloudinit/handlers/DataSource.py b/cloudinit/handlers/DataSource.py new file mode 100644 index 00000000..e2a9150d --- /dev/null +++ b/cloudinit/handlers/DataSource.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) diff --git a/cloudinit/handlers/DataSourceCloudStack.py b/cloudinit/handlers/DataSourceCloudStack.py new file mode 100644 index 00000000..5afdf7b6 --- /dev/null +++ b/cloudinit/handlers/DataSourceCloudStack.py @@ -0,0 +1,92 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Cosmin Luta +# +# Author: Cosmin Luta +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +from socket import inet_ntoa +import time +import boto.utils as boto_utils +from struct import pack + + +class DataSourceCloudStack(DataSource.DataSource): + api_ver = 'latest' + seeddir = base_seeddir + '/cs' + metadata_address = None + + def __init__(self, sys_cfg=None): + DataSource.DataSource.__init__(self, sys_cfg) + # Cloudstack has its metadata/userdata URLs located at + # http:///latest/ + self.metadata_address = "http://%s/" % self.get_default_gateway() + + def get_default_gateway(self): + """ Returns the default gateway ip address in the dotted format + """ + with open("/proc/net/route", "r") as f: + for line in f.readlines(): + items = line.split("\t") + if items[1] == "00000000": + # found the default route, get the gateway + gw = inet_ntoa(pack(" +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +import json +import subprocess + +DEFAULT_IID = "iid-dsconfigdrive" + + +class DataSourceConfigDrive(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/config_drive' + cfg = {} + userdata_raw = None + metadata = None + dsmode = "local" + + def __str__(self): + mstr = "DataSourceConfigDrive[%s]" % self.dsmode + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = None + md = {} + ud = "" + + defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} + + if os.path.isdir(self.seeddir): + try: + (md, ud) = read_config_drive_dir(self.seeddir) + found = self.seeddir + except nonConfigDriveDir: + pass + + if not found: + dev = cfg_drive_device() + if dev: + try: + (md, ud) = util.mount_callback_umount(dev, + read_config_drive_dir) + found = dev + except (nonConfigDriveDir, util.mountFailedError): + pass + + if not found: + return False + + if 'dsconfig' in md: + self.cfg = md['dscfg'] + + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. + if 'network-interfaces' in md and self.dsmode == "local": + if md['dsmode'] == "pass": + log.info("updating network interfaces from configdrive") + else: + log.debug("updating network interfaces from configdrive") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = found + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return(self.metadata['public-keys']) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceConfigDriveNet(DataSourceConfigDrive): + dsmode = "net" + + +class nonConfigDriveDir(Exception): + pass + + +def cfg_drive_device(): + """ get the config drive device. return a string like '/dev/vdb' + or None (if there is no non-root device attached). This does not + check the contents, only reports that if there *were* a config_drive + attached, it would be this device. + per config_drive documentation, this is + "associated as the last available disk on the instance" + """ + + if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: + return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) + + # we are looking for a raw block device (sda, not sda1) with a vfat + # filesystem on it. + + letters = "abcdefghijklmnopqrstuvwxyz" + devs = util.find_devs_with("TYPE=vfat") + + # filter out anything not ending in a letter (ignore partitions) + devs = [f for f in devs if f[-1] in letters] + + # sort them in reverse so "last" device is first + devs.sort(reverse=True) + + if len(devs): + return(devs[0]) + + return(None) + + +def read_config_drive_dir(source_dir): + """ + read_config_drive_dir(source_dir): + read source_dir, and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a nonConfigDriveDir + """ + md = {} + ud = "" + + flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") + found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] + keydata = "" + + if len(found) == 0: + raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + + if "etc/network/interfaces" in found: + with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: + md['network-interfaces'] = fp.read() + + if "root/.ssh/authorized_keys" in found: + with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: + keydata = fp.read() + + meta_js = {} + + if "meta.js" in found: + content = '' + with open("%s/%s" % (source_dir, "meta.js")) as fp: + content = fp.read() + md['meta_js'] = content + try: + meta_js = json.loads(content) + except ValueError: + raise nonConfigDriveDir("%s: %s" % + (source_dir, "invalid json in meta.js")) + + keydata = meta_js.get('public-keys', keydata) + + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + for copy in ('dsmode', 'instance-id', 'dscfg'): + if copy in meta_js: + md[copy] = meta_js[copy] + + if 'user-data' in meta_js: + ud = meta_js['user-data'] + + return(md, ud) + +datasources = ( + (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + +if __name__ == "__main__": + def main(): + import sys + import pprint + print cfg_drive_device() + (md, ud) = read_config_drive_dir(sys.argv[1]) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + print(ud) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/handlers/DataSourceEc2.py b/cloudinit/handlers/DataSourceEc2.py new file mode 100644 index 00000000..7051ecda --- /dev/null +++ b/cloudinit/handlers/DataSourceEc2.py @@ -0,0 +1,217 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import socket +import time +import boto.utils as boto_utils +import os.path + + +class DataSourceEc2(DataSource.DataSource): + api_ver = '2009-04-04' + seeddir = base_seeddir + '/ec2' + metadata_address = "http://169.254.169.254" + + def __str__(self): + return("DataSourceEc2") + + def get_data(self): + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + self.userdata_raw = seedret['user-data'] + self.metadata = seedret['meta-data'] + log.debug("using seeded ec2 data in %s" % self.seeddir) + return True + + try: + if not self.wait_for_metadata_service(): + return False + start = time.time() + self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, + None, self.metadata_address) + self.metadata = boto_utils.get_instance_metadata(self.api_ver, + self.metadata_address) + log.debug("crawl of metadata service took %ds" % (time.time() - + start)) + return True + except Exception as e: + print e + return False + + def get_instance_id(self): + return(self.metadata['instance-id']) + + def get_availability_zone(self): + return(self.metadata['placement']['availability-zone']) + + def get_local_mirror(self): + return(self.get_mirror_from_availability_zone()) + + def get_mirror_from_availability_zone(self, availability_zone=None): + # availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone == None: + availability_zone = self.get_availability_zone() + + fallback = None + + if self.is_vpc(): + return fallback + + try: + host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) + return 'http://%s/ubuntu/' % host + except: + return fallback + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + + if not hasattr(mcfg, "get"): + mcfg = {} + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] + mdurls = mcfg.get("metadata_urls", def_mdurls) + + # Remove addresses from the list that wont resolve. + filtered = [x for x in mdurls if util.is_resolvable_url(x)] + + if set(filtered) != set(mdurls): + log.debug("removed the following from metadata urls: %s" % + list((set(mdurls) - set(filtered)))) + + if len(filtered): + mdurls = filtered + else: + log.warn("Empty metadata url list! using default list") + mdurls = def_mdurls + + urls = [] + url2base = {False: False} + for url in mdurls: + cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) + urls.append(cur) + url2base[cur] = url + + starttime = time.time() + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) + + if url: + log.debug("Using metadata source: '%s'" % url2base[url]) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + self.metadata_address = url2base[url] + return (bool(url)) + + def device_name_to_device(self, name): + # consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if 'block-device-mapping' not in self.metadata: + return(None) + + found = None + for entname, device in self.metadata['block-device-mapping'].items(): + if entname == name: + found = device + break + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + found = device + if found == None: + log.debug("unable to convert %s to a device" % name) + return None + + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + ofound = found + short = os.path.basename(found) + + if not found.startswith("/"): + found = "/dev/%s" % found + + if os.path.exists(found): + return(found) + + for nfrom, tlist in mappings.items(): + if not short.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short[len(nfrom):]) + if os.path.exists(cand): + log.debug("remapped device name %s => %s" % (found, cand)) + return(cand) + + # on t1.micro, ephemeral0 will appear in block-device-mapping from + # metadata, but it will not exist on disk (and never will) + # at this pint, we've verified that the path did not exist + # in the special case of 'ephemeral0' return None to avoid bogus + # fstab entry (LP: #744019) + if name == "ephemeral0": + return None + return ofound + + def is_vpc(self): + # per comment in LP: #615545 + ph = "public-hostname" + p4 = "public-ipv4" + if ((ph not in self.metadata or self.metadata[ph] == "") and + (p4 not in self.metadata or self.metadata[p4] == "")): + return True + return False + + +datasources = [ + (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceMAAS.py b/cloudinit/handlers/DataSourceMAAS.py new file mode 100644 index 00000000..61a0038f --- /dev/null +++ b/cloudinit/handlers/DataSourceMAAS.py @@ -0,0 +1,345 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import urllib2 +import time + + +MD_VERSION = "2012-03-01" + + +class DataSourceMAAS(DataSource.DataSource): + """ + DataSourceMAAS reads instance information from MAAS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMAAS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MAASSeedDirNone: + pass + except MAASSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + self.userdata_raw = userdata + self.metadata = metadata + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * local-hostname + * user-data + """ + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} + + if not os.path.isdir(seed_d): + raise MAASSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + return(check_seed_contents(md, seed_d)) + + +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * //meta-data/instance-id + * //meta-data/local-hostname + * //user-data + """ + files = ('meta-data/local-hostname', + 'meta-data/instance-id', + 'meta-data/public-keys', + 'user-data') + + base_url = "%s/%s" % (seed_url, version) + md = {} + for fname in files: + url = "%s/%s" % (base_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + try: + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MAASSeedDirMalformed or MAASSeedDirNone + """ + md_required = ('instance-id', 'local-hostname') + found = content.keys() + + if len(content) == 0: + raise MAASSeedDirNone("%s: no data files found" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content.get('user-data', "") + md = {} + for (key, val) in content.iteritems(): + if key == 'user-data': + continue + md[key] = val + + return(userdata, md) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MAASSeedDirNone(Exception): + pass + + +class MAASSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ + import argparse + import pprint + + parser = argparse.ArgumentParser(description='Interact with MAAS DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + if 'datasource' in cfg: + cfg = cfg['datasource']['MAAS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) + + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) + + main() diff --git a/cloudinit/handlers/DataSourceNoCloud.py b/cloudinit/handlers/DataSourceNoCloud.py new file mode 100644 index 00000000..e8c56b8f --- /dev/null +++ b/cloudinit/handlers/DataSourceNoCloud.py @@ -0,0 +1,232 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import subprocess + + +class DataSourceNoCloud(DataSource.DataSource): + metadata = None + userdata = None + userdata_raw = None + supported_seed_starts = ("/", "file://") + dsmode = "local" + seed = None + cmdline_id = "ds=nocloud" + seeddir = base_seeddir + '/nocloud' + + def __str__(self): + mstr = "DataSourceNoCloud" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + defaults = { + "instance-id": "nocloud", "dsmode": self.dsmode + } + + found = [] + md = {} + ud = "" + + try: + # parse the kernel command line, getting data passed in + if parse_cmdline_data(self.cmdline_id, md): + found.append("cmdline") + except: + util.logexc(log) + return False + + # check to see if the seeddir has data. + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + md = util.mergedict(md, seedret['meta-data']) + ud = seedret['user-data'] + found.append(self.seeddir) + log.debug("using seeded cache data in %s" % self.seeddir) + + # if the datasource config had a 'seedfrom' entry, then that takes + # precedence over a 'seedfrom' that was found in a filesystem + # but not over external medi + if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: + found.append("ds_config") + md["seedfrom"] = self.ds_cfg['seedfrom'] + + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label_list = util.find_devs_with("LABEL=cidata") + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + (newmd, newud) = util.mount_callback_umount(dev, + util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # for seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. + # If they want dsmode of local, then they must + # specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + log.debug("using data from %s" % dev) + found.append(dev) + break + except OSError, e: + if e.errno != errno.ENOENT: + raise + except util.mountFailedError: + log.warn("Failed to mount %s when looking for seed" % dev) + + # there was no indication on kernel cmdline or data + # in the seeddir suggesting this handler should be used. + if len(found) == 0: + return False + + seeded_interfaces = None + + # the special argument "seedfrom" indicates we should + # attempt to seed the userdata / metadata from its value + # its primarily value is in allowing the user to type less + # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg + if "seedfrom" in md: + seedfrom = md["seedfrom"] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + if 'network-interfaces' in md: + seeded_interfaces = self.dsmode + + # this could throw errors, but the user told us to do it + # so if errors are raised, let them raise + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + # values in the command line override those from the seed + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + + # update the network-interfaces if metadata had 'network-interfaces' + # entry and this is the local datasource, or 'seedfrom' was used + # and the source of the seed was self.dsmode + # ('local' for NoCloud, 'net' for NoCloudNet') + if ('network-interfaces' in md and + (self.dsmode in ("local", seeded_interfaces))): + log.info("updating network interfaces from nocloud") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + +# returns true or false indicating if cmdline indicated +# that this module should be used +# example cmdline: +# root=LABEL=uec-rootfs ro ds=nocloud +def parse_cmdline_data(ds_id, fill, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + cmdline = " %s " % cmdline + + if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): + return False + + argline = "" + # cmdline can contain: + # ds=nocloud[;key=val;key=val] + for tok in cmdline.split(): + if tok.startswith(ds_id): + argline = tok.split("=", 1) + + # argline array is now 'nocloud' followed optionally by + # a ';' and then key=value pairs also terminated with ';' + tmp = argline[1].split(";") + if len(tmp) > 1: + kvpairs = tmp[1:] + else: + kvpairs = () + + # short2long mapping to save cmdline typing + s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} + for item in kvpairs: + try: + (k, v) = item.split("=", 1) + except: + k = item + v = None + if k in s2l: + k = s2l[k] + fill[k] = v + + return(True) + + +class DataSourceNoCloudNet(DataSourceNoCloud): + cmdline_id = "ds=nocloud-net" + supported_seed_starts = ("http://", "https://", "ftp://") + seeddir = base_seeddir + '/nocloud-net' + dsmode = "net" + + +datasources = ( + (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceOVF.py b/cloudinit/handlers/DataSourceOVF.py new file mode 100644 index 00000000..a0b1b518 --- /dev/null +++ b/cloudinit/handlers/DataSourceOVF.py @@ -0,0 +1,332 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +from xml.dom import minidom +import base64 +import re +import tempfile +import subprocess + + +class DataSourceOVF(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/ovf' + environment = None + cfg = {} + userdata_raw = None + metadata = None + supported_seed_starts = ("/", "file://") + + def __str__(self): + mstr = "DataSourceOVF" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = [] + md = {} + ud = "" + + defaults = { + "instance-id": "iid-dsovf" + } + + (seedfile, contents) = get_ovf_env(base_seeddir) + if seedfile: + # found a seed dir + seed = "%s/%s" % (base_seeddir, seedfile) + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + + found.append(seed) + else: + np = {'iso': transport_iso9660, + 'vmware-guestd': transport_vmware_guestd, } + name = None + for name, transfunc in np.iteritems(): + (contents, _dev, _fname) = transfunc() + if contents: + break + + if contents: + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + found.append(name) + + # There was no OVF transports found + if len(found) == 0: + return False + + if 'seedfrom' in md and md['seedfrom']: + seedfrom = md['seedfrom'] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + self.cfg = cfg + return True + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return([self.metadata['public-keys'], ]) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceOVFNet(DataSourceOVF): + seeddir = base_seeddir + '/ovf-net' + supported_seed_starts = ("http://", "https://", "ftp://") + + +# this will return a dict with some content +# meta-data, user-data +def read_ovf_environment(contents): + props = getProperties(contents) + md = {} + cfg = {} + ud = "" + cfg_props = ['password', ] + md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] + for prop, val in props.iteritems(): + if prop == 'hostname': + prop = "local-hostname" + if prop in md_props: + md[prop] = val + elif prop in cfg_props: + cfg[prop] = val + elif prop == "user-data": + try: + ud = base64.decodestring(val) + except: + ud = val + return(md, ud, cfg) + + +# returns tuple of filename (in 'dirname', and the contents of the file) +# on "not found", returns 'None' for filename and False for contents +def get_ovf_env(dirname): + env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") + for fname in env_names: + if os.path.isfile("%s/%s" % (dirname, fname)): + fp = open("%s/%s" % (dirname, fname)) + contents = fp.read() + fp.close() + return(fname, contents) + return(None, False) + + +# transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + + # default_regex matches values in + # /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + envname = "CLOUD_INIT_CDROM_DEV_REGEX" + default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + + devname_regex = os.environ.get(envname, default_regex) + cdmatch = re.compile(devname_regex) + + # go through mounts to see if it was already mounted + fp = open("/proc/mounts") + mounts = fp.readlines() + fp.close() + + mounted = {} + for mpline in mounts: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mounted[dev] = (dev, fstype, mp, False) + mp = mp.replace("\\040", " ") + if fstype != "iso9660" and require_iso: + continue + + if cdmatch.match(dev[5:]) == None: # take off '/dev/' + continue + + (fname, contents) = get_ovf_env(mp) + if contents is not False: + return(contents, dev, fname) + + tmpd = None + dvnull = None + + devs = os.listdir("/dev/") + devs.sort() + + for dev in devs: + fullp = "/dev/%s" % dev + + if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + continue + + fp = None + try: + fp = open(fullp, "rb") + fp.read(512) + fp.close() + except: + if fp: + fp.close() + continue + + if tmpd is None: + tmpd = tempfile.mkdtemp() + if dvnull is None: + try: + dvnull = open("/dev/null") + except: + pass + + cmd = ["mount", "-o", "ro", fullp, tmpd] + if require_iso: + cmd.extend(('-t', 'iso9660')) + + rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) + if rc: + continue + + (fname, contents) = get_ovf_env(tmpd) + + subprocess.call(["umount", tmpd]) + + if contents is not False: + os.rmdir(tmpd) + return(contents, fullp, fname) + + if tmpd: + os.rmdir(tmpd) + + if dvnull: + dvnull.close() + + return(False, None, None) + + +def transport_vmware_guestd(): + # http://blogs.vmware.com/vapp/2009/07/ \ + # selfconfiguration-and-the-ovf-environment.html + # try: + # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] + # (out, err) = subp(cmd) + # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') + # except: + # # would need to error check here and see why this failed + # # to know if log/error should be raised + # return(False, None, None) + return(False, None, None) + + +def findChild(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return(ret) + + +def getProperties(environString): + dom = minidom.parseString(environString) + if dom.documentElement.localName != "Environment": + raise Exception("No Environment Node") + + if not dom.documentElement.hasChildNodes(): + raise Exception("No Child Nodes") + + envNsURI = "http://schemas.dmtf.org/ovf/environment/1" + + # could also check here that elem.namespaceURI == + # "http://schemas.dmtf.org/ovf/environment/1" + propSections = findChild(dom.documentElement, + lambda n: n.localName == "PropertySection") + + if len(propSections) == 0: + raise Exception("No 'PropertySection's") + + props = {} + propElems = findChild(propSections[0], lambda n: n.localName == "Property") + + for elem in propElems: + key = elem.attributes.getNamedItemNS(envNsURI, "key").value + val = elem.attributes.getNamedItemNS(envNsURI, "value").value + props[key] = val + + return(props) + + +datasources = ( + (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), + (DataSourceOVFNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + envStr = open(sys.argv[1]).read() + props = getProperties(envStr) + import pprint + pprint.pprint(props) + + md, ud, cfg = read_ovf_environment(envStr) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + pprint.pprint(ud) + print "=== cfg ===" + pprint.pprint(cfg) + + main() diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py new file mode 100644 index 00000000..a16bdde6 --- /dev/null +++ b/cloudinit/handlers/__init__.py @@ -0,0 +1,274 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2008-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Chuck Short +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import yaml +import cloudinit +import cloudinit.util as util +import sys +import traceback +import os +import subprocess +import time + +per_instance = cloudinit.per_instance +per_always = cloudinit.per_always +per_once = cloudinit.per_once + + +class CloudConfig(): + cfgfile = None + cfg = None + + def __init__(self, cfgfile, cloud=None, ds_deps=None): + if cloud == None: + self.cloud = cloudinit.CloudInit(ds_deps) + self.cloud.get_data_source() + else: + self.cloud = cloud + self.cfg = self.get_config_obj(cfgfile) + + def get_config_obj(self, cfgfile): + try: + cfg = util.read_conf(cfgfile) + except: + # TODO: this 'log' could/should be passed in + cloudinit.log.critical("Failed loading of cloud config '%s'. " + "Continuing with empty config\n" % cfgfile) + cloudinit.log.debug(traceback.format_exc() + "\n") + cfg = None + if cfg is None: + cfg = {} + + try: + ds_cfg = self.cloud.datasource.get_config_obj() + except: + ds_cfg = {} + + cfg = util.mergedict(cfg, ds_cfg) + return(util.mergedict(cfg, self.cloud.cfg)) + + def handle(self, name, args, freq=None): + try: + mod = __import__("cc_" + name.replace("-", "_"), globals()) + def_freq = getattr(mod, "frequency", per_instance) + handler = getattr(mod, "handle") + + if not freq: + freq = def_freq + + self.cloud.sem_and_run("config-" + name, freq, handler, + [name, self.cfg, self.cloud, cloudinit.log, args]) + except: + raise + + +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg, name): + if name not in cfg: + return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item, str): + module_list.append((item,)) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + + +def run_cc_modules(cc, module_list, log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args)) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name, freq, run_args)) + failures.append(name) + + return(failures) + + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [None, None] + if not 'output' in cfg: + return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: + return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout and stderr + if isinstance(modecfg, str): + ret = [modecfg, modecfg] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": + ret[1] = ret[0] + + swlist = [">>", ">", "|"] + for i in range(len(ret)): + if not ret[i]: + continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s, val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % (">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + +def run_per_instance(name, func, args, clear_on_fail=False): + semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) + if os.path.exists(semfile): + return + + util.write_file(semfile, str(time.time())) + try: + func(*args) + except: + if clear_on_fail: + os.unlink(semfile) + raise + + +# apt_get top level command (install, update...), and args to pass it +def apt_get(tlc, args=None): + if args is None: + args = [] + e = os.environ.copy() + e['DEBIAN_FRONTEND'] = 'noninteractive' + cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', + '--assume-yes', tlc] + cmd.extend(args) + subprocess.check_call(cmd, env=e) + + +def update_package_sources(): + run_per_instance("update-sources", apt_get, ("update",)) + + +def install_packages(pkglist): + update_package_sources() + apt_get("install", pkglist) diff --git a/cloudinit/handlers/cc_apt_pipelining.py b/cloudinit/handlers/cc_apt_pipelining.py new file mode 100644 index 00000000..0286a9ae --- /dev/null +++ b/cloudinit/handlers/cc_apt_pipelining.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" + + +def handle(_name, cfg, _cloud, log, _args): + + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() + + if apt_pipe_value == "false": + write_apt_snippet("0", log) + + elif apt_pipe_value in ("none", "unchanged", "os"): + return + + elif apt_pipe_value in str(range(0, 6)): + write_apt_snippet(apt_pipe_value, log) + + else: + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ Writes f_name with apt pipeline depth 'setting' """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) + + util.write_file(f_name, file_contents) + + log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/handlers/cc_apt_update_upgrade.py b/cloudinit/handlers/cc_apt_update_upgrade.py new file mode 100644 index 00000000..a7049bce --- /dev/null +++ b/cloudinit/handlers/cc_apt_update_upgrade.py @@ -0,0 +1,241 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback +import os +import glob +import cloudinit.CloudConfig as cc + + +def handle(_name, cfg, cloud, log, _args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + release = get_release() + + mirror = find_apt_mirror(cloud, cfg) + + log.debug("selected mirror at: %s" % mirror) + + if not util.get_cfg_option_bool(cfg, \ + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy \"%s\";\n" + with open(proxy_filename, "w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + + # process 'apt_sources' + if 'apt_sources' in cfg: + errors = add_sources(cfg['apt_sources'], + {'MIRROR': mirror, 'RELEASE': release}) + for e in errors: + log.warn("Source Error: %s\n" % ':'.join(e)) + + dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) + if dconf_sel: + log.debug("setting debconf selections per cloud config") + try: + util.subp(('debconf-set-selections', '-'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections") + log.debug(traceback.format_exc()) + + pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cc.update_package_sources() + except subprocess.CalledProcessError as e: + log.warn("apt-get update failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if upgrade: + try: + cc.apt_get("upgrade") + except subprocess.CalledProcessError as e: + log.warn("apt upgrade failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if len(pkglist): + try: + cc.install_packages(pkglist) + except subprocess.CalledProcessError as e: + log.warn("Failed to install packages: %s " % pkglist) + log.debug(traceback.format_exc()) + errors.append(e) + + if len(errors): + raise errors[0] + + return(True) + + +def mirror2lists_fileprefix(mirror): + string = mirror + # take of http:// or ftp:// + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if(oprefix == nprefix): + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + + +def get_release(): + stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], + stdout=subprocess.PIPE).communicate() + return(str(stdout).strip()) + + +def generate_sources_list(codename, mirror): + util.render_to_file('sources.list', '/etc/apt/sources.list', \ + {'mirror': mirror, 'codename': codename}) + + +def add_sources(srclist, searchList=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if searchList is None: + searchList = {} + elst = [] + + for ent in srclist: + if 'source' not in ent: + elst.append(["", "missing source"]) + continue + + source = ent['source'] + if source.startswith("ppa:"): + try: + util.subp(["add-apt-repository", source]) + except: + elst.append([source, "add-apt-repository failed"]) + continue + + source = util.render_string(source, searchList) + + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = "%s/%s" % \ + ("/etc/apt/sources.list.d/", ent['filename']) + + if ('keyid' in ent and 'key' not in ent): + ks = "keyserver.ubuntu.com" + if 'keyserver' in ent: + ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + elst.append([source, "failed to get key from %s" % ks]) + continue + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + elst.append([source, "failed add key"]) + + try: + util.write_file(ent['filename'], source + "\n", omode="ab") + except: + elst.append([source, "failed write to file %s" % ent['filename']]) + + return(elst) + + +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + cfg_mirror = cfg.get("apt_mirror", None) + if cfg_mirror: + mirror = cfg["apt_mirror"] + elif "apt_mirror_search" in cfg: + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + if not mirror: + doms.extend((".localdomain", "",)) + + mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % post) + + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/handlers/cc_bootcmd.py b/cloudinit/handlers/cc_bootcmd.py new file mode 100644 index 00000000..f584da02 --- /dev/null +++ b/cloudinit/handlers/cc_bootcmd.py @@ -0,0 +1,48 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +import tempfile +import os +from cloudinit.CloudConfig import per_always +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if "bootcmd" not in cfg: + return + + try: + content = util.shellify(cfg["bootcmd"]) + tmpf = tempfile.TemporaryFile() + tmpf.write(content) + tmpf.seek(0) + except: + log.warn("failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) + tmpf.close() + except: + log.warn("failed to run commands from bootcmd") + raise diff --git a/cloudinit/handlers/cc_byobu.py b/cloudinit/handlers/cc_byobu.py new file mode 100644 index 00000000..e821b261 --- /dev/null +++ b/cloudinit/handlers/cc_byobu.py @@ -0,0 +1,77 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + value = args[0] + else: + value = util.get_cfg_option_str(cfg, "byobu_by_default", "") + + if not value: + return + + if value == "user" or value == "system": + value = "enable-%s" % value + + valid = ("enable-user", "enable-system", "enable", + "disable-user", "disable-system", "disable") + if not value in valid: + log.warn("Unknown value %s for byobu_by_default" % value) + + mod_user = value.endswith("-user") + mod_sys = value.endswith("-system") + if value.startswith("enable"): + bl_inst = "install" + dc_val = "byobu byobu/launch-by-default boolean true" + mod_sys = True + else: + if value == "disable": + mod_user = True + mod_sys = True + bl_inst = "uninstall" + dc_val = "byobu byobu/launch-by-default boolean false" + + shcmd = "" + if mod_user: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += " || X=$(($X+1)); " + if mod_sys: + shcmd += "echo \"%s\" | debconf-set-selections" % dc_val + shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" + shcmd += " || X=$(($X+1)); " + + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] + + log.debug("setting byobu to %s" % value) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_ca_certs.py b/cloudinit/handlers/cc_ca_certs.py new file mode 100644 index 00000000..3af6238a --- /dev/null +++ b/cloudinit/handlers/cc_ca_certs.py @@ -0,0 +1,90 @@ +# vi: ts=4 expandtab +# +# Author: Mike Milner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +from subprocess import check_call +from cloudinit.util import (write_file, get_cfg_option_list_or_str, + delete_dir_contents, subp) + +CA_CERT_PATH = "/usr/share/ca-certificates/" +CA_CERT_FILENAME = "cloud-init-ca-certs.crt" +CA_CERT_CONFIG = "/etc/ca-certificates.conf" +CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" + + +def update_ca_certs(): + """ + Updates the CA certificate cache on the current machine. + """ + check_call(["update-ca-certificates"]) + + +def add_ca_certs(certs): + """ + Adds certificates to the system. To actually apply the new certificates + you must also call L{update_ca_certs}. + + @param certs: A list of certificate strings. + """ + if certs: + cert_file_contents = "\n".join(certs) + cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) + write_file(cert_file_fullpath, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. + write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + + +def remove_default_ca_certs(): + """ + Removes all default trusted CA certificates from the system. To actually + apply the change you must also call L{update_ca_certs}. + """ + delete_dir_contents(CA_CERT_PATH) + delete_dir_contents(CA_CERT_SYSTEM_PATH) + write_file(CA_CERT_CONFIG, "", mode=0644) + debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" + subp(('debconf-set-selections', '-'), debconf_sel) + + +def handle(_name, cfg, _cloud, log, _args): + """ + Call to handle ca-cert sections in cloud-config file. + + @param name: The module name "ca-cert" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + # If there isn't a ca-certs section in the configuration don't do anything + if "ca-certs" not in cfg: + return + ca_cert_cfg = cfg['ca-certs'] + + # If there is a remove-defaults option set to true, remove the system + # default trusted CA certs first. + if ca_cert_cfg.get("remove-defaults", False): + log.debug("removing default certificates") + remove_default_ca_certs() + + # If we are given any new trusted CA certs to add, add them. + if "trusted" in ca_cert_cfg: + trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + if trusted_certs: + log.debug("adding %d certificates" % len(trusted_certs)) + add_ca_certs(trusted_certs) + + # Update the system with the new cert configuration. + update_ca_certs() diff --git a/cloudinit/handlers/cc_chef.py b/cloudinit/handlers/cc_chef.py new file mode 100644 index 00000000..941e04fe --- /dev/null +++ b/cloudinit/handlers/cc_chef.py @@ -0,0 +1,119 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Avishai Ish-Shalom +# Author: Mike Moulton +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import json +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +ruby_version_default = "1.8" + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything + if 'chef' not in cfg: + return + chef_cfg = cfg['chef'] + + # ensure the chef directories we use exist + mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + + # set the validation key based on the presence of either 'validation_key' + # or 'validation_cert'. In the case where both exist, 'validation_key' + # takes precedence + for key in ('validation_key', 'validation_cert'): + if key in chef_cfg and chef_cfg[key]: + with open('/etc/chef/validation.pem', 'w') as validation_key_fh: + validation_key_fh.write(chef_cfg[key]) + break + + # create the chef config from template + util.render_to_file('chef_client.rb', '/etc/chef/client.rb', + {'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name']}) + + # set the firstboot json + with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + firstboot_json_fh.write(json.dumps(initial_json)) + + # If chef is not installed, we install chef based on 'install_type' + if not os.path.isfile('/usr/bin/chef-client'): + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # this will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + ruby_version_default) + install_chef_from_gems(ruby_version, chef_version) + # and finally, run chef-client + log.debug('running chef-client') + subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', + '-s', '20']) + else: + # this will install and run the chef-client from packages + cc.install_packages(('chef',)) + + +def get_ruby_packages(version): + # return a list of packages needed to install ruby at version + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + if version == "1.8": + pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) + return(pkgs) + + +def install_chef_from_gems(ruby_version, chef_version=None): + cc.install_packages(get_ruby_packages(ruby_version)) + if not os.path.exists('/usr/bin/gem'): + os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + if not os.path.exists('/usr/bin/ruby'): + os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + if chef_version: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) + else: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) + + +def ensure_dir(d): + if not os.path.exists(d): + os.makedirs(d) + + +def mkdirs(dirs): + for d in dirs: + ensure_dir(d) diff --git a/cloudinit/handlers/cc_disable_ec2_metadata.py b/cloudinit/handlers/cc_disable_ec2_metadata.py new file mode 100644 index 00000000..6b31ea8e --- /dev/null +++ b/cloudinit/handlers/cc_disable_ec2_metadata.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, _log, _args): + if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + fwall = "route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) diff --git a/cloudinit/handlers/cc_final_message.py b/cloudinit/handlers/cc_final_message.py new file mode 100644 index 00000000..abb4ca32 --- /dev/null +++ b/cloudinit/handlers/cc_final_message.py @@ -0,0 +1,58 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + + try: + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) + except: + ts = "na" + + try: + subs = {'UPTIME': uptime, 'TIMESTAMP': ts} + sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/handlers/cc_foo.py b/cloudinit/handlers/cc_foo.py new file mode 100644 index 00000000..35ec3fa7 --- /dev/null +++ b/cloudinit/handlers/cc_foo.py @@ -0,0 +1,29 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#import cloudinit +#import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance + + +def handle(_name, _cfg, _cloud, _log, _args): + print "hi" diff --git a/cloudinit/handlers/cc_grub_dpkg.py b/cloudinit/handlers/cc_grub_dpkg.py new file mode 100644 index 00000000..9f3a7eaf --- /dev/null +++ b/cloudinit/handlers/cc_grub_dpkg.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import traceback +import os + + +def handle(_name, cfg, _cloud, log, _args): + idevs = None + idevs_empty = None + + if "grub-dpkg" in cfg: + idevs = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices_empty", None) + + if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + if idevs == None: + idevs = "" + if idevs_empty == None: + idevs_empty = "true" + else: + if idevs_empty == None: + idevs_empty = "false" + if idevs == None: + idevs = "/dev/sda" + for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + if os.path.exists(dev): + idevs = dev + break + + # now idevs and idevs_empty are set to determined values + # or, those set by user + + dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ + "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty + log.debug("setting grub debconf-set-selections with '%s','%s'" % + (idevs, idevs_empty)) + + try: + util.subp(('debconf-set-selections'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections for grub-dpkg") + log.debug(traceback.format_exc()) diff --git a/cloudinit/handlers/cc_keys_to_console.py b/cloudinit/handlers/cc_keys_to_console.py new file mode 100644 index 00000000..73a477c0 --- /dev/null +++ b/cloudinit/handlers/cc_keys_to_console.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +import subprocess + +frequency = per_instance + + +def handle(_name, cfg, _cloud, log, _args): + cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] + fp_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_fp_console_blacklist", []) + key_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_key_console_blacklist", ["ssh-dss"]) + try: + confp = open('/dev/console', "wb") + cmd.append(','.join(fp_blacklist)) + cmd.append(','.join(key_blacklist)) + subprocess.call(cmd, stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/handlers/cc_landscape.py b/cloudinit/handlers/cc_landscape.py new file mode 100644 index 00000000..a4113cbe --- /dev/null +++ b/cloudinit/handlers/cc_landscape.py @@ -0,0 +1,75 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +from cloudinit.CloudConfig import per_instance +from configobj import ConfigObj + +frequency = per_instance + +lsc_client_cfg_file = "/etc/landscape/client.conf" + +# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 +lsc_builtincfg = { + 'client': { + 'log_level': "info", + 'url': "https://landscape.canonical.com/message-system", + 'ping_url': "http://landscape.canonical.com/ping", + 'data_path': "/var/lib/landscape/client", + } +} + + +def handle(_name, cfg, _cloud, log, _args): + """ + Basically turn a top level 'landscape' entry with a 'client' dict + and render it to ConfigObj format under '[client]' section in + /etc/landscape/client.conf + """ + + ls_cloudcfg = cfg.get("landscape", {}) + + if not isinstance(ls_cloudcfg, dict): + raise(Exception("'landscape' existed in config, but not a dict")) + + merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + + if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): + os.makedirs(os.path.dirname(lsc_client_cfg_file)) + + with open(lsc_client_cfg_file, "w") as fp: + merged.write(fp) + + log.debug("updated %s" % lsc_client_cfg_file) + + +def mergeTogether(objs): + """ + merge together ConfigObj objects or things that ConfigObj() will take in + later entries override earlier + """ + cfg = ConfigObj({}) + for obj in objs: + if isinstance(obj, ConfigObj): + cfg.merge(obj) + else: + cfg.merge(ConfigObj(obj)) + return cfg diff --git a/cloudinit/handlers/cc_locale.py b/cloudinit/handlers/cc_locale.py new file mode 100644 index 00000000..2bb22fdb --- /dev/null +++ b/cloudinit/handlers/cc_locale.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os.path +import subprocess +import traceback + + +def apply_locale(locale, cfgfile): + if os.path.exists('/usr/sbin/locale-gen'): + subprocess.Popen(['locale-gen', locale]).communicate() + if os.path.exists('/usr/sbin/update-locale'): + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', cfgfile, {'locale': locale}) + + +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) + + locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", + "/etc/default/locale") + + if not locale: + return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale, locale_cfgfile) + except Exception as e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/handlers/cc_mcollective.py b/cloudinit/handlers/cc_mcollective.py new file mode 100644 index 00000000..a2a6230c --- /dev/null +++ b/cloudinit/handlers/cc_mcollective.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Marc Cluet +# Based on code by Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +pubcert_file = "/etc/mcollective/ssl/server-public.pem" +pricert_file = "/etc/mcollective/ssl/server-private.pem" + + +# Our fake header section +class FakeSecHead(object): + def __init__(self, fp): + self.fp = fp + self.sechead = '[nullsection]\n' + + def readline(self): + if self.sechead: + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a mcollective key in the configuration don't do anything + if 'mcollective' not in cfg: + return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... + cc.install_packages(("mcollective",)) + + # ... and then update the mcollective configuration + if 'conf' in mcollective_cfg: + # Create object for reading server.cfg values + mcollective_config = ConfigParser.ConfigParser() + # Read server.cfg values from original file in order to be able to mix + # the rest up + mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' + 'server.cfg'))) + for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + if cfg_name == 'public-cert': + util.write_file(pubcert_file, cfg, mode=0644) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_public', pubcert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + elif cfg_name == 'private-cert': + util.write_file(pricert_file, cfg, mode=0600) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_private', pricert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + else: + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + mcollective_config.set(cfg_name, o, v) + # We got all our config as wanted we'll rename + # the previous server.cfg and create our new one + os.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + outputfile = StringIO.StringIO() + mcollective_config.write(outputfile) + # Now we got the whole file, write to disk except first line + # Note below, that we've just used ConfigParser because it generally + # works. Below, we remove the initial 'nullsection' header + # and then change 'key = value' to 'key: value'. The global + # search and replace of '=' with ':' could be problematic though. + # this most likely needs fixing. + util.write_file('/etc/mcollective/server.cfg', + outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', + ':'), + mode=0644) + + # Start mcollective + subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/handlers/cc_mounts.py b/cloudinit/handlers/cc_mounts.py new file mode 100644 index 00000000..6cdd74e8 --- /dev/null +++ b/cloudinit/handlers/cc_mounts.py @@ -0,0 +1,179 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os +import re +from string import whitespace # pylint: disable=W0402 + + +def is_mdname(name): + # return true if this is a metadata service name + if name in ["ami", "root", "swap"]: + return True + # names 'ephemeral0' or 'ephemeral1' + # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' + for enumname in ("ephemeral", "ebs"): + if name.startswith(enumname) and name.find(":") == -1: + return True + return False + + +def handle(_name, cfg, cloud, log, _args): + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + defvals = cfg.get("mount_default_fields", defvals) + + # these are our default set of mounts + defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"]] + + cfgmnt = [] + if "mounts" in cfg: + cfgmnt = cfg["mounts"] + + # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 + shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" + shortname = re.compile(shortname_filter) + + for i in range(len(cfgmnt)): + # skip something that wasn't a list + if not isinstance(cfgmnt[i], list): + continue + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + if cfgmnt[i][0] == "ephemeral": + cfgmnt[i][0] = "ephemeral0" + + if is_mdname(cfgmnt[i][0]): + newname = cloud.device_name_to_device(cfgmnt[i][0]) + if not newname: + log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + cfgmnt[i][1] = None + else: + if newname.startswith("/"): + cfgmnt[i][0] = newname + else: + cfgmnt[i][0] = "/dev/%s" % newname + else: + if shortname.match(cfgmnt[i][0]): + cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + + # in case the user did not quote a field (likely fs-freq, fs_passno) + # but do not convert None to 'None' (LP: #898365) + for j in range(len(cfgmnt[i])): + if isinstance(cfgmnt[i][j], int): + cfgmnt[i][j] = str(cfgmnt[i][j]) + + for i in range(len(cfgmnt)): + # fill in values with defaults from defvals above + for j in range(len(defvals)): + if len(cfgmnt[i]) <= j: + cfgmnt[i].append(defvals[j]) + elif cfgmnt[i][j] is None: + cfgmnt[i][j] = defvals[j] + + # if the second entry in the list is 'None' this + # clears all previous entries of that same 'fs_spec' + # (fs_spec is the first field in /etc/fstab, ie, that device) + if cfgmnt[i][1] is None: + for j in range(i): + if cfgmnt[j][0] == cfgmnt[i][0]: + cfgmnt[j][1] = None + + # for each of the "default" mounts, add them only if no other + # entry has the same device name + for defmnt in defmnts: + devname = cloud.device_name_to_device(defmnt[0]) + if devname is None: + continue + if devname.startswith("/"): + defmnt[0] = devname + else: + defmnt[0] = "/dev/%s" % devname + + cfgmnt_has = False + for cfgm in cfgmnt: + if cfgm[0] == defmnt[0]: + cfgmnt_has = True + break + + if cfgmnt_has: + continue + cfgmnt.append(defmnt) + + # now, each entry in the cfgmnt list has all fstab values + # if the second field is None (not the string, the value) we skip it + actlist = [x for x in cfgmnt if x[1] is not None] + + if len(actlist) == 0: + return + + comment = "comment=cloudconfig" + cc_lines = [] + needswap = False + dirs = [] + for line in actlist: + # write 'comment' in the fs_mntops, entry, claiming this + line[3] = "%s,comment=cloudconfig" % line[3] + if line[2] == "swap": + needswap = True + if line[1].startswith("/"): + dirs.append(line[1]) + cc_lines.append('\t'.join(line)) + + fstab_lines = [] + fstab = open("/etc/fstab", "r+") + ws = re.compile("[%s]+" % whitespace) + for line in fstab.read().splitlines(): + try: + toks = ws.split(line) + if toks[3].find(comment) != -1: + continue + except: + pass + fstab_lines.append(line) + + fstab_lines.extend(cc_lines) + + fstab.seek(0) + fstab.write("%s\n" % '\n'.join(fstab_lines)) + fstab.truncate() + fstab.close() + + if needswap: + try: + util.subp(("swapon", "-a")) + except: + log.warn("Failed to enable swap") + + for d in dirs: + if os.path.exists(d): + continue + try: + os.makedirs(d) + except: + log.warn("Failed to make '%s' config-mount\n", d) + + try: + util.subp(("mount", "-a")) + except: + log.warn("'mount -a' failed") diff --git a/cloudinit/handlers/cc_phone_home.py b/cloudinit/handlers/cc_phone_home.py new file mode 100644 index 00000000..a7ff74e1 --- /dev/null +++ b/cloudinit/handlers/cc_phone_home.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', + 'hostname'] + + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id +# +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + ph_cfg = util.read_conf(args[0]) + else: + if not 'phone_home' in cfg: + return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries', 10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = {} + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = {} + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + + null_exc = object() + last_e = null_exc + for i in range(0, tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i + 1)) + return + except Exception as e: + log.debug("failed to post to %s on try %i" % (url, i + 1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e is not null_exc: + raise(last_e) + + return diff --git a/cloudinit/handlers/cc_puppet.py b/cloudinit/handlers/cc_puppet.py new file mode 100644 index 00000000..6fc475f6 --- /dev/null +++ b/cloudinit/handlers/cc_puppet.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import pwd +import socket +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a puppet key in the configuration don't do anything + if 'puppet' not in cfg: + return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... + cc.install_packages(("puppet",)) + + # ... and then update the puppet configuration + if 'conf' in puppet_cfg: + # Add all sections from the conf object to puppet.conf + puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + # Create object for reading puppet.conf values + puppet_config = ConfigParser.ConfigParser() + # Read puppet.conf values from original file in order to be able to + # mix the rest up + puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in + puppet_conf_fh.readlines()))) + # Close original file, no longer needed + puppet_conf_fh.close() + for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # ca_cert configuration is a special case + # Dump the puppetmaster ca certificate in the correct place + if cfg_name == 'ca_cert': + # Puppet ssl sub-directory isn't created yet + # Create it with the proper permissions and ownership + os.makedirs('/var/lib/puppet/ssl') + os.chmod('/var/lib/puppet/ssl', 0771) + os.chown('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + os.makedirs('/var/lib/puppet/ssl/certs/') + os.chown('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') + ca_fh.write(cfg) + ca_fh.close() + os.chown('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) + util.restorecon_if_possible('/var/lib/puppet', recursive=True) + else: + #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) + # If puppet.conf already has this section we don't want to + # write it again + if puppet_config.has_section(cfg_name) == False: + puppet_config.add_section(cfg_name) + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + if o == 'certname': + # Expand %f as the fqdn + v = v.replace("%f", socket.getfqdn()) + # Expand %i as the instance id + v = v.replace("%i", + cloud.datasource.get_instance_id()) + # certname needs to be downcase + v = v.lower() + puppet_config.set(cfg_name, o, v) + #puppet_conf_fh.write("%s=%s\n" % (o, v)) + # We got all our config as wanted we'll rename + # the previous puppet.conf and create our new one + os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + with open('/etc/puppet/puppet.conf', 'wb') as configfile: + puppet_config.write(configfile) + util.restorecon_if_possible('/etc/puppet/puppet.conf') + # Set puppet to automatically start + if os.path.exists('/etc/default/puppet'): + subprocess.check_call(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet']) + elif os.path.exists('/bin/systemctl'): + subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + elif os.path.exists('/sbin/chkconfig'): + subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + else: + log.warn("Do not know how to enable puppet service on this system") + # Start puppetd + subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/handlers/cc_resizefs.py b/cloudinit/handlers/cc_resizefs.py new file mode 100644 index 00000000..2dc66def --- /dev/null +++ b/cloudinit/handlers/cc_resizefs.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import os +import stat +import sys +import time +import tempfile +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = False + if str(args[0]).lower() in ['true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if str(resize_root).lower() in ['false', '0']: + return + + # we use mktemp rather than mkstemp because early in boot nothing + # else should be able to race us for this, and we need to mknod. + devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") + + try: + st_dev = os.stat("/").st_dev + dev = os.makedev(os.major(st_dev), os.minor(st_dev)) + os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + except: + if util.is_container(): + log.debug("inside container, ignoring mknod failure in resizefs") + return + log.warn("Failed to make device node to resize /") + raise + + cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + try: + (fstype, _err) = util.subp(cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % + (os.major(st_dev), os.minor(st_dev), cmd)) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + if str(fstype).startswith("ext"): + resize_cmd = ['resize2fs', devpth] + elif fstype == "xfs": + resize_cmd = ['xfs_growfs', devpth] + else: + os.unlink(devpth) + log.debug("not resizing unknown filesystem %s" % fstype) + return + + if resize_root == "noblock": + fid = os.fork() + if fid == 0: + try: + do_resize(resize_cmd, devpth, log) + os._exit(0) # pylint: disable=W0212 + except Exception as exc: + sys.stderr.write("Failed: %s" % exc) + os._exit(1) # pylint: disable=W0212 + else: + do_resize(resize_cmd, devpth, log) + + log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % + (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), + resize_root)) + + return + + +def do_resize(resize_cmd, devpth, log): + try: + start = time.time() + util.subp(resize_cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to resize filesystem (%s)" % resize_cmd) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + os.unlink(devpth) + log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/handlers/cc_rightscale_userdata.py b/cloudinit/handlers/cc_rightscale_userdata.py new file mode 100644 index 00000000..5ed0848f --- /dev/null +++ b/cloudinit/handlers/cc_rightscale_userdata.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + + +def handle(_name, _cfg, cloud, log, _args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(ud) + if not my_hookname in mdict: + return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d, i) + i = i + 1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception as e: + if not first_e: + first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/handlers/cc_rsyslog.py b/cloudinit/handlers/cc_rsyslog.py new file mode 100644 index 00000000..ac7f2c74 --- /dev/null +++ b/cloudinit/handlers/cc_rsyslog.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit +import logging +import cloudinit.util as util +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + + +def handle(_name, cfg, _cloud, log, _args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: + return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + files = [] + elst = [] + for ent in cfg['rsyslog']: + if isinstance(ent, dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir, filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception as e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception as e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/handlers/cc_runcmd.py b/cloudinit/handlers/cc_runcmd.py new file mode 100644 index 00000000..f7e8c671 --- /dev/null +++ b/cloudinit/handlers/cc_runcmd.py @@ -0,0 +1,32 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if "runcmd" not in cfg: + return + outfile = "%s/runcmd" % cloud.get_ipath('scripts') + try: + content = util.shellify(cfg["runcmd"]) + util.write_file(outfile, content, 0700) + except: + log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/handlers/cc_salt_minion.py b/cloudinit/handlers/cc_salt_minion.py new file mode 100644 index 00000000..1a3b5039 --- /dev/null +++ b/cloudinit/handlers/cc_salt_minion.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Author: Jeff Bauer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import subprocess +import cloudinit.CloudConfig as cc +import yaml + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a salt key in the configuration don't do anything + if 'salt_minion' not in cfg: + return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... + cc.install_packages(("salt",)) + config_dir = '/etc/salt' + if not os.path.isdir(config_dir): + os.makedirs(config_dir) + # ... and then update the salt configuration + if 'conf' in salt_cfg: + # Add all sections from the conf object to /etc/salt/minion + minion_config = os.path.join(config_dir, 'minion') + yaml.dump(salt_cfg['conf'], + file(minion_config, 'w'), + default_flow_style=False) + # ... copy the key pair if specified + if 'public_key' in salt_cfg and 'private_key' in salt_cfg: + pki_dir = '/etc/salt/pki' + cumask = os.umask(077) + if not os.path.isdir(pki_dir): + os.makedirs(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + with open(pub_name, 'w') as f: + f.write(salt_cfg['public_key']) + with open(pem_name, 'w') as f: + f.write(salt_cfg['private_key']) + os.umask(cumask) + + # Start salt-minion + subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/handlers/cc_scripts_per_boot.py b/cloudinit/handlers/cc_scripts_per_boot.py new file mode 100644 index 00000000..41a74754 --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_boot.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +from cloudinit import get_cpath + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_per_instance.py b/cloudinit/handlers/cc_scripts_per_instance.py new file mode 100644 index 00000000..a2981eab --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_instance.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_cpath + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_per_once.py b/cloudinit/handlers/cc_scripts_per_once.py new file mode 100644 index 00000000..a69151da --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_once.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_once +from cloudinit import get_cpath + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_user.py b/cloudinit/handlers/cc_scripts_user.py new file mode 100644 index 00000000..933aa4e0 --- /dev/null +++ b/cloudinit/handlers/cc_scripts_user.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_set_hostname.py b/cloudinit/handlers/cc_set_hostname.py new file mode 100644 index 00000000..acea74d9 --- /dev/null +++ b/cloudinit/handlers/cc_set_hostname.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + set_hostname(hostname, log) + except Exception: + util.logexc(log) + log.warn("failed to set hostname to %s\n", hostname) + + return(True) + + +def set_hostname(hostname, log): + util.subp(['hostname', hostname]) + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/handlers/cc_set_passwords.py b/cloudinit/handlers/cc_set_passwords.py new file mode 100644 index 00000000..9d0bbdb8 --- /dev/null +++ b/cloudinit/handlers/cc_set_passwords.py @@ -0,0 +1,129 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import sys +import random +from string import letters, digits # pylint: disable=W0402 + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + # if run from command line, and give args, wipe the chpasswd['list'] + password = args[0] + if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: + del cfg['chpasswd']['list'] + else: + password = util.get_cfg_option_str(cfg, "password", None) + + expire = True + pw_auth = "no" + change_pwauth = False + plist = None + + if 'chpasswd' in cfg: + chfg = cfg['chpasswd'] + plist = util.get_cfg_option_str(chfg, 'list', plist) + expire = util.get_cfg_option_bool(chfg, 'expire', expire) + + if not plist and password: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + plist = "%s:%s" % (user, password) + + errors = [] + if plist: + plist_in = [] + randlist = [] + users = [] + for line in plist.splitlines(): + u, p = line.split(':', 1) + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) + + ch_in = '\n'.join(plist_in) + try: + util.subp(['chpasswd'], ch_in) + log.debug("changed password for %s:" % users) + except Exception as e: + errors.append(e) + log.warn("failed to set passwords with chpasswd: %s" % e) + + if len(randlist): + sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + '\n'.join(randlist))) + + if expire: + enum = len(errors) + for u in users: + try: + util.subp(['passwd', '--expire', u]) + except Exception as e: + errors.append(e) + log.warn("failed to expire account for %s" % u) + if enum == len(errors): + log.debug("expired passwords for: %s" % u) + + if 'ssh_pwauth' in cfg: + val = str(cfg['ssh_pwauth']).lower() + if val in ("true", "1", "yes"): + pw_auth = "yes" + change_pwauth = True + elif val in ("false", "0", "no"): + pw_auth = "no" + change_pwauth = True + else: + change_pwauth = False + + if change_pwauth: + pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" + msg = "set PasswordAuthentication to '%s'" % pw_auth + try: + cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), + '/etc/ssh/sshd_config'] + util.subp(cmd) + log.debug(msg) + except Exception as e: + log.warn("failed %s" % msg) + errors.append(e) + + try: + p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), + 'restart']) + log.debug("restarted sshd") + except: + log.warn("restart of ssh failed") + + if len(errors): + raise(errors[0]) + + return + + +def rand_str(strlen=32, select_from=letters + digits): + return("".join([random.choice(select_from) for _x in range(0, strlen)])) + + +def rand_user_password(pwlen=9): + selfrom = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/handlers/cc_ssh.py b/cloudinit/handlers/cc_ssh.py new file mode 100644 index 00000000..48eb58bc --- /dev/null +++ b/cloudinit/handlers/cc_ssh.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import os +import glob +import subprocess + +DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ +"rather than the user \\\"root\\\".\';echo;sleep 10\"" + + +def handle(_name, cfg, cloud, log, _args): + + # remove the static keys from the pristine image + if cfg.get("ssh_deletekeys", True): + for f in glob.glob("/etc/ssh/ssh_host_*key*"): + try: + os.unlink(f) + except: + pass + + if "ssh_keys" in cfg: + # if there are keys in cloud-config, use them + key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + } + + for key, val in cfg["ssh_keys"].items(): + if key in key2file: + util.write_file(key2file[key][0], val, key2file[key][1]) + + priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', } + + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): + if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + continue + pair = (key2file[priv][0], key2file[pub][0]) + subprocess.call(('sh', '-xc', cmd % pair)) + log.debug("generated %s from %s" % pair) + else: + # if not, generate them + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', + ['rsa', 'dsa', 'ecdsa']): + keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + if not os.path.exists(keyfile): + subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', + '-f', keyfile]) + + util.restorecon_if_possible('/etc/ssh', recursive=True) + + try: + user = util.get_cfg_option_str(cfg, 'user') + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) + disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", + DISABLE_ROOT_OPTS) + keys = cloud.get_public_ssh_keys() + + if "ssh_authorized_keys" in cfg: + cfgkeys = cfg["ssh_authorized_keys"] + keys.extend(cfgkeys) + + apply_credentials(keys, user, disable_root, disable_root_opts, log) + except: + util.logexc(log) + log.warn("applying credentials failed!\n") + + +def apply_credentials(keys, user, disable_root, + disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) + if user: + sshutil.setup_user_keys(keys, user, '', log) + + if disable_root: + key_prefix = disable_root_opts.replace('$USER', user) + else: + key_prefix = '' + + sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/handlers/cc_ssh_import_id.py b/cloudinit/handlers/cc_ssh_import_id.py new file mode 100644 index 00000000..bbf5bd83 --- /dev/null +++ b/cloudinit/handlers/cc_ssh_import_id.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + user = args[0] + ids = [] + if len(args) > 1: + ids = args[1:] + else: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + + if len(ids) == 0: + return + + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids + + log.debug("importing ssh ids. cmd = %s" % cmd) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_timezone.py b/cloudinit/handlers/cc_timezone.py new file mode 100644 index 00000000..e5c9901b --- /dev/null +++ b/cloudinit/handlers/cc_timezone.py @@ -0,0 +1,67 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +from cloudinit import util +import os.path +import shutil + +frequency = per_instance +tz_base = "/usr/share/zoneinfo" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + timezone = args[0] + else: + timezone = util.get_cfg_option_str(cfg, "timezone", False) + + if not timezone: + return + + tz_file = "%s/%s" % (tz_base, timezone) + + if not os.path.isfile(tz_file): + log.debug("Invalid timezone %s" % tz_file) + raise Exception("Invalid timezone %s" % tz_file) + + try: + fp = open("/etc/timezone", "wb") + fp.write("%s\n" % timezone) + fp.close() + except: + log.debug("failed to write to /etc/timezone") + raise + if os.path.exists("/etc/sysconfig/clock"): + try: + with open("/etc/sysconfig/clock", "w") as fp: + fp.write('ZONE="%s"\n' % timezone) + except: + log.debug("failed to write to /etc/sysconfig/clock") + raise + + try: + shutil.copy(tz_file, "/etc/localtime") + except: + log.debug("failed to copy %s to /etc/localtime" % tz_file) + raise + + log.debug("set timezone to %s" % timezone) + return diff --git a/cloudinit/handlers/cc_update_etc_hosts.py b/cloudinit/handlers/cc_update_etc_hosts.py new file mode 100644 index 00000000..6ad2fca8 --- /dev/null +++ b/cloudinit/handlers/cc_update_etc_hosts.py @@ -0,0 +1,87 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +import StringIO + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + + manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) + if manage_hosts in ("True", "true", True, "template"): + # render from template file + try: + if not hostname: + log.info("manage_etc_hosts was set, but no hostname found") + return + + util.render_to_file('hosts', '/etc/hosts', + {'hostname': hostname, 'fqdn': fqdn}) + except Exception: + log.warn("failed to update /etc/hosts") + raise + elif manage_hosts == "localhost": + log.debug("managing 127.0.1.1 in /etc/hosts") + update_etc_hosts(hostname, fqdn, log) + return + else: + if manage_hosts not in ("False", False): + log.warn("Unknown value for manage_etc_hosts. Assuming False") + else: + log.debug("not managing /etc/hosts") + + +def update_etc_hosts(hostname, fqdn, _log): + with open('/etc/hosts', 'r') as etchosts: + header = "# Added by cloud-init\n" + hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) + need_write = False + need_change = True + new_etchosts = StringIO.StringIO() + for line in etchosts: + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: + new_etchosts.write(line) + continue + if line == header: + continue + ip, hosts = split_line[0], split_line[1:] + if ip == "127.0.1.1": + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change == True: + line = "%s%s" % (header, hosts_line) + need_change = False + need_write = True + new_etchosts.write(line) + etchosts.close() + if need_change == True: + new_etchosts.write("%s%s" % (header, hosts_line)) + need_write = True + if need_write == True: + new_etcfile = open('/etc/hosts', 'wb') + new_etcfile.write(new_etchosts.getvalue()) + new_etcfile.close() + new_etchosts.close() + return diff --git a/cloudinit/handlers/cc_update_hostname.py b/cloudinit/handlers/cc_update_hostname.py new file mode 100644 index 00000000..b9d1919a --- /dev/null +++ b/cloudinit/handlers/cc_update_hostname.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not updating hostname") + return + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") + update_hostname(hostname, prev, log) + except Exception: + log.warn("failed to set hostname\n") + raise + + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename, "r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError as e: + if e.errno != errno.ENOENT: + raise + return default + + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception as e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname, "%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname, fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file, etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/importer.py b/cloudinit/importer.py new file mode 100644 index 00000000..deea0956 --- /dev/null +++ b/cloudinit/importer.py @@ -0,0 +1,11 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import sys + + +def import_module(module_name): + try: + __import__(module_name) + return sys.modules.get(module_name, None) + except ImportError as err: + raise RuntimeError('Could not load module %s: %s' % (module_name, err)) diff --git a/cloudinit/includer.py b/cloudinit/includer.py new file mode 100644 index 00000000..d1022c5a --- /dev/null +++ b/cloudinit/includer.py @@ -0,0 +1,65 @@ +import os +import re + +from cloudinit import downloader as down +from cloudinit import exceptions as excp +from cloudinit import log as logging +from cloudinit import shell as sh + +INCLUDE_PATT = re.compile("^#(opt_include|include)[ \t](.*)$", re.MULTILINE) +OPT_PATS = ['opt_include'] + +LOG = logging.getLogger(__name__) + + +class Includer(object): + + def __init__(self, root_fn, stack_limit=10): + self.root_fn = root_fn + self.stack_limit = stack_limit + + def _read_file(self, fname): + return sh.read_file(fname) + + def _read(self, fname, stack, rel): + if len(stack) >= self.stack_limit: + raise excp.StackExceeded("Stack limit of %s reached while including %s" % (self.stack_limit, fname)) + + canon_fname = self._canon_name(fname, rel) + if canon_fname in stack: + raise excp.RecursiveInclude("File %s recursively included" % (canon_fname)) + + stack.add(canon_fname) + new_rel = os.path.dirname(canon_fname) + contents = self._read_file(canon_fname) + + def include_cb(match): + is_optional = (match.group(1).lower() in OPT_PATS) + fn = match.group(2).strip() + if not fn: + # Should we die?? + return match.group(0) + else: + try: + LOG.debug("Including file %s", fn) + return self._read(fn, stack, new_rel) + except IOError: + if is_optional: + return '' + else: + raise + + adjusted_contents = INCLUDE_PATT.sub(include_cb, contents) + stack.remove(fname) + return adjusted_contents + + def _canon_name(self, fname, rel): + fname = fname.strip() + if not fname.startswith("/"): + fname = os.path.sep.join([rel, fname]) + return os.path.realpath(fname) + + def read(self, relative_to="."): + stack = set() + return self._read(self.root_fn, stack, rel=relative_to) + diff --git a/cloudinit/log.py b/cloudinit/log.py new file mode 100644 index 00000000..4d270045 --- /dev/null +++ b/cloudinit/log.py @@ -0,0 +1,94 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import logging +import logging.handlers +import sys + +# Logging levels for easy access +CRITICAL = logging.CRITICAL +FATAL = logging.FATAL +ERROR = logging.ERROR +WARNING = logging.WARNING +WARN = logging.WARN +INFO = logging.INFO +DEBUG = logging.DEBUG +NOTSET = logging.NOTSET + +# File log rotation settings +ROTATE_AMOUNT = 10 # Only keep the past 9 + 1 active +ROTATE_SIZE = 10 * 1024 * 1024 # 10 MB + + +class ConsoleFormatter(logging.Formatter): + + def _get_mini_level(self, record): + if record.levelno in [INFO, NOTSET] or not record.levelname: + return '' + lvlname = record.levelname + return lvlname[0].upper() + ": " + + def format(self, record): + record.message = record.getMessage() + rdict = dict(record.__dict__) + rdict['minilevelname'] = self._get_mini_level(record) + # Skipping exception info for the console... + return self._fmt % (rdict) + + +def setupLogging(level, filename=None, filelevel=logging.DEBUG): + root = getLogger() + consolelg = logging.StreamHandler(sys.stdout) + consolelg.setFormatter(ConsoleFormatter('%(minilevelname)s%(message)s')) + consolelg.setLevel(level) + root.addHandler(consolelg) + if filename: + filelg = logging.handlers.RotatingFileHandler(filename, maxBytes=ROTATE_SIZE, backupCount=ROTATE_AMOUNT) + filelg.setFormatter(logging.Formatter('%(levelname)s: @%(name)s : %(message)s')) + filelg.setLevel(filelevel) + root.addHandler(filelg) + root.setLevel(level) + + +def logging_set_from_cfg(cfg): + log_cfgs = [] + logcfg = util.get_cfg_option_str(cfg, "log_cfg", False) + if logcfg: + # if there is a 'logcfg' entry in the config, respect + # it, it is the old keyname + log_cfgs = [logcfg] + elif "log_cfgs" in cfg: + for cfg in cfg['log_cfgs']: + if isinstance(cfg, list): + log_cfgs.append('\n'.join(cfg)) + else: + log_cfgs.append() + + if not len(log_cfgs): + sys.stderr.write("Warning, no logging configured\n") + return + + for logcfg in log_cfgs: + try: + logging.config.fileConfig(StringIO.StringIO(logcfg)) + return + except: + pass + + raise Exception("no valid logging found\n") + + +def getLogger(name='cloudinit'): + return logging.getLogger(name) + + +# Fixes this annoyance... +# No handlers could be found for logger XXX annoying output... +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logger = getLogger() +logger.addHandler(NullHandler()) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py new file mode 100644 index 00000000..a081fbe8 --- /dev/null +++ b/cloudinit/ssh_util.py @@ -0,0 +1,227 @@ +#!/usr/bin/python +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import cloudinit.util as util + + +class AuthKeyEntry(): + # lines are options, keytype, base64-encoded key, comment + # man page says the following which I did not understand: + # The options field is optional; its presence is determined by whether + # the line starts with a number or not (the options field never starts + # with a number) + options = None + keytype = None + base64 = None + comment = None + is_comment = False + line_in = "" + + def __init__(self, line, def_opt=None): + line = line.rstrip("\n\r") + self.line_in = line + if line.startswith("#") or line.strip() == "": + self.is_comment = True + else: + ent = line.strip() + toks = ent.split(None, 3) + if len(toks) == 1: + self.base64 = toks[0] + elif len(toks) == 2: + (self.base64, self.comment) = toks + elif len(toks) == 3: + (self.keytype, self.base64, self.comment) = toks + elif len(toks) == 4: + i = 0 + ent = line.strip() + quoted = False + # taken from auth_rsa_key_allowed in auth-rsa.c + try: + while (i < len(ent) and + ((quoted) or (ent[i] not in (" ", "\t")))): + curc = ent[i] + nextc = ent[i + 1] + if curc == "\\" and nextc == '"': + i = i + 1 + elif curc == '"': + quoted = not quoted + i = i + 1 + except IndexError: + self.is_comment = True + return + + try: + self.options = ent[0:i] + (self.keytype, self.base64, self.comment) = \ + ent[i + 1:].split(None, 3) + except ValueError: + # we did not understand this line + self.is_comment = True + + if self.options == None and def_opt: + self.options = def_opt + + return + + def debug(self): + print("line_in=%s\ncomment: %s\noptions=%s\nkeytype=%s\nbase64=%s\n" + "comment=%s\n" % (self.line_in, self.is_comment, self.options, + self.keytype, self.base64, self.comment)), + + def __repr__(self): + if self.is_comment: + return(self.line_in) + else: + toks = [] + for e in (self.options, self.keytype, self.base64, self.comment): + if e: + toks.append(e) + + return(' '.join(toks)) + + +def update_authorized_keys(fname, keys): + # keys is a list of AuthKeyEntries + # key_prefix is the prefix (options) to prepend + try: + fp = open(fname, "r") + lines = fp.readlines() # lines have carriage return + fp.close() + except IOError: + lines = [] + + ka_stats = {} # keys_added status + for k in keys: + ka_stats[k] = False + + to_add = [] + for key in keys: + to_add.append(key) + + for i in range(0, len(lines)): + ent = AuthKeyEntry(lines[i]) + for k in keys: + if k.base64 == ent.base64 and not k.is_comment: + ent = k + try: + to_add.remove(k) + except ValueError: + pass + lines[i] = str(ent) + + # now append any entries we did not match above + for key in to_add: + lines.append(str(key)) + + if len(lines) == 0: + return("") + else: + return('\n'.join(lines) + "\n") + + +def setup_user_keys(keys, user, key_prefix, log=None): + import pwd + saved_umask = os.umask(077) + + pwent = pwd.getpwnam(user) + + ssh_dir = '%s/.ssh' % pwent.pw_dir + if not os.path.exists(ssh_dir): + os.mkdir(ssh_dir) + os.chown(ssh_dir, pwent.pw_uid, pwent.pw_gid) + + try: + ssh_cfg = parse_ssh_config() + akeys = ssh_cfg.get("AuthorizedKeysFile", "%h/.ssh/authorized_keys") + akeys = akeys.replace("%h", pwent.pw_dir) + akeys = akeys.replace("%u", user) + if not akeys.startswith('/'): + akeys = os.path.join(pwent.pw_dir, akeys) + authorized_keys = akeys + except Exception: + authorized_keys = '%s/.ssh/authorized_keys' % pwent.pw_dir + if log: + util.logexc(log) + + key_entries = [] + for k in keys: + ke = AuthKeyEntry(k, def_opt=key_prefix) + key_entries.append(ke) + + content = update_authorized_keys(authorized_keys, key_entries) + util.write_file(authorized_keys, content, 0600) + + os.chown(authorized_keys, pwent.pw_uid, pwent.pw_gid) + util.restorecon_if_possible(ssh_dir, recursive=True) + + os.umask(saved_umask) + + +def parse_ssh_config(fname="/etc/ssh/sshd_config"): + ret = {} + fp = open(fname) + for l in fp.readlines(): + l = l.strip() + if not l or l.startswith("#"): + continue + key, val = l.split(None, 1) + ret[key] = val + fp.close() + return(ret) + +if __name__ == "__main__": + def main(): + import sys + # usage: orig_file, new_keys, [key_prefix] + # prints out merged, where 'new_keys' will trump old + ## example + ## ### begin auth_keys ### + # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= smoser-work + # ssh-rsa AAAAB3NzaC1xxxxxxxxxCmXp5Kt5/82cD/VN3NtHw== smoser@brickies + # ### end authorized_keys ### + # + # ### begin new_keys ### + # ssh-rsa nonmatch smoser@newhost + # ssh-rsa AAAAB3NzaC1xxxxxxxxxV3csgm8cJn7UveKHkYjJp8= new_comment + # ### end new_keys ### + # + # Then run as: + # program auth_keys new_keys \ + # 'no-port-forwarding,command=\"echo hi world;\"' + def_prefix = None + orig_key_file = sys.argv[1] + new_key_file = sys.argv[2] + if len(sys.argv) > 3: + def_prefix = sys.argv[3] + fp = open(new_key_file) + + newkeys = [] + for line in fp.readlines(): + newkeys.append(AuthKeyEntry(line, def_prefix)) + + fp.close() + print update_authorized_keys(orig_key_file, newkeys) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/templater.py b/cloudinit/templater.py new file mode 100644 index 00000000..b6b320ab --- /dev/null +++ b/cloudinit/templater.py @@ -0,0 +1,17 @@ +import os + +from Cheetah.Template import Template + +from cloudinit import util + +TEMPLATE_DIR = '/etc/cloud/templates/' + + +def render_to_file(template, outfile, searchList): + contents = Template(file=os.path.join(TEMPLATE_DIR, template), + searchList=[searchList]).respond() + util.write_file(outfile, contents) + + +def render_string(template, searchList): + return Template(template, searchList=[searchList]).respond() diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py new file mode 100644 index 00000000..ec914480 --- /dev/null +++ b/cloudinit/user_data.py @@ -0,0 +1,262 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import email + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +import yaml +import cloudinit +import cloudinit.util as util +import hashlib +import urllib + + +starts_with_mappings = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook', + '#cloud-config-archive': 'text/cloud-config-archive', +} + + +# if 'string' is compressed return decompressed otherwise return it +def decomp_str(string): + import StringIO + import gzip + try: + uncomp = gzip.GzipFile(None, "rb", 1, StringIO.StringIO(string)).read() + return(uncomp) + except: + return(string) + + +def do_include(content, appendmsg): + import os + # is just a list of urls, one per line + # also support '#include ' + includeonce = False + for line in content.splitlines(): + if line == "#include": + continue + if line == "#include-once": + includeonce = True + continue + if line.startswith("#include-once"): + line = line[len("#include-once"):].lstrip() + includeonce = True + elif line.startswith("#include"): + line = line[len("#include"):].lstrip() + if line.startswith("#"): + continue + if line.strip() == "": + continue + + # urls cannot not have leading or trailing white space + msum = hashlib.md5() # pylint: disable=E1101 + msum.update(line.strip()) + includeonce_filename = "%s/urlcache/%s" % ( + cloudinit.get_ipath_cur("data"), msum.hexdigest()) + try: + if includeonce and os.path.isfile(includeonce_filename): + with open(includeonce_filename, "r") as fp: + content = fp.read() + else: + content = urllib.urlopen(line).read() + if includeonce: + util.write_file(includeonce_filename, content, mode=0600) + except Exception: + raise + + process_includes(message_from_string(decomp_str(content)), appendmsg) + + +def explode_cc_archive(archive, appendmsg): + for ent in yaml.load(archive): + # ent can be one of: + # dict { 'filename' : 'value', 'content' : 'value', 'type' : 'value' } + # filename and type not be present + # or + # scalar(payload) + + def_type = "text/cloud-config" + if isinstance(ent, str): + ent = {'content': ent} + + content = ent.get('content', '') + mtype = ent.get('type', None) + if mtype == None: + mtype = type_from_startswith(content, def_type) + + maintype, subtype = mtype.split('/', 1) + if maintype == "text": + msg = MIMEText(content, _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(content) + + if 'filename' in ent: + msg.add_header('Content-Disposition', 'attachment', + filename=ent['filename']) + + for header in ent.keys(): + if header in ('content', 'filename', 'type'): + continue + msg.add_header(header, ent['header']) + + _attach_part(appendmsg, msg) + + +def multi_part_count(outermsg, newcount=None): + """ + Return the number of attachments to this MIMEMultipart by looking + at its 'Number-Attachments' header. + """ + nfield = 'Number-Attachments' + if nfield not in outermsg: + outermsg[nfield] = "0" + + if newcount != None: + outermsg.replace_header(nfield, str(newcount)) + + return(int(outermsg.get('Number-Attachments', 0))) + + +def _attach_part(outermsg, part): + """ + Attach an part to an outer message. outermsg must be a MIMEMultipart. + Modifies a header in outermsg to keep track of number of attachments. + """ + cur = multi_part_count(outermsg) + if not part.get_filename(None): + part.add_header('Content-Disposition', 'attachment', + filename='part-%03d' % (cur + 1)) + outermsg.attach(part) + multi_part_count(outermsg, cur + 1) + + +def type_from_startswith(payload, default=None): + # slist is sorted longest first + slist = sorted(starts_with_mappings.keys(), key=lambda e: 0 - len(e)) + for sstr in slist: + if payload.startswith(sstr): + return(starts_with_mappings[sstr]) + return default + + +def process_includes(msg, appendmsg=None): + if appendmsg == None: + appendmsg = MIMEMultipart() + + for part in msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = None + ctype_orig = part.get_content_type() + + payload = part.get_payload(decode=True) + + if ctype_orig in ("text/plain", "text/x-not-multipart"): + ctype = type_from_startswith(payload) + + if ctype is None: + ctype = ctype_orig + + if ctype in ('text/x-include-url', 'text/x-include-once-url'): + do_include(payload, appendmsg) + continue + + if ctype == "text/cloud-config-archive": + explode_cc_archive(payload, appendmsg) + continue + + if 'Content-Type' in msg: + msg.replace_header('Content-Type', ctype) + else: + msg['Content-Type'] = ctype + + _attach_part(appendmsg, part) + + +def message_from_string(data, headers=None): + if headers is None: + headers = {} + if "mime-version:" in data[0:4096].lower(): + msg = email.message_from_string(data) + for (key, val) in headers.items(): + if key in msg: + msg.replace_header(key, val) + else: + msg[key] = val + else: + mtype = headers.get("Content-Type", "text/x-not-multipart") + maintype, subtype = mtype.split("/", 1) + msg = MIMEBase(maintype, subtype, *headers) + msg.set_payload(data) + + return(msg) + + +# this is heavily wasteful, reads through userdata string input +def preprocess_userdata(data): + newmsg = MIMEMultipart() + process_includes(message_from_string(decomp_str(data)), newmsg) + return(newmsg.as_string()) + + +# callback is a function that will be called with (data, content_type, +# filename, payload) +def walk_userdata(istr, callback, data=None): + partnum = 0 + for part in message_from_string(istr).walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = part.get_content_type() + if ctype is None: + ctype = 'application/octet-stream' + + filename = part.get_filename() + if not filename: + filename = 'part-%03d' % partnum + + callback(data, ctype, filename, part.get_payload(decode=True)) + + partnum = partnum + 1 + + +if __name__ == "__main__": + def main(): + import sys + data = decomp_str(file(sys.argv[1]).read()) + newmsg = MIMEMultipart() + process_includes(message_from_string(data), newmsg) + print newmsg + print "#found %s parts" % multi_part_count(newmsg) + + main() -- cgit v1.2.3