From 8900f9cba622eeaf3810003c5a6ff7522312277b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:42:38 -0700 Subject: 1. Adding some new helper files that split off file inclusion, templating, importing, constant usage. 1. Move all datasources to a new sources directory 1. Rename some files to be more consistent with python file/module naming. --- cloudinit/handlers/DataSource.py | 214 ++++++++++++++++ cloudinit/handlers/DataSourceCloudStack.py | 92 +++++++ cloudinit/handlers/DataSourceConfigDrive.py | 231 +++++++++++++++++ cloudinit/handlers/DataSourceEc2.py | 217 ++++++++++++++++ cloudinit/handlers/DataSourceMAAS.py | 345 ++++++++++++++++++++++++++ cloudinit/handlers/DataSourceNoCloud.py | 232 +++++++++++++++++ cloudinit/handlers/DataSourceOVF.py | 332 +++++++++++++++++++++++++ cloudinit/handlers/__init__.py | 274 ++++++++++++++++++++ cloudinit/handlers/cc_apt_pipelining.py | 53 ++++ cloudinit/handlers/cc_apt_update_upgrade.py | 241 ++++++++++++++++++ cloudinit/handlers/cc_bootcmd.py | 48 ++++ cloudinit/handlers/cc_byobu.py | 77 ++++++ cloudinit/handlers/cc_ca_certs.py | 90 +++++++ cloudinit/handlers/cc_chef.py | 119 +++++++++ cloudinit/handlers/cc_disable_ec2_metadata.py | 30 +++ cloudinit/handlers/cc_final_message.py | 58 +++++ cloudinit/handlers/cc_foo.py | 29 +++ cloudinit/handlers/cc_grub_dpkg.py | 64 +++++ cloudinit/handlers/cc_keys_to_console.py | 42 ++++ cloudinit/handlers/cc_landscape.py | 75 ++++++ cloudinit/handlers/cc_locale.py | 54 ++++ cloudinit/handlers/cc_mcollective.py | 99 ++++++++ cloudinit/handlers/cc_mounts.py | 179 +++++++++++++ cloudinit/handlers/cc_phone_home.py | 106 ++++++++ cloudinit/handlers/cc_puppet.py | 108 ++++++++ cloudinit/handlers/cc_resizefs.py | 108 ++++++++ cloudinit/handlers/cc_rightscale_userdata.py | 78 ++++++ cloudinit/handlers/cc_rsyslog.py | 101 ++++++++ cloudinit/handlers/cc_runcmd.py | 32 +++ cloudinit/handlers/cc_salt_minion.py | 56 +++++ cloudinit/handlers/cc_scripts_per_boot.py | 34 +++ cloudinit/handlers/cc_scripts_per_instance.py | 34 +++ cloudinit/handlers/cc_scripts_per_once.py | 34 +++ cloudinit/handlers/cc_scripts_user.py | 34 +++ cloudinit/handlers/cc_set_hostname.py | 42 ++++ cloudinit/handlers/cc_set_passwords.py | 129 ++++++++++ cloudinit/handlers/cc_ssh.py | 106 ++++++++ cloudinit/handlers/cc_ssh_import_id.py | 50 ++++ cloudinit/handlers/cc_timezone.py | 67 +++++ cloudinit/handlers/cc_update_etc_hosts.py | 87 +++++++ cloudinit/handlers/cc_update_hostname.py | 101 ++++++++ 41 files changed, 4502 insertions(+) create mode 100644 cloudinit/handlers/DataSource.py create mode 100644 cloudinit/handlers/DataSourceCloudStack.py create mode 100644 cloudinit/handlers/DataSourceConfigDrive.py create mode 100644 cloudinit/handlers/DataSourceEc2.py create mode 100644 cloudinit/handlers/DataSourceMAAS.py create mode 100644 cloudinit/handlers/DataSourceNoCloud.py create mode 100644 cloudinit/handlers/DataSourceOVF.py create mode 100644 cloudinit/handlers/__init__.py create mode 100644 cloudinit/handlers/cc_apt_pipelining.py create mode 100644 cloudinit/handlers/cc_apt_update_upgrade.py create mode 100644 cloudinit/handlers/cc_bootcmd.py create mode 100644 cloudinit/handlers/cc_byobu.py create mode 100644 cloudinit/handlers/cc_ca_certs.py create mode 100644 cloudinit/handlers/cc_chef.py create mode 100644 cloudinit/handlers/cc_disable_ec2_metadata.py create mode 100644 cloudinit/handlers/cc_final_message.py create mode 100644 cloudinit/handlers/cc_foo.py create mode 100644 cloudinit/handlers/cc_grub_dpkg.py create mode 100644 cloudinit/handlers/cc_keys_to_console.py create mode 100644 cloudinit/handlers/cc_landscape.py create mode 100644 cloudinit/handlers/cc_locale.py create mode 100644 cloudinit/handlers/cc_mcollective.py create mode 100644 cloudinit/handlers/cc_mounts.py create mode 100644 cloudinit/handlers/cc_phone_home.py create mode 100644 cloudinit/handlers/cc_puppet.py create mode 100644 cloudinit/handlers/cc_resizefs.py create mode 100644 cloudinit/handlers/cc_rightscale_userdata.py create mode 100644 cloudinit/handlers/cc_rsyslog.py create mode 100644 cloudinit/handlers/cc_runcmd.py create mode 100644 cloudinit/handlers/cc_salt_minion.py create mode 100644 cloudinit/handlers/cc_scripts_per_boot.py create mode 100644 cloudinit/handlers/cc_scripts_per_instance.py create mode 100644 cloudinit/handlers/cc_scripts_per_once.py create mode 100644 cloudinit/handlers/cc_scripts_user.py create mode 100644 cloudinit/handlers/cc_set_hostname.py create mode 100644 cloudinit/handlers/cc_set_passwords.py create mode 100644 cloudinit/handlers/cc_ssh.py create mode 100644 cloudinit/handlers/cc_ssh_import_id.py create mode 100644 cloudinit/handlers/cc_timezone.py create mode 100644 cloudinit/handlers/cc_update_etc_hosts.py create mode 100644 cloudinit/handlers/cc_update_hostname.py (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/DataSource.py b/cloudinit/handlers/DataSource.py new file mode 100644 index 00000000..e2a9150d --- /dev/null +++ b/cloudinit/handlers/DataSource.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) diff --git a/cloudinit/handlers/DataSourceCloudStack.py b/cloudinit/handlers/DataSourceCloudStack.py new file mode 100644 index 00000000..5afdf7b6 --- /dev/null +++ b/cloudinit/handlers/DataSourceCloudStack.py @@ -0,0 +1,92 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Cosmin Luta +# +# Author: Cosmin Luta +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +from socket import inet_ntoa +import time +import boto.utils as boto_utils +from struct import pack + + +class DataSourceCloudStack(DataSource.DataSource): + api_ver = 'latest' + seeddir = base_seeddir + '/cs' + metadata_address = None + + def __init__(self, sys_cfg=None): + DataSource.DataSource.__init__(self, sys_cfg) + # Cloudstack has its metadata/userdata URLs located at + # http:///latest/ + self.metadata_address = "http://%s/" % self.get_default_gateway() + + def get_default_gateway(self): + """ Returns the default gateway ip address in the dotted format + """ + with open("/proc/net/route", "r") as f: + for line in f.readlines(): + items = line.split("\t") + if items[1] == "00000000": + # found the default route, get the gateway + gw = inet_ntoa(pack(" +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +import json +import subprocess + +DEFAULT_IID = "iid-dsconfigdrive" + + +class DataSourceConfigDrive(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/config_drive' + cfg = {} + userdata_raw = None + metadata = None + dsmode = "local" + + def __str__(self): + mstr = "DataSourceConfigDrive[%s]" % self.dsmode + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = None + md = {} + ud = "" + + defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} + + if os.path.isdir(self.seeddir): + try: + (md, ud) = read_config_drive_dir(self.seeddir) + found = self.seeddir + except nonConfigDriveDir: + pass + + if not found: + dev = cfg_drive_device() + if dev: + try: + (md, ud) = util.mount_callback_umount(dev, + read_config_drive_dir) + found = dev + except (nonConfigDriveDir, util.mountFailedError): + pass + + if not found: + return False + + if 'dsconfig' in md: + self.cfg = md['dscfg'] + + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. + if 'network-interfaces' in md and self.dsmode == "local": + if md['dsmode'] == "pass": + log.info("updating network interfaces from configdrive") + else: + log.debug("updating network interfaces from configdrive") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = found + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return(self.metadata['public-keys']) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceConfigDriveNet(DataSourceConfigDrive): + dsmode = "net" + + +class nonConfigDriveDir(Exception): + pass + + +def cfg_drive_device(): + """ get the config drive device. return a string like '/dev/vdb' + or None (if there is no non-root device attached). This does not + check the contents, only reports that if there *were* a config_drive + attached, it would be this device. + per config_drive documentation, this is + "associated as the last available disk on the instance" + """ + + if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: + return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) + + # we are looking for a raw block device (sda, not sda1) with a vfat + # filesystem on it. + + letters = "abcdefghijklmnopqrstuvwxyz" + devs = util.find_devs_with("TYPE=vfat") + + # filter out anything not ending in a letter (ignore partitions) + devs = [f for f in devs if f[-1] in letters] + + # sort them in reverse so "last" device is first + devs.sort(reverse=True) + + if len(devs): + return(devs[0]) + + return(None) + + +def read_config_drive_dir(source_dir): + """ + read_config_drive_dir(source_dir): + read source_dir, and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a nonConfigDriveDir + """ + md = {} + ud = "" + + flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") + found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] + keydata = "" + + if len(found) == 0: + raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + + if "etc/network/interfaces" in found: + with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: + md['network-interfaces'] = fp.read() + + if "root/.ssh/authorized_keys" in found: + with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: + keydata = fp.read() + + meta_js = {} + + if "meta.js" in found: + content = '' + with open("%s/%s" % (source_dir, "meta.js")) as fp: + content = fp.read() + md['meta_js'] = content + try: + meta_js = json.loads(content) + except ValueError: + raise nonConfigDriveDir("%s: %s" % + (source_dir, "invalid json in meta.js")) + + keydata = meta_js.get('public-keys', keydata) + + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + for copy in ('dsmode', 'instance-id', 'dscfg'): + if copy in meta_js: + md[copy] = meta_js[copy] + + if 'user-data' in meta_js: + ud = meta_js['user-data'] + + return(md, ud) + +datasources = ( + (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + +if __name__ == "__main__": + def main(): + import sys + import pprint + print cfg_drive_device() + (md, ud) = read_config_drive_dir(sys.argv[1]) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + print(ud) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/handlers/DataSourceEc2.py b/cloudinit/handlers/DataSourceEc2.py new file mode 100644 index 00000000..7051ecda --- /dev/null +++ b/cloudinit/handlers/DataSourceEc2.py @@ -0,0 +1,217 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import socket +import time +import boto.utils as boto_utils +import os.path + + +class DataSourceEc2(DataSource.DataSource): + api_ver = '2009-04-04' + seeddir = base_seeddir + '/ec2' + metadata_address = "http://169.254.169.254" + + def __str__(self): + return("DataSourceEc2") + + def get_data(self): + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + self.userdata_raw = seedret['user-data'] + self.metadata = seedret['meta-data'] + log.debug("using seeded ec2 data in %s" % self.seeddir) + return True + + try: + if not self.wait_for_metadata_service(): + return False + start = time.time() + self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, + None, self.metadata_address) + self.metadata = boto_utils.get_instance_metadata(self.api_ver, + self.metadata_address) + log.debug("crawl of metadata service took %ds" % (time.time() - + start)) + return True + except Exception as e: + print e + return False + + def get_instance_id(self): + return(self.metadata['instance-id']) + + def get_availability_zone(self): + return(self.metadata['placement']['availability-zone']) + + def get_local_mirror(self): + return(self.get_mirror_from_availability_zone()) + + def get_mirror_from_availability_zone(self, availability_zone=None): + # availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone == None: + availability_zone = self.get_availability_zone() + + fallback = None + + if self.is_vpc(): + return fallback + + try: + host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) + return 'http://%s/ubuntu/' % host + except: + return fallback + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + + if not hasattr(mcfg, "get"): + mcfg = {} + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] + mdurls = mcfg.get("metadata_urls", def_mdurls) + + # Remove addresses from the list that wont resolve. + filtered = [x for x in mdurls if util.is_resolvable_url(x)] + + if set(filtered) != set(mdurls): + log.debug("removed the following from metadata urls: %s" % + list((set(mdurls) - set(filtered)))) + + if len(filtered): + mdurls = filtered + else: + log.warn("Empty metadata url list! using default list") + mdurls = def_mdurls + + urls = [] + url2base = {False: False} + for url in mdurls: + cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) + urls.append(cur) + url2base[cur] = url + + starttime = time.time() + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) + + if url: + log.debug("Using metadata source: '%s'" % url2base[url]) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + self.metadata_address = url2base[url] + return (bool(url)) + + def device_name_to_device(self, name): + # consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if 'block-device-mapping' not in self.metadata: + return(None) + + found = None + for entname, device in self.metadata['block-device-mapping'].items(): + if entname == name: + found = device + break + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + found = device + if found == None: + log.debug("unable to convert %s to a device" % name) + return None + + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + ofound = found + short = os.path.basename(found) + + if not found.startswith("/"): + found = "/dev/%s" % found + + if os.path.exists(found): + return(found) + + for nfrom, tlist in mappings.items(): + if not short.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short[len(nfrom):]) + if os.path.exists(cand): + log.debug("remapped device name %s => %s" % (found, cand)) + return(cand) + + # on t1.micro, ephemeral0 will appear in block-device-mapping from + # metadata, but it will not exist on disk (and never will) + # at this pint, we've verified that the path did not exist + # in the special case of 'ephemeral0' return None to avoid bogus + # fstab entry (LP: #744019) + if name == "ephemeral0": + return None + return ofound + + def is_vpc(self): + # per comment in LP: #615545 + ph = "public-hostname" + p4 = "public-ipv4" + if ((ph not in self.metadata or self.metadata[ph] == "") and + (p4 not in self.metadata or self.metadata[p4] == "")): + return True + return False + + +datasources = [ + (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceMAAS.py b/cloudinit/handlers/DataSourceMAAS.py new file mode 100644 index 00000000..61a0038f --- /dev/null +++ b/cloudinit/handlers/DataSourceMAAS.py @@ -0,0 +1,345 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import urllib2 +import time + + +MD_VERSION = "2012-03-01" + + +class DataSourceMAAS(DataSource.DataSource): + """ + DataSourceMAAS reads instance information from MAAS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMAAS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MAASSeedDirNone: + pass + except MAASSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + self.userdata_raw = userdata + self.metadata = metadata + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * local-hostname + * user-data + """ + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} + + if not os.path.isdir(seed_d): + raise MAASSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + return(check_seed_contents(md, seed_d)) + + +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * //meta-data/instance-id + * //meta-data/local-hostname + * //user-data + """ + files = ('meta-data/local-hostname', + 'meta-data/instance-id', + 'meta-data/public-keys', + 'user-data') + + base_url = "%s/%s" % (seed_url, version) + md = {} + for fname in files: + url = "%s/%s" % (base_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + try: + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MAASSeedDirMalformed or MAASSeedDirNone + """ + md_required = ('instance-id', 'local-hostname') + found = content.keys() + + if len(content) == 0: + raise MAASSeedDirNone("%s: no data files found" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content.get('user-data', "") + md = {} + for (key, val) in content.iteritems(): + if key == 'user-data': + continue + md[key] = val + + return(userdata, md) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MAASSeedDirNone(Exception): + pass + + +class MAASSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ + import argparse + import pprint + + parser = argparse.ArgumentParser(description='Interact with MAAS DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + if 'datasource' in cfg: + cfg = cfg['datasource']['MAAS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) + + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) + + main() diff --git a/cloudinit/handlers/DataSourceNoCloud.py b/cloudinit/handlers/DataSourceNoCloud.py new file mode 100644 index 00000000..e8c56b8f --- /dev/null +++ b/cloudinit/handlers/DataSourceNoCloud.py @@ -0,0 +1,232 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import subprocess + + +class DataSourceNoCloud(DataSource.DataSource): + metadata = None + userdata = None + userdata_raw = None + supported_seed_starts = ("/", "file://") + dsmode = "local" + seed = None + cmdline_id = "ds=nocloud" + seeddir = base_seeddir + '/nocloud' + + def __str__(self): + mstr = "DataSourceNoCloud" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + defaults = { + "instance-id": "nocloud", "dsmode": self.dsmode + } + + found = [] + md = {} + ud = "" + + try: + # parse the kernel command line, getting data passed in + if parse_cmdline_data(self.cmdline_id, md): + found.append("cmdline") + except: + util.logexc(log) + return False + + # check to see if the seeddir has data. + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + md = util.mergedict(md, seedret['meta-data']) + ud = seedret['user-data'] + found.append(self.seeddir) + log.debug("using seeded cache data in %s" % self.seeddir) + + # if the datasource config had a 'seedfrom' entry, then that takes + # precedence over a 'seedfrom' that was found in a filesystem + # but not over external medi + if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: + found.append("ds_config") + md["seedfrom"] = self.ds_cfg['seedfrom'] + + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label_list = util.find_devs_with("LABEL=cidata") + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + (newmd, newud) = util.mount_callback_umount(dev, + util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # for seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. + # If they want dsmode of local, then they must + # specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + log.debug("using data from %s" % dev) + found.append(dev) + break + except OSError, e: + if e.errno != errno.ENOENT: + raise + except util.mountFailedError: + log.warn("Failed to mount %s when looking for seed" % dev) + + # there was no indication on kernel cmdline or data + # in the seeddir suggesting this handler should be used. + if len(found) == 0: + return False + + seeded_interfaces = None + + # the special argument "seedfrom" indicates we should + # attempt to seed the userdata / metadata from its value + # its primarily value is in allowing the user to type less + # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg + if "seedfrom" in md: + seedfrom = md["seedfrom"] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + if 'network-interfaces' in md: + seeded_interfaces = self.dsmode + + # this could throw errors, but the user told us to do it + # so if errors are raised, let them raise + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + # values in the command line override those from the seed + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + + # update the network-interfaces if metadata had 'network-interfaces' + # entry and this is the local datasource, or 'seedfrom' was used + # and the source of the seed was self.dsmode + # ('local' for NoCloud, 'net' for NoCloudNet') + if ('network-interfaces' in md and + (self.dsmode in ("local", seeded_interfaces))): + log.info("updating network interfaces from nocloud") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + +# returns true or false indicating if cmdline indicated +# that this module should be used +# example cmdline: +# root=LABEL=uec-rootfs ro ds=nocloud +def parse_cmdline_data(ds_id, fill, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + cmdline = " %s " % cmdline + + if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): + return False + + argline = "" + # cmdline can contain: + # ds=nocloud[;key=val;key=val] + for tok in cmdline.split(): + if tok.startswith(ds_id): + argline = tok.split("=", 1) + + # argline array is now 'nocloud' followed optionally by + # a ';' and then key=value pairs also terminated with ';' + tmp = argline[1].split(";") + if len(tmp) > 1: + kvpairs = tmp[1:] + else: + kvpairs = () + + # short2long mapping to save cmdline typing + s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} + for item in kvpairs: + try: + (k, v) = item.split("=", 1) + except: + k = item + v = None + if k in s2l: + k = s2l[k] + fill[k] = v + + return(True) + + +class DataSourceNoCloudNet(DataSourceNoCloud): + cmdline_id = "ds=nocloud-net" + supported_seed_starts = ("http://", "https://", "ftp://") + seeddir = base_seeddir + '/nocloud-net' + dsmode = "net" + + +datasources = ( + (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceOVF.py b/cloudinit/handlers/DataSourceOVF.py new file mode 100644 index 00000000..a0b1b518 --- /dev/null +++ b/cloudinit/handlers/DataSourceOVF.py @@ -0,0 +1,332 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +from xml.dom import minidom +import base64 +import re +import tempfile +import subprocess + + +class DataSourceOVF(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/ovf' + environment = None + cfg = {} + userdata_raw = None + metadata = None + supported_seed_starts = ("/", "file://") + + def __str__(self): + mstr = "DataSourceOVF" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = [] + md = {} + ud = "" + + defaults = { + "instance-id": "iid-dsovf" + } + + (seedfile, contents) = get_ovf_env(base_seeddir) + if seedfile: + # found a seed dir + seed = "%s/%s" % (base_seeddir, seedfile) + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + + found.append(seed) + else: + np = {'iso': transport_iso9660, + 'vmware-guestd': transport_vmware_guestd, } + name = None + for name, transfunc in np.iteritems(): + (contents, _dev, _fname) = transfunc() + if contents: + break + + if contents: + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + found.append(name) + + # There was no OVF transports found + if len(found) == 0: + return False + + if 'seedfrom' in md and md['seedfrom']: + seedfrom = md['seedfrom'] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + self.cfg = cfg + return True + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return([self.metadata['public-keys'], ]) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceOVFNet(DataSourceOVF): + seeddir = base_seeddir + '/ovf-net' + supported_seed_starts = ("http://", "https://", "ftp://") + + +# this will return a dict with some content +# meta-data, user-data +def read_ovf_environment(contents): + props = getProperties(contents) + md = {} + cfg = {} + ud = "" + cfg_props = ['password', ] + md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] + for prop, val in props.iteritems(): + if prop == 'hostname': + prop = "local-hostname" + if prop in md_props: + md[prop] = val + elif prop in cfg_props: + cfg[prop] = val + elif prop == "user-data": + try: + ud = base64.decodestring(val) + except: + ud = val + return(md, ud, cfg) + + +# returns tuple of filename (in 'dirname', and the contents of the file) +# on "not found", returns 'None' for filename and False for contents +def get_ovf_env(dirname): + env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") + for fname in env_names: + if os.path.isfile("%s/%s" % (dirname, fname)): + fp = open("%s/%s" % (dirname, fname)) + contents = fp.read() + fp.close() + return(fname, contents) + return(None, False) + + +# transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + + # default_regex matches values in + # /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + envname = "CLOUD_INIT_CDROM_DEV_REGEX" + default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + + devname_regex = os.environ.get(envname, default_regex) + cdmatch = re.compile(devname_regex) + + # go through mounts to see if it was already mounted + fp = open("/proc/mounts") + mounts = fp.readlines() + fp.close() + + mounted = {} + for mpline in mounts: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mounted[dev] = (dev, fstype, mp, False) + mp = mp.replace("\\040", " ") + if fstype != "iso9660" and require_iso: + continue + + if cdmatch.match(dev[5:]) == None: # take off '/dev/' + continue + + (fname, contents) = get_ovf_env(mp) + if contents is not False: + return(contents, dev, fname) + + tmpd = None + dvnull = None + + devs = os.listdir("/dev/") + devs.sort() + + for dev in devs: + fullp = "/dev/%s" % dev + + if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + continue + + fp = None + try: + fp = open(fullp, "rb") + fp.read(512) + fp.close() + except: + if fp: + fp.close() + continue + + if tmpd is None: + tmpd = tempfile.mkdtemp() + if dvnull is None: + try: + dvnull = open("/dev/null") + except: + pass + + cmd = ["mount", "-o", "ro", fullp, tmpd] + if require_iso: + cmd.extend(('-t', 'iso9660')) + + rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) + if rc: + continue + + (fname, contents) = get_ovf_env(tmpd) + + subprocess.call(["umount", tmpd]) + + if contents is not False: + os.rmdir(tmpd) + return(contents, fullp, fname) + + if tmpd: + os.rmdir(tmpd) + + if dvnull: + dvnull.close() + + return(False, None, None) + + +def transport_vmware_guestd(): + # http://blogs.vmware.com/vapp/2009/07/ \ + # selfconfiguration-and-the-ovf-environment.html + # try: + # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] + # (out, err) = subp(cmd) + # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') + # except: + # # would need to error check here and see why this failed + # # to know if log/error should be raised + # return(False, None, None) + return(False, None, None) + + +def findChild(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return(ret) + + +def getProperties(environString): + dom = minidom.parseString(environString) + if dom.documentElement.localName != "Environment": + raise Exception("No Environment Node") + + if not dom.documentElement.hasChildNodes(): + raise Exception("No Child Nodes") + + envNsURI = "http://schemas.dmtf.org/ovf/environment/1" + + # could also check here that elem.namespaceURI == + # "http://schemas.dmtf.org/ovf/environment/1" + propSections = findChild(dom.documentElement, + lambda n: n.localName == "PropertySection") + + if len(propSections) == 0: + raise Exception("No 'PropertySection's") + + props = {} + propElems = findChild(propSections[0], lambda n: n.localName == "Property") + + for elem in propElems: + key = elem.attributes.getNamedItemNS(envNsURI, "key").value + val = elem.attributes.getNamedItemNS(envNsURI, "value").value + props[key] = val + + return(props) + + +datasources = ( + (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), + (DataSourceOVFNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + envStr = open(sys.argv[1]).read() + props = getProperties(envStr) + import pprint + pprint.pprint(props) + + md, ud, cfg = read_ovf_environment(envStr) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + pprint.pprint(ud) + print "=== cfg ===" + pprint.pprint(cfg) + + main() diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py new file mode 100644 index 00000000..a16bdde6 --- /dev/null +++ b/cloudinit/handlers/__init__.py @@ -0,0 +1,274 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2008-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Chuck Short +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import yaml +import cloudinit +import cloudinit.util as util +import sys +import traceback +import os +import subprocess +import time + +per_instance = cloudinit.per_instance +per_always = cloudinit.per_always +per_once = cloudinit.per_once + + +class CloudConfig(): + cfgfile = None + cfg = None + + def __init__(self, cfgfile, cloud=None, ds_deps=None): + if cloud == None: + self.cloud = cloudinit.CloudInit(ds_deps) + self.cloud.get_data_source() + else: + self.cloud = cloud + self.cfg = self.get_config_obj(cfgfile) + + def get_config_obj(self, cfgfile): + try: + cfg = util.read_conf(cfgfile) + except: + # TODO: this 'log' could/should be passed in + cloudinit.log.critical("Failed loading of cloud config '%s'. " + "Continuing with empty config\n" % cfgfile) + cloudinit.log.debug(traceback.format_exc() + "\n") + cfg = None + if cfg is None: + cfg = {} + + try: + ds_cfg = self.cloud.datasource.get_config_obj() + except: + ds_cfg = {} + + cfg = util.mergedict(cfg, ds_cfg) + return(util.mergedict(cfg, self.cloud.cfg)) + + def handle(self, name, args, freq=None): + try: + mod = __import__("cc_" + name.replace("-", "_"), globals()) + def_freq = getattr(mod, "frequency", per_instance) + handler = getattr(mod, "handle") + + if not freq: + freq = def_freq + + self.cloud.sem_and_run("config-" + name, freq, handler, + [name, self.cfg, self.cloud, cloudinit.log, args]) + except: + raise + + +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg, name): + if name not in cfg: + return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item, str): + module_list.append((item,)) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + + +def run_cc_modules(cc, module_list, log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args)) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name, freq, run_args)) + failures.append(name) + + return(failures) + + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [None, None] + if not 'output' in cfg: + return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: + return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout and stderr + if isinstance(modecfg, str): + ret = [modecfg, modecfg] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": + ret[1] = ret[0] + + swlist = [">>", ">", "|"] + for i in range(len(ret)): + if not ret[i]: + continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s, val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % (">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + +def run_per_instance(name, func, args, clear_on_fail=False): + semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) + if os.path.exists(semfile): + return + + util.write_file(semfile, str(time.time())) + try: + func(*args) + except: + if clear_on_fail: + os.unlink(semfile) + raise + + +# apt_get top level command (install, update...), and args to pass it +def apt_get(tlc, args=None): + if args is None: + args = [] + e = os.environ.copy() + e['DEBIAN_FRONTEND'] = 'noninteractive' + cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', + '--assume-yes', tlc] + cmd.extend(args) + subprocess.check_call(cmd, env=e) + + +def update_package_sources(): + run_per_instance("update-sources", apt_get, ("update",)) + + +def install_packages(pkglist): + update_package_sources() + apt_get("install", pkglist) diff --git a/cloudinit/handlers/cc_apt_pipelining.py b/cloudinit/handlers/cc_apt_pipelining.py new file mode 100644 index 00000000..0286a9ae --- /dev/null +++ b/cloudinit/handlers/cc_apt_pipelining.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" + + +def handle(_name, cfg, _cloud, log, _args): + + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() + + if apt_pipe_value == "false": + write_apt_snippet("0", log) + + elif apt_pipe_value in ("none", "unchanged", "os"): + return + + elif apt_pipe_value in str(range(0, 6)): + write_apt_snippet(apt_pipe_value, log) + + else: + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ Writes f_name with apt pipeline depth 'setting' """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) + + util.write_file(f_name, file_contents) + + log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/handlers/cc_apt_update_upgrade.py b/cloudinit/handlers/cc_apt_update_upgrade.py new file mode 100644 index 00000000..a7049bce --- /dev/null +++ b/cloudinit/handlers/cc_apt_update_upgrade.py @@ -0,0 +1,241 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback +import os +import glob +import cloudinit.CloudConfig as cc + + +def handle(_name, cfg, cloud, log, _args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + release = get_release() + + mirror = find_apt_mirror(cloud, cfg) + + log.debug("selected mirror at: %s" % mirror) + + if not util.get_cfg_option_bool(cfg, \ + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy \"%s\";\n" + with open(proxy_filename, "w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + + # process 'apt_sources' + if 'apt_sources' in cfg: + errors = add_sources(cfg['apt_sources'], + {'MIRROR': mirror, 'RELEASE': release}) + for e in errors: + log.warn("Source Error: %s\n" % ':'.join(e)) + + dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) + if dconf_sel: + log.debug("setting debconf selections per cloud config") + try: + util.subp(('debconf-set-selections', '-'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections") + log.debug(traceback.format_exc()) + + pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cc.update_package_sources() + except subprocess.CalledProcessError as e: + log.warn("apt-get update failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if upgrade: + try: + cc.apt_get("upgrade") + except subprocess.CalledProcessError as e: + log.warn("apt upgrade failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if len(pkglist): + try: + cc.install_packages(pkglist) + except subprocess.CalledProcessError as e: + log.warn("Failed to install packages: %s " % pkglist) + log.debug(traceback.format_exc()) + errors.append(e) + + if len(errors): + raise errors[0] + + return(True) + + +def mirror2lists_fileprefix(mirror): + string = mirror + # take of http:// or ftp:// + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if(oprefix == nprefix): + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + + +def get_release(): + stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], + stdout=subprocess.PIPE).communicate() + return(str(stdout).strip()) + + +def generate_sources_list(codename, mirror): + util.render_to_file('sources.list', '/etc/apt/sources.list', \ + {'mirror': mirror, 'codename': codename}) + + +def add_sources(srclist, searchList=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if searchList is None: + searchList = {} + elst = [] + + for ent in srclist: + if 'source' not in ent: + elst.append(["", "missing source"]) + continue + + source = ent['source'] + if source.startswith("ppa:"): + try: + util.subp(["add-apt-repository", source]) + except: + elst.append([source, "add-apt-repository failed"]) + continue + + source = util.render_string(source, searchList) + + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = "%s/%s" % \ + ("/etc/apt/sources.list.d/", ent['filename']) + + if ('keyid' in ent and 'key' not in ent): + ks = "keyserver.ubuntu.com" + if 'keyserver' in ent: + ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + elst.append([source, "failed to get key from %s" % ks]) + continue + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + elst.append([source, "failed add key"]) + + try: + util.write_file(ent['filename'], source + "\n", omode="ab") + except: + elst.append([source, "failed write to file %s" % ent['filename']]) + + return(elst) + + +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + cfg_mirror = cfg.get("apt_mirror", None) + if cfg_mirror: + mirror = cfg["apt_mirror"] + elif "apt_mirror_search" in cfg: + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + if not mirror: + doms.extend((".localdomain", "",)) + + mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % post) + + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/handlers/cc_bootcmd.py b/cloudinit/handlers/cc_bootcmd.py new file mode 100644 index 00000000..f584da02 --- /dev/null +++ b/cloudinit/handlers/cc_bootcmd.py @@ -0,0 +1,48 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +import tempfile +import os +from cloudinit.CloudConfig import per_always +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if "bootcmd" not in cfg: + return + + try: + content = util.shellify(cfg["bootcmd"]) + tmpf = tempfile.TemporaryFile() + tmpf.write(content) + tmpf.seek(0) + except: + log.warn("failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) + tmpf.close() + except: + log.warn("failed to run commands from bootcmd") + raise diff --git a/cloudinit/handlers/cc_byobu.py b/cloudinit/handlers/cc_byobu.py new file mode 100644 index 00000000..e821b261 --- /dev/null +++ b/cloudinit/handlers/cc_byobu.py @@ -0,0 +1,77 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + value = args[0] + else: + value = util.get_cfg_option_str(cfg, "byobu_by_default", "") + + if not value: + return + + if value == "user" or value == "system": + value = "enable-%s" % value + + valid = ("enable-user", "enable-system", "enable", + "disable-user", "disable-system", "disable") + if not value in valid: + log.warn("Unknown value %s for byobu_by_default" % value) + + mod_user = value.endswith("-user") + mod_sys = value.endswith("-system") + if value.startswith("enable"): + bl_inst = "install" + dc_val = "byobu byobu/launch-by-default boolean true" + mod_sys = True + else: + if value == "disable": + mod_user = True + mod_sys = True + bl_inst = "uninstall" + dc_val = "byobu byobu/launch-by-default boolean false" + + shcmd = "" + if mod_user: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += " || X=$(($X+1)); " + if mod_sys: + shcmd += "echo \"%s\" | debconf-set-selections" % dc_val + shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" + shcmd += " || X=$(($X+1)); " + + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] + + log.debug("setting byobu to %s" % value) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_ca_certs.py b/cloudinit/handlers/cc_ca_certs.py new file mode 100644 index 00000000..3af6238a --- /dev/null +++ b/cloudinit/handlers/cc_ca_certs.py @@ -0,0 +1,90 @@ +# vi: ts=4 expandtab +# +# Author: Mike Milner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +from subprocess import check_call +from cloudinit.util import (write_file, get_cfg_option_list_or_str, + delete_dir_contents, subp) + +CA_CERT_PATH = "/usr/share/ca-certificates/" +CA_CERT_FILENAME = "cloud-init-ca-certs.crt" +CA_CERT_CONFIG = "/etc/ca-certificates.conf" +CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" + + +def update_ca_certs(): + """ + Updates the CA certificate cache on the current machine. + """ + check_call(["update-ca-certificates"]) + + +def add_ca_certs(certs): + """ + Adds certificates to the system. To actually apply the new certificates + you must also call L{update_ca_certs}. + + @param certs: A list of certificate strings. + """ + if certs: + cert_file_contents = "\n".join(certs) + cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) + write_file(cert_file_fullpath, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. + write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + + +def remove_default_ca_certs(): + """ + Removes all default trusted CA certificates from the system. To actually + apply the change you must also call L{update_ca_certs}. + """ + delete_dir_contents(CA_CERT_PATH) + delete_dir_contents(CA_CERT_SYSTEM_PATH) + write_file(CA_CERT_CONFIG, "", mode=0644) + debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" + subp(('debconf-set-selections', '-'), debconf_sel) + + +def handle(_name, cfg, _cloud, log, _args): + """ + Call to handle ca-cert sections in cloud-config file. + + @param name: The module name "ca-cert" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + # If there isn't a ca-certs section in the configuration don't do anything + if "ca-certs" not in cfg: + return + ca_cert_cfg = cfg['ca-certs'] + + # If there is a remove-defaults option set to true, remove the system + # default trusted CA certs first. + if ca_cert_cfg.get("remove-defaults", False): + log.debug("removing default certificates") + remove_default_ca_certs() + + # If we are given any new trusted CA certs to add, add them. + if "trusted" in ca_cert_cfg: + trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + if trusted_certs: + log.debug("adding %d certificates" % len(trusted_certs)) + add_ca_certs(trusted_certs) + + # Update the system with the new cert configuration. + update_ca_certs() diff --git a/cloudinit/handlers/cc_chef.py b/cloudinit/handlers/cc_chef.py new file mode 100644 index 00000000..941e04fe --- /dev/null +++ b/cloudinit/handlers/cc_chef.py @@ -0,0 +1,119 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Avishai Ish-Shalom +# Author: Mike Moulton +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import json +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +ruby_version_default = "1.8" + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything + if 'chef' not in cfg: + return + chef_cfg = cfg['chef'] + + # ensure the chef directories we use exist + mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + + # set the validation key based on the presence of either 'validation_key' + # or 'validation_cert'. In the case where both exist, 'validation_key' + # takes precedence + for key in ('validation_key', 'validation_cert'): + if key in chef_cfg and chef_cfg[key]: + with open('/etc/chef/validation.pem', 'w') as validation_key_fh: + validation_key_fh.write(chef_cfg[key]) + break + + # create the chef config from template + util.render_to_file('chef_client.rb', '/etc/chef/client.rb', + {'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name']}) + + # set the firstboot json + with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + firstboot_json_fh.write(json.dumps(initial_json)) + + # If chef is not installed, we install chef based on 'install_type' + if not os.path.isfile('/usr/bin/chef-client'): + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # this will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + ruby_version_default) + install_chef_from_gems(ruby_version, chef_version) + # and finally, run chef-client + log.debug('running chef-client') + subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', + '-s', '20']) + else: + # this will install and run the chef-client from packages + cc.install_packages(('chef',)) + + +def get_ruby_packages(version): + # return a list of packages needed to install ruby at version + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + if version == "1.8": + pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) + return(pkgs) + + +def install_chef_from_gems(ruby_version, chef_version=None): + cc.install_packages(get_ruby_packages(ruby_version)) + if not os.path.exists('/usr/bin/gem'): + os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + if not os.path.exists('/usr/bin/ruby'): + os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + if chef_version: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) + else: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) + + +def ensure_dir(d): + if not os.path.exists(d): + os.makedirs(d) + + +def mkdirs(dirs): + for d in dirs: + ensure_dir(d) diff --git a/cloudinit/handlers/cc_disable_ec2_metadata.py b/cloudinit/handlers/cc_disable_ec2_metadata.py new file mode 100644 index 00000000..6b31ea8e --- /dev/null +++ b/cloudinit/handlers/cc_disable_ec2_metadata.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, _log, _args): + if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + fwall = "route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) diff --git a/cloudinit/handlers/cc_final_message.py b/cloudinit/handlers/cc_final_message.py new file mode 100644 index 00000000..abb4ca32 --- /dev/null +++ b/cloudinit/handlers/cc_final_message.py @@ -0,0 +1,58 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + + try: + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) + except: + ts = "na" + + try: + subs = {'UPTIME': uptime, 'TIMESTAMP': ts} + sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/handlers/cc_foo.py b/cloudinit/handlers/cc_foo.py new file mode 100644 index 00000000..35ec3fa7 --- /dev/null +++ b/cloudinit/handlers/cc_foo.py @@ -0,0 +1,29 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#import cloudinit +#import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance + + +def handle(_name, _cfg, _cloud, _log, _args): + print "hi" diff --git a/cloudinit/handlers/cc_grub_dpkg.py b/cloudinit/handlers/cc_grub_dpkg.py new file mode 100644 index 00000000..9f3a7eaf --- /dev/null +++ b/cloudinit/handlers/cc_grub_dpkg.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import traceback +import os + + +def handle(_name, cfg, _cloud, log, _args): + idevs = None + idevs_empty = None + + if "grub-dpkg" in cfg: + idevs = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices_empty", None) + + if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + if idevs == None: + idevs = "" + if idevs_empty == None: + idevs_empty = "true" + else: + if idevs_empty == None: + idevs_empty = "false" + if idevs == None: + idevs = "/dev/sda" + for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + if os.path.exists(dev): + idevs = dev + break + + # now idevs and idevs_empty are set to determined values + # or, those set by user + + dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ + "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty + log.debug("setting grub debconf-set-selections with '%s','%s'" % + (idevs, idevs_empty)) + + try: + util.subp(('debconf-set-selections'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections for grub-dpkg") + log.debug(traceback.format_exc()) diff --git a/cloudinit/handlers/cc_keys_to_console.py b/cloudinit/handlers/cc_keys_to_console.py new file mode 100644 index 00000000..73a477c0 --- /dev/null +++ b/cloudinit/handlers/cc_keys_to_console.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +import subprocess + +frequency = per_instance + + +def handle(_name, cfg, _cloud, log, _args): + cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] + fp_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_fp_console_blacklist", []) + key_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_key_console_blacklist", ["ssh-dss"]) + try: + confp = open('/dev/console', "wb") + cmd.append(','.join(fp_blacklist)) + cmd.append(','.join(key_blacklist)) + subprocess.call(cmd, stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/handlers/cc_landscape.py b/cloudinit/handlers/cc_landscape.py new file mode 100644 index 00000000..a4113cbe --- /dev/null +++ b/cloudinit/handlers/cc_landscape.py @@ -0,0 +1,75 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +from cloudinit.CloudConfig import per_instance +from configobj import ConfigObj + +frequency = per_instance + +lsc_client_cfg_file = "/etc/landscape/client.conf" + +# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 +lsc_builtincfg = { + 'client': { + 'log_level': "info", + 'url': "https://landscape.canonical.com/message-system", + 'ping_url': "http://landscape.canonical.com/ping", + 'data_path': "/var/lib/landscape/client", + } +} + + +def handle(_name, cfg, _cloud, log, _args): + """ + Basically turn a top level 'landscape' entry with a 'client' dict + and render it to ConfigObj format under '[client]' section in + /etc/landscape/client.conf + """ + + ls_cloudcfg = cfg.get("landscape", {}) + + if not isinstance(ls_cloudcfg, dict): + raise(Exception("'landscape' existed in config, but not a dict")) + + merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + + if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): + os.makedirs(os.path.dirname(lsc_client_cfg_file)) + + with open(lsc_client_cfg_file, "w") as fp: + merged.write(fp) + + log.debug("updated %s" % lsc_client_cfg_file) + + +def mergeTogether(objs): + """ + merge together ConfigObj objects or things that ConfigObj() will take in + later entries override earlier + """ + cfg = ConfigObj({}) + for obj in objs: + if isinstance(obj, ConfigObj): + cfg.merge(obj) + else: + cfg.merge(ConfigObj(obj)) + return cfg diff --git a/cloudinit/handlers/cc_locale.py b/cloudinit/handlers/cc_locale.py new file mode 100644 index 00000000..2bb22fdb --- /dev/null +++ b/cloudinit/handlers/cc_locale.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os.path +import subprocess +import traceback + + +def apply_locale(locale, cfgfile): + if os.path.exists('/usr/sbin/locale-gen'): + subprocess.Popen(['locale-gen', locale]).communicate() + if os.path.exists('/usr/sbin/update-locale'): + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', cfgfile, {'locale': locale}) + + +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) + + locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", + "/etc/default/locale") + + if not locale: + return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale, locale_cfgfile) + except Exception as e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/handlers/cc_mcollective.py b/cloudinit/handlers/cc_mcollective.py new file mode 100644 index 00000000..a2a6230c --- /dev/null +++ b/cloudinit/handlers/cc_mcollective.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Marc Cluet +# Based on code by Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +pubcert_file = "/etc/mcollective/ssl/server-public.pem" +pricert_file = "/etc/mcollective/ssl/server-private.pem" + + +# Our fake header section +class FakeSecHead(object): + def __init__(self, fp): + self.fp = fp + self.sechead = '[nullsection]\n' + + def readline(self): + if self.sechead: + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a mcollective key in the configuration don't do anything + if 'mcollective' not in cfg: + return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... + cc.install_packages(("mcollective",)) + + # ... and then update the mcollective configuration + if 'conf' in mcollective_cfg: + # Create object for reading server.cfg values + mcollective_config = ConfigParser.ConfigParser() + # Read server.cfg values from original file in order to be able to mix + # the rest up + mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' + 'server.cfg'))) + for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + if cfg_name == 'public-cert': + util.write_file(pubcert_file, cfg, mode=0644) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_public', pubcert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + elif cfg_name == 'private-cert': + util.write_file(pricert_file, cfg, mode=0600) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_private', pricert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + else: + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + mcollective_config.set(cfg_name, o, v) + # We got all our config as wanted we'll rename + # the previous server.cfg and create our new one + os.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + outputfile = StringIO.StringIO() + mcollective_config.write(outputfile) + # Now we got the whole file, write to disk except first line + # Note below, that we've just used ConfigParser because it generally + # works. Below, we remove the initial 'nullsection' header + # and then change 'key = value' to 'key: value'. The global + # search and replace of '=' with ':' could be problematic though. + # this most likely needs fixing. + util.write_file('/etc/mcollective/server.cfg', + outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', + ':'), + mode=0644) + + # Start mcollective + subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/handlers/cc_mounts.py b/cloudinit/handlers/cc_mounts.py new file mode 100644 index 00000000..6cdd74e8 --- /dev/null +++ b/cloudinit/handlers/cc_mounts.py @@ -0,0 +1,179 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os +import re +from string import whitespace # pylint: disable=W0402 + + +def is_mdname(name): + # return true if this is a metadata service name + if name in ["ami", "root", "swap"]: + return True + # names 'ephemeral0' or 'ephemeral1' + # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' + for enumname in ("ephemeral", "ebs"): + if name.startswith(enumname) and name.find(":") == -1: + return True + return False + + +def handle(_name, cfg, cloud, log, _args): + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + defvals = cfg.get("mount_default_fields", defvals) + + # these are our default set of mounts + defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"]] + + cfgmnt = [] + if "mounts" in cfg: + cfgmnt = cfg["mounts"] + + # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 + shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" + shortname = re.compile(shortname_filter) + + for i in range(len(cfgmnt)): + # skip something that wasn't a list + if not isinstance(cfgmnt[i], list): + continue + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + if cfgmnt[i][0] == "ephemeral": + cfgmnt[i][0] = "ephemeral0" + + if is_mdname(cfgmnt[i][0]): + newname = cloud.device_name_to_device(cfgmnt[i][0]) + if not newname: + log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + cfgmnt[i][1] = None + else: + if newname.startswith("/"): + cfgmnt[i][0] = newname + else: + cfgmnt[i][0] = "/dev/%s" % newname + else: + if shortname.match(cfgmnt[i][0]): + cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + + # in case the user did not quote a field (likely fs-freq, fs_passno) + # but do not convert None to 'None' (LP: #898365) + for j in range(len(cfgmnt[i])): + if isinstance(cfgmnt[i][j], int): + cfgmnt[i][j] = str(cfgmnt[i][j]) + + for i in range(len(cfgmnt)): + # fill in values with defaults from defvals above + for j in range(len(defvals)): + if len(cfgmnt[i]) <= j: + cfgmnt[i].append(defvals[j]) + elif cfgmnt[i][j] is None: + cfgmnt[i][j] = defvals[j] + + # if the second entry in the list is 'None' this + # clears all previous entries of that same 'fs_spec' + # (fs_spec is the first field in /etc/fstab, ie, that device) + if cfgmnt[i][1] is None: + for j in range(i): + if cfgmnt[j][0] == cfgmnt[i][0]: + cfgmnt[j][1] = None + + # for each of the "default" mounts, add them only if no other + # entry has the same device name + for defmnt in defmnts: + devname = cloud.device_name_to_device(defmnt[0]) + if devname is None: + continue + if devname.startswith("/"): + defmnt[0] = devname + else: + defmnt[0] = "/dev/%s" % devname + + cfgmnt_has = False + for cfgm in cfgmnt: + if cfgm[0] == defmnt[0]: + cfgmnt_has = True + break + + if cfgmnt_has: + continue + cfgmnt.append(defmnt) + + # now, each entry in the cfgmnt list has all fstab values + # if the second field is None (not the string, the value) we skip it + actlist = [x for x in cfgmnt if x[1] is not None] + + if len(actlist) == 0: + return + + comment = "comment=cloudconfig" + cc_lines = [] + needswap = False + dirs = [] + for line in actlist: + # write 'comment' in the fs_mntops, entry, claiming this + line[3] = "%s,comment=cloudconfig" % line[3] + if line[2] == "swap": + needswap = True + if line[1].startswith("/"): + dirs.append(line[1]) + cc_lines.append('\t'.join(line)) + + fstab_lines = [] + fstab = open("/etc/fstab", "r+") + ws = re.compile("[%s]+" % whitespace) + for line in fstab.read().splitlines(): + try: + toks = ws.split(line) + if toks[3].find(comment) != -1: + continue + except: + pass + fstab_lines.append(line) + + fstab_lines.extend(cc_lines) + + fstab.seek(0) + fstab.write("%s\n" % '\n'.join(fstab_lines)) + fstab.truncate() + fstab.close() + + if needswap: + try: + util.subp(("swapon", "-a")) + except: + log.warn("Failed to enable swap") + + for d in dirs: + if os.path.exists(d): + continue + try: + os.makedirs(d) + except: + log.warn("Failed to make '%s' config-mount\n", d) + + try: + util.subp(("mount", "-a")) + except: + log.warn("'mount -a' failed") diff --git a/cloudinit/handlers/cc_phone_home.py b/cloudinit/handlers/cc_phone_home.py new file mode 100644 index 00000000..a7ff74e1 --- /dev/null +++ b/cloudinit/handlers/cc_phone_home.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', + 'hostname'] + + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id +# +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + ph_cfg = util.read_conf(args[0]) + else: + if not 'phone_home' in cfg: + return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries', 10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = {} + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = {} + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + + null_exc = object() + last_e = null_exc + for i in range(0, tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i + 1)) + return + except Exception as e: + log.debug("failed to post to %s on try %i" % (url, i + 1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e is not null_exc: + raise(last_e) + + return diff --git a/cloudinit/handlers/cc_puppet.py b/cloudinit/handlers/cc_puppet.py new file mode 100644 index 00000000..6fc475f6 --- /dev/null +++ b/cloudinit/handlers/cc_puppet.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import pwd +import socket +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a puppet key in the configuration don't do anything + if 'puppet' not in cfg: + return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... + cc.install_packages(("puppet",)) + + # ... and then update the puppet configuration + if 'conf' in puppet_cfg: + # Add all sections from the conf object to puppet.conf + puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + # Create object for reading puppet.conf values + puppet_config = ConfigParser.ConfigParser() + # Read puppet.conf values from original file in order to be able to + # mix the rest up + puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in + puppet_conf_fh.readlines()))) + # Close original file, no longer needed + puppet_conf_fh.close() + for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # ca_cert configuration is a special case + # Dump the puppetmaster ca certificate in the correct place + if cfg_name == 'ca_cert': + # Puppet ssl sub-directory isn't created yet + # Create it with the proper permissions and ownership + os.makedirs('/var/lib/puppet/ssl') + os.chmod('/var/lib/puppet/ssl', 0771) + os.chown('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + os.makedirs('/var/lib/puppet/ssl/certs/') + os.chown('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') + ca_fh.write(cfg) + ca_fh.close() + os.chown('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) + util.restorecon_if_possible('/var/lib/puppet', recursive=True) + else: + #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) + # If puppet.conf already has this section we don't want to + # write it again + if puppet_config.has_section(cfg_name) == False: + puppet_config.add_section(cfg_name) + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + if o == 'certname': + # Expand %f as the fqdn + v = v.replace("%f", socket.getfqdn()) + # Expand %i as the instance id + v = v.replace("%i", + cloud.datasource.get_instance_id()) + # certname needs to be downcase + v = v.lower() + puppet_config.set(cfg_name, o, v) + #puppet_conf_fh.write("%s=%s\n" % (o, v)) + # We got all our config as wanted we'll rename + # the previous puppet.conf and create our new one + os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + with open('/etc/puppet/puppet.conf', 'wb') as configfile: + puppet_config.write(configfile) + util.restorecon_if_possible('/etc/puppet/puppet.conf') + # Set puppet to automatically start + if os.path.exists('/etc/default/puppet'): + subprocess.check_call(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet']) + elif os.path.exists('/bin/systemctl'): + subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + elif os.path.exists('/sbin/chkconfig'): + subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + else: + log.warn("Do not know how to enable puppet service on this system") + # Start puppetd + subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/handlers/cc_resizefs.py b/cloudinit/handlers/cc_resizefs.py new file mode 100644 index 00000000..2dc66def --- /dev/null +++ b/cloudinit/handlers/cc_resizefs.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import os +import stat +import sys +import time +import tempfile +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = False + if str(args[0]).lower() in ['true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if str(resize_root).lower() in ['false', '0']: + return + + # we use mktemp rather than mkstemp because early in boot nothing + # else should be able to race us for this, and we need to mknod. + devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") + + try: + st_dev = os.stat("/").st_dev + dev = os.makedev(os.major(st_dev), os.minor(st_dev)) + os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + except: + if util.is_container(): + log.debug("inside container, ignoring mknod failure in resizefs") + return + log.warn("Failed to make device node to resize /") + raise + + cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + try: + (fstype, _err) = util.subp(cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % + (os.major(st_dev), os.minor(st_dev), cmd)) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + if str(fstype).startswith("ext"): + resize_cmd = ['resize2fs', devpth] + elif fstype == "xfs": + resize_cmd = ['xfs_growfs', devpth] + else: + os.unlink(devpth) + log.debug("not resizing unknown filesystem %s" % fstype) + return + + if resize_root == "noblock": + fid = os.fork() + if fid == 0: + try: + do_resize(resize_cmd, devpth, log) + os._exit(0) # pylint: disable=W0212 + except Exception as exc: + sys.stderr.write("Failed: %s" % exc) + os._exit(1) # pylint: disable=W0212 + else: + do_resize(resize_cmd, devpth, log) + + log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % + (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), + resize_root)) + + return + + +def do_resize(resize_cmd, devpth, log): + try: + start = time.time() + util.subp(resize_cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to resize filesystem (%s)" % resize_cmd) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + os.unlink(devpth) + log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/handlers/cc_rightscale_userdata.py b/cloudinit/handlers/cc_rightscale_userdata.py new file mode 100644 index 00000000..5ed0848f --- /dev/null +++ b/cloudinit/handlers/cc_rightscale_userdata.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + + +def handle(_name, _cfg, cloud, log, _args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(ud) + if not my_hookname in mdict: + return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d, i) + i = i + 1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception as e: + if not first_e: + first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/handlers/cc_rsyslog.py b/cloudinit/handlers/cc_rsyslog.py new file mode 100644 index 00000000..ac7f2c74 --- /dev/null +++ b/cloudinit/handlers/cc_rsyslog.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit +import logging +import cloudinit.util as util +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + + +def handle(_name, cfg, _cloud, log, _args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: + return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + files = [] + elst = [] + for ent in cfg['rsyslog']: + if isinstance(ent, dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir, filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception as e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception as e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/handlers/cc_runcmd.py b/cloudinit/handlers/cc_runcmd.py new file mode 100644 index 00000000..f7e8c671 --- /dev/null +++ b/cloudinit/handlers/cc_runcmd.py @@ -0,0 +1,32 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if "runcmd" not in cfg: + return + outfile = "%s/runcmd" % cloud.get_ipath('scripts') + try: + content = util.shellify(cfg["runcmd"]) + util.write_file(outfile, content, 0700) + except: + log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/handlers/cc_salt_minion.py b/cloudinit/handlers/cc_salt_minion.py new file mode 100644 index 00000000..1a3b5039 --- /dev/null +++ b/cloudinit/handlers/cc_salt_minion.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Author: Jeff Bauer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import subprocess +import cloudinit.CloudConfig as cc +import yaml + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a salt key in the configuration don't do anything + if 'salt_minion' not in cfg: + return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... + cc.install_packages(("salt",)) + config_dir = '/etc/salt' + if not os.path.isdir(config_dir): + os.makedirs(config_dir) + # ... and then update the salt configuration + if 'conf' in salt_cfg: + # Add all sections from the conf object to /etc/salt/minion + minion_config = os.path.join(config_dir, 'minion') + yaml.dump(salt_cfg['conf'], + file(minion_config, 'w'), + default_flow_style=False) + # ... copy the key pair if specified + if 'public_key' in salt_cfg and 'private_key' in salt_cfg: + pki_dir = '/etc/salt/pki' + cumask = os.umask(077) + if not os.path.isdir(pki_dir): + os.makedirs(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + with open(pub_name, 'w') as f: + f.write(salt_cfg['public_key']) + with open(pem_name, 'w') as f: + f.write(salt_cfg['private_key']) + os.umask(cumask) + + # Start salt-minion + subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/handlers/cc_scripts_per_boot.py b/cloudinit/handlers/cc_scripts_per_boot.py new file mode 100644 index 00000000..41a74754 --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_boot.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +from cloudinit import get_cpath + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_per_instance.py b/cloudinit/handlers/cc_scripts_per_instance.py new file mode 100644 index 00000000..a2981eab --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_instance.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_cpath + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_per_once.py b/cloudinit/handlers/cc_scripts_per_once.py new file mode 100644 index 00000000..a69151da --- /dev/null +++ b/cloudinit/handlers/cc_scripts_per_once.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_once +from cloudinit import get_cpath + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_scripts_user.py b/cloudinit/handlers/cc_scripts_user.py new file mode 100644 index 00000000..933aa4e0 --- /dev/null +++ b/cloudinit/handlers/cc_scripts_user.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/handlers/cc_set_hostname.py b/cloudinit/handlers/cc_set_hostname.py new file mode 100644 index 00000000..acea74d9 --- /dev/null +++ b/cloudinit/handlers/cc_set_hostname.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + set_hostname(hostname, log) + except Exception: + util.logexc(log) + log.warn("failed to set hostname to %s\n", hostname) + + return(True) + + +def set_hostname(hostname, log): + util.subp(['hostname', hostname]) + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/handlers/cc_set_passwords.py b/cloudinit/handlers/cc_set_passwords.py new file mode 100644 index 00000000..9d0bbdb8 --- /dev/null +++ b/cloudinit/handlers/cc_set_passwords.py @@ -0,0 +1,129 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import sys +import random +from string import letters, digits # pylint: disable=W0402 + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + # if run from command line, and give args, wipe the chpasswd['list'] + password = args[0] + if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: + del cfg['chpasswd']['list'] + else: + password = util.get_cfg_option_str(cfg, "password", None) + + expire = True + pw_auth = "no" + change_pwauth = False + plist = None + + if 'chpasswd' in cfg: + chfg = cfg['chpasswd'] + plist = util.get_cfg_option_str(chfg, 'list', plist) + expire = util.get_cfg_option_bool(chfg, 'expire', expire) + + if not plist and password: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + plist = "%s:%s" % (user, password) + + errors = [] + if plist: + plist_in = [] + randlist = [] + users = [] + for line in plist.splitlines(): + u, p = line.split(':', 1) + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) + + ch_in = '\n'.join(plist_in) + try: + util.subp(['chpasswd'], ch_in) + log.debug("changed password for %s:" % users) + except Exception as e: + errors.append(e) + log.warn("failed to set passwords with chpasswd: %s" % e) + + if len(randlist): + sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + '\n'.join(randlist))) + + if expire: + enum = len(errors) + for u in users: + try: + util.subp(['passwd', '--expire', u]) + except Exception as e: + errors.append(e) + log.warn("failed to expire account for %s" % u) + if enum == len(errors): + log.debug("expired passwords for: %s" % u) + + if 'ssh_pwauth' in cfg: + val = str(cfg['ssh_pwauth']).lower() + if val in ("true", "1", "yes"): + pw_auth = "yes" + change_pwauth = True + elif val in ("false", "0", "no"): + pw_auth = "no" + change_pwauth = True + else: + change_pwauth = False + + if change_pwauth: + pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" + msg = "set PasswordAuthentication to '%s'" % pw_auth + try: + cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), + '/etc/ssh/sshd_config'] + util.subp(cmd) + log.debug(msg) + except Exception as e: + log.warn("failed %s" % msg) + errors.append(e) + + try: + p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), + 'restart']) + log.debug("restarted sshd") + except: + log.warn("restart of ssh failed") + + if len(errors): + raise(errors[0]) + + return + + +def rand_str(strlen=32, select_from=letters + digits): + return("".join([random.choice(select_from) for _x in range(0, strlen)])) + + +def rand_user_password(pwlen=9): + selfrom = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/handlers/cc_ssh.py b/cloudinit/handlers/cc_ssh.py new file mode 100644 index 00000000..48eb58bc --- /dev/null +++ b/cloudinit/handlers/cc_ssh.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import os +import glob +import subprocess + +DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ +"rather than the user \\\"root\\\".\';echo;sleep 10\"" + + +def handle(_name, cfg, cloud, log, _args): + + # remove the static keys from the pristine image + if cfg.get("ssh_deletekeys", True): + for f in glob.glob("/etc/ssh/ssh_host_*key*"): + try: + os.unlink(f) + except: + pass + + if "ssh_keys" in cfg: + # if there are keys in cloud-config, use them + key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + } + + for key, val in cfg["ssh_keys"].items(): + if key in key2file: + util.write_file(key2file[key][0], val, key2file[key][1]) + + priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', } + + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): + if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + continue + pair = (key2file[priv][0], key2file[pub][0]) + subprocess.call(('sh', '-xc', cmd % pair)) + log.debug("generated %s from %s" % pair) + else: + # if not, generate them + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', + ['rsa', 'dsa', 'ecdsa']): + keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + if not os.path.exists(keyfile): + subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', + '-f', keyfile]) + + util.restorecon_if_possible('/etc/ssh', recursive=True) + + try: + user = util.get_cfg_option_str(cfg, 'user') + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) + disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", + DISABLE_ROOT_OPTS) + keys = cloud.get_public_ssh_keys() + + if "ssh_authorized_keys" in cfg: + cfgkeys = cfg["ssh_authorized_keys"] + keys.extend(cfgkeys) + + apply_credentials(keys, user, disable_root, disable_root_opts, log) + except: + util.logexc(log) + log.warn("applying credentials failed!\n") + + +def apply_credentials(keys, user, disable_root, + disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) + if user: + sshutil.setup_user_keys(keys, user, '', log) + + if disable_root: + key_prefix = disable_root_opts.replace('$USER', user) + else: + key_prefix = '' + + sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/handlers/cc_ssh_import_id.py b/cloudinit/handlers/cc_ssh_import_id.py new file mode 100644 index 00000000..bbf5bd83 --- /dev/null +++ b/cloudinit/handlers/cc_ssh_import_id.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + user = args[0] + ids = [] + if len(args) > 1: + ids = args[1:] + else: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + + if len(ids) == 0: + return + + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids + + log.debug("importing ssh ids. cmd = %s" % cmd) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_timezone.py b/cloudinit/handlers/cc_timezone.py new file mode 100644 index 00000000..e5c9901b --- /dev/null +++ b/cloudinit/handlers/cc_timezone.py @@ -0,0 +1,67 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +from cloudinit import util +import os.path +import shutil + +frequency = per_instance +tz_base = "/usr/share/zoneinfo" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + timezone = args[0] + else: + timezone = util.get_cfg_option_str(cfg, "timezone", False) + + if not timezone: + return + + tz_file = "%s/%s" % (tz_base, timezone) + + if not os.path.isfile(tz_file): + log.debug("Invalid timezone %s" % tz_file) + raise Exception("Invalid timezone %s" % tz_file) + + try: + fp = open("/etc/timezone", "wb") + fp.write("%s\n" % timezone) + fp.close() + except: + log.debug("failed to write to /etc/timezone") + raise + if os.path.exists("/etc/sysconfig/clock"): + try: + with open("/etc/sysconfig/clock", "w") as fp: + fp.write('ZONE="%s"\n' % timezone) + except: + log.debug("failed to write to /etc/sysconfig/clock") + raise + + try: + shutil.copy(tz_file, "/etc/localtime") + except: + log.debug("failed to copy %s to /etc/localtime" % tz_file) + raise + + log.debug("set timezone to %s" % timezone) + return diff --git a/cloudinit/handlers/cc_update_etc_hosts.py b/cloudinit/handlers/cc_update_etc_hosts.py new file mode 100644 index 00000000..6ad2fca8 --- /dev/null +++ b/cloudinit/handlers/cc_update_etc_hosts.py @@ -0,0 +1,87 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +import StringIO + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + + manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) + if manage_hosts in ("True", "true", True, "template"): + # render from template file + try: + if not hostname: + log.info("manage_etc_hosts was set, but no hostname found") + return + + util.render_to_file('hosts', '/etc/hosts', + {'hostname': hostname, 'fqdn': fqdn}) + except Exception: + log.warn("failed to update /etc/hosts") + raise + elif manage_hosts == "localhost": + log.debug("managing 127.0.1.1 in /etc/hosts") + update_etc_hosts(hostname, fqdn, log) + return + else: + if manage_hosts not in ("False", False): + log.warn("Unknown value for manage_etc_hosts. Assuming False") + else: + log.debug("not managing /etc/hosts") + + +def update_etc_hosts(hostname, fqdn, _log): + with open('/etc/hosts', 'r') as etchosts: + header = "# Added by cloud-init\n" + hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) + need_write = False + need_change = True + new_etchosts = StringIO.StringIO() + for line in etchosts: + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: + new_etchosts.write(line) + continue + if line == header: + continue + ip, hosts = split_line[0], split_line[1:] + if ip == "127.0.1.1": + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change == True: + line = "%s%s" % (header, hosts_line) + need_change = False + need_write = True + new_etchosts.write(line) + etchosts.close() + if need_change == True: + new_etchosts.write("%s%s" % (header, hosts_line)) + need_write = True + if need_write == True: + new_etcfile = open('/etc/hosts', 'wb') + new_etcfile.write(new_etchosts.getvalue()) + new_etcfile.close() + new_etchosts.close() + return diff --git a/cloudinit/handlers/cc_update_hostname.py b/cloudinit/handlers/cc_update_hostname.py new file mode 100644 index 00000000..b9d1919a --- /dev/null +++ b/cloudinit/handlers/cc_update_hostname.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not updating hostname") + return + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") + update_hostname(hostname, prev, log) + except Exception: + log.warn("failed to set hostname\n") + raise + + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename, "r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError as e: + if e.errno != errno.ENOENT: + raise + return default + + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception as e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname, "%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname, fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file, etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() -- cgit v1.2.3 From 869402301c9793cece24a9357ee3c13dcdafb6e2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 7 Jun 2012 12:45:28 -0700 Subject: Darn it. Those shouldn't be there! --- cloudinit/handlers/DataSource.py | 214 ----------------- cloudinit/handlers/DataSourceCloudStack.py | 92 -------- cloudinit/handlers/DataSourceConfigDrive.py | 231 ------------------- cloudinit/handlers/DataSourceEc2.py | 217 ----------------- cloudinit/handlers/DataSourceMAAS.py | 345 ---------------------------- cloudinit/handlers/DataSourceNoCloud.py | 232 ------------------- cloudinit/handlers/DataSourceOVF.py | 332 -------------------------- cloudinit/sources/DataSource.py | 214 +++++++++++++++++ cloudinit/sources/DataSourceCloudStack.py | 92 ++++++++ cloudinit/sources/DataSourceConfigDrive.py | 231 +++++++++++++++++++ cloudinit/sources/DataSourceEc2.py | 217 +++++++++++++++++ cloudinit/sources/DataSourceMAAS.py | 345 ++++++++++++++++++++++++++++ cloudinit/sources/DataSourceNoCloud.py | 232 +++++++++++++++++++ cloudinit/sources/DataSourceOVF.py | 332 ++++++++++++++++++++++++++ cloudinit/sources/__init__.py | 0 15 files changed, 1663 insertions(+), 1663 deletions(-) delete mode 100644 cloudinit/handlers/DataSource.py delete mode 100644 cloudinit/handlers/DataSourceCloudStack.py delete mode 100644 cloudinit/handlers/DataSourceConfigDrive.py delete mode 100644 cloudinit/handlers/DataSourceEc2.py delete mode 100644 cloudinit/handlers/DataSourceMAAS.py delete mode 100644 cloudinit/handlers/DataSourceNoCloud.py delete mode 100644 cloudinit/handlers/DataSourceOVF.py create mode 100644 cloudinit/sources/DataSource.py create mode 100644 cloudinit/sources/DataSourceCloudStack.py create mode 100644 cloudinit/sources/DataSourceConfigDrive.py create mode 100644 cloudinit/sources/DataSourceEc2.py create mode 100644 cloudinit/sources/DataSourceMAAS.py create mode 100644 cloudinit/sources/DataSourceNoCloud.py create mode 100644 cloudinit/sources/DataSourceOVF.py create mode 100644 cloudinit/sources/__init__.py (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/DataSource.py b/cloudinit/handlers/DataSource.py deleted file mode 100644 index e2a9150d..00000000 --- a/cloudinit/handlers/DataSource.py +++ /dev/null @@ -1,214 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -DEP_FILESYSTEM = "FILESYSTEM" -DEP_NETWORK = "NETWORK" - -import cloudinit.UserDataHandler as ud -import cloudinit.util as util -import socket - - -class DataSource: - userdata = None - metadata = None - userdata_raw = None - cfgname = "" - # system config (passed in from cloudinit, - # cloud-config before input from the DataSource) - sys_cfg = {} - # datasource config, the cloud-config['datasource']['__name__'] - ds_cfg = {} # datasource config - - def __init__(self, sys_cfg=None): - if not self.cfgname: - name = str(self.__class__).split(".")[-1] - if name.startswith("DataSource"): - name = name[len("DataSource"):] - self.cfgname = name - if sys_cfg: - self.sys_cfg = sys_cfg - - self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, - ("datasource", self.cfgname), self.ds_cfg) - - def get_userdata(self): - if self.userdata == None: - self.userdata = ud.preprocess_userdata(self.userdata_raw) - return self.userdata - - def get_userdata_raw(self): - return(self.userdata_raw) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return({}) - - def get_public_ssh_keys(self): - keys = [] - if 'public-keys' not in self.metadata: - return([]) - - if isinstance(self.metadata['public-keys'], str): - return(str(self.metadata['public-keys']).splitlines()) - - if isinstance(self.metadata['public-keys'], list): - return(self.metadata['public-keys']) - - for _keyname, klist in self.metadata['public-keys'].items(): - # lp:506332 uec metadata service responds with - # data that makes boto populate a string for 'klist' rather - # than a list. - if isinstance(klist, str): - klist = [klist] - for pkey in klist: - # there is an empty string at the end of the keylist, trim it - if pkey: - keys.append(pkey) - - return(keys) - - def device_name_to_device(self, _name): - # translate a 'name' to a device - # the primary function at this point is on ec2 - # to consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - return(None) - - def get_locale(self): - return('en_US.UTF-8') - - def get_local_mirror(self): - return None - - def get_instance_id(self): - if 'instance-id' not in self.metadata: - return "iid-datasource" - return(self.metadata['instance-id']) - - def get_hostname(self, fqdn=False): - defdomain = "localdomain" - defhost = "localhost" - - domain = defdomain - if not 'local-hostname' in self.metadata: - - # this is somewhat questionable really. - # the cloud datasource was asked for a hostname - # and didn't have one. raising error might be more appropriate - # but instead, basically look up the existing hostname - toks = [] - - hostname = socket.gethostname() - - fqdn = util.get_fqdn_from_hosts(hostname) - - if fqdn and fqdn.find(".") > 0: - toks = str(fqdn).split(".") - elif hostname: - toks = [hostname, defdomain] - else: - toks = [defhost, defdomain] - - else: - # if there is an ipv4 address in 'local-hostname', then - # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] - if is_ipv4(lhost): - toks = "ip-%s" % lhost.replace(".", "-") - else: - toks = lhost.split(".") - - if len(toks) > 1: - hostname = toks[0] - domain = '.'.join(toks[1:]) - else: - hostname = toks[0] - - if fqdn: - return "%s.%s" % (hostname, domain) - else: - return hostname - - -# return a list of classes that have the same depends as 'depends' -# iterate through cfg_list, loading "DataSourceCollections" modules -# and calling their "get_datasource_list". -# return an ordered list of classes that match -# -# - modules must be named "DataSource", where 'item' is an entry -# in cfg_list -# - if pkglist is given, it will iterate try loading from that package -# ie, pkglist=[ "foo", "" ] -# will first try to load foo.DataSource -# then DataSource -def list_sources(cfg_list, depends, pkglist=None): - if pkglist is None: - pkglist = [] - retlist = [] - for ds_coll in cfg_list: - for pkg in pkglist: - if pkg: - pkg = "%s." % pkg - try: - mod = __import__("%sDataSource%s" % (pkg, ds_coll)) - if pkg: - mod = getattr(mod, "DataSource%s" % ds_coll) - lister = getattr(mod, "get_datasource_list") - retlist.extend(lister(depends)) - break - except: - raise - return(retlist) - - -# depends is a list of dependencies (DEP_FILESYSTEM) -# dslist is a list of 2 item lists -# dslist = [ -# ( class, ( depends-that-this-class-needs ) ) -# } -# it returns a list of 'class' that matched these deps exactly -# it is a helper function for DataSourceCollections -def list_from_depends(depends, dslist): - retlist = [] - depset = set(depends) - for elem in dslist: - (cls, deps) = elem - if depset == set(deps): - retlist.append(cls) - return(retlist) - - -def is_ipv4(instr): - """ determine if input string is a ipv4 address. return boolean""" - toks = instr.split('.') - if len(toks) != 4: - return False - - try: - toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] - except: - return False - - return (len(toks) == 4) diff --git a/cloudinit/handlers/DataSourceCloudStack.py b/cloudinit/handlers/DataSourceCloudStack.py deleted file mode 100644 index 5afdf7b6..00000000 --- a/cloudinit/handlers/DataSourceCloudStack.py +++ /dev/null @@ -1,92 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Cosmin Luta -# -# Author: Cosmin Luta -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -from socket import inet_ntoa -import time -import boto.utils as boto_utils -from struct import pack - - -class DataSourceCloudStack(DataSource.DataSource): - api_ver = 'latest' - seeddir = base_seeddir + '/cs' - metadata_address = None - - def __init__(self, sys_cfg=None): - DataSource.DataSource.__init__(self, sys_cfg) - # Cloudstack has its metadata/userdata URLs located at - # http:///latest/ - self.metadata_address = "http://%s/" % self.get_default_gateway() - - def get_default_gateway(self): - """ Returns the default gateway ip address in the dotted format - """ - with open("/proc/net/route", "r") as f: - for line in f.readlines(): - items = line.split("\t") - if items[1] == "00000000": - # found the default route, get the gateway - gw = inet_ntoa(pack(" -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -import json -import subprocess - -DEFAULT_IID = "iid-dsconfigdrive" - - -class DataSourceConfigDrive(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/config_drive' - cfg = {} - userdata_raw = None - metadata = None - dsmode = "local" - - def __str__(self): - mstr = "DataSourceConfigDrive[%s]" % self.dsmode - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = None - md = {} - ud = "" - - defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} - - if os.path.isdir(self.seeddir): - try: - (md, ud) = read_config_drive_dir(self.seeddir) - found = self.seeddir - except nonConfigDriveDir: - pass - - if not found: - dev = cfg_drive_device() - if dev: - try: - (md, ud) = util.mount_callback_umount(dev, - read_config_drive_dir) - found = dev - except (nonConfigDriveDir, util.mountFailedError): - pass - - if not found: - return False - - if 'dsconfig' in md: - self.cfg = md['dscfg'] - - md = util.mergedict(md, defaults) - - # update interfaces and ifup only on the local datasource - # this way the DataSourceConfigDriveNet doesn't do it also. - if 'network-interfaces' in md and self.dsmode == "local": - if md['dsmode'] == "pass": - log.info("updating network interfaces from configdrive") - else: - log.debug("updating network interfaces from configdrive") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = found - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return(self.metadata['public-keys']) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceConfigDriveNet(DataSourceConfigDrive): - dsmode = "net" - - -class nonConfigDriveDir(Exception): - pass - - -def cfg_drive_device(): - """ get the config drive device. return a string like '/dev/vdb' - or None (if there is no non-root device attached). This does not - check the contents, only reports that if there *were* a config_drive - attached, it would be this device. - per config_drive documentation, this is - "associated as the last available disk on the instance" - """ - - if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: - return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) - - # we are looking for a raw block device (sda, not sda1) with a vfat - # filesystem on it. - - letters = "abcdefghijklmnopqrstuvwxyz" - devs = util.find_devs_with("TYPE=vfat") - - # filter out anything not ending in a letter (ignore partitions) - devs = [f for f in devs if f[-1] in letters] - - # sort them in reverse so "last" device is first - devs.sort(reverse=True) - - if len(devs): - return(devs[0]) - - return(None) - - -def read_config_drive_dir(source_dir): - """ - read_config_drive_dir(source_dir): - read source_dir, and return a tuple with metadata dict and user-data - string populated. If not a valid dir, raise a nonConfigDriveDir - """ - md = {} - ud = "" - - flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") - found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] - keydata = "" - - if len(found) == 0: - raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) - - if "etc/network/interfaces" in found: - with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: - md['network-interfaces'] = fp.read() - - if "root/.ssh/authorized_keys" in found: - with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: - keydata = fp.read() - - meta_js = {} - - if "meta.js" in found: - content = '' - with open("%s/%s" % (source_dir, "meta.js")) as fp: - content = fp.read() - md['meta_js'] = content - try: - meta_js = json.loads(content) - except ValueError: - raise nonConfigDriveDir("%s: %s" % - (source_dir, "invalid json in meta.js")) - - keydata = meta_js.get('public-keys', keydata) - - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - for copy in ('dsmode', 'instance-id', 'dscfg'): - if copy in meta_js: - md[copy] = meta_js[copy] - - if 'user-data' in meta_js: - ud = meta_js['user-data'] - - return(md, ud) - -datasources = ( - (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - -if __name__ == "__main__": - def main(): - import sys - import pprint - print cfg_drive_device() - (md, ud) = read_config_drive_dir(sys.argv[1]) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - print(ud) - - main() - -# vi: ts=4 expandtab diff --git a/cloudinit/handlers/DataSourceEc2.py b/cloudinit/handlers/DataSourceEc2.py deleted file mode 100644 index 7051ecda..00000000 --- a/cloudinit/handlers/DataSourceEc2.py +++ /dev/null @@ -1,217 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import socket -import time -import boto.utils as boto_utils -import os.path - - -class DataSourceEc2(DataSource.DataSource): - api_ver = '2009-04-04' - seeddir = base_seeddir + '/ec2' - metadata_address = "http://169.254.169.254" - - def __str__(self): - return("DataSourceEc2") - - def get_data(self): - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - self.userdata_raw = seedret['user-data'] - self.metadata = seedret['meta-data'] - log.debug("using seeded ec2 data in %s" % self.seeddir) - return True - - try: - if not self.wait_for_metadata_service(): - return False - start = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) - log.debug("crawl of metadata service took %ds" % (time.time() - - start)) - return True - except Exception as e: - print e - return False - - def get_instance_id(self): - return(self.metadata['instance-id']) - - def get_availability_zone(self): - return(self.metadata['placement']['availability-zone']) - - def get_local_mirror(self): - return(self.get_mirror_from_availability_zone()) - - def get_mirror_from_availability_zone(self, availability_zone=None): - # availability is like 'us-west-1b' or 'eu-west-1a' - if availability_zone == None: - availability_zone = self.get_availability_zone() - - fallback = None - - if self.is_vpc(): - return fallback - - try: - host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] - socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) - return 'http://%s/ubuntu/' % host - except: - return fallback - - def wait_for_metadata_service(self): - mcfg = self.ds_cfg - - if not hasattr(mcfg, "get"): - mcfg = {} - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] - mdurls = mcfg.get("metadata_urls", def_mdurls) - - # Remove addresses from the list that wont resolve. - filtered = [x for x in mdurls if util.is_resolvable_url(x)] - - if set(filtered) != set(mdurls): - log.debug("removed the following from metadata urls: %s" % - list((set(mdurls) - set(filtered)))) - - if len(filtered): - mdurls = filtered - else: - log.warn("Empty metadata url list! using default list") - mdurls = def_mdurls - - urls = [] - url2base = {False: False} - for url in mdurls: - cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) - urls.append(cur) - url2base[cur] = url - - starttime = time.time() - url = util.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=log.warn) - - if url: - log.debug("Using metadata source: '%s'" % url2base[url]) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - self.metadata_address = url2base[url] - return (bool(url)) - - def device_name_to_device(self, name): - # consult metadata service, that has - # ephemeral0: sdb - # and return 'sdb' for input 'ephemeral0' - if 'block-device-mapping' not in self.metadata: - return(None) - - found = None - for entname, device in self.metadata['block-device-mapping'].items(): - if entname == name: - found = device - break - # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' - if entname == "ephemeral" and name == "ephemeral0": - found = device - if found == None: - log.debug("unable to convert %s to a device" % name) - return None - - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} - ofound = found - short = os.path.basename(found) - - if not found.startswith("/"): - found = "/dev/%s" % found - - if os.path.exists(found): - return(found) - - for nfrom, tlist in mappings.items(): - if not short.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short[len(nfrom):]) - if os.path.exists(cand): - log.debug("remapped device name %s => %s" % (found, cand)) - return(cand) - - # on t1.micro, ephemeral0 will appear in block-device-mapping from - # metadata, but it will not exist on disk (and never will) - # at this pint, we've verified that the path did not exist - # in the special case of 'ephemeral0' return None to avoid bogus - # fstab entry (LP: #744019) - if name == "ephemeral0": - return None - return ofound - - def is_vpc(self): - # per comment in LP: #615545 - ph = "public-hostname" - p4 = "public-ipv4" - if ((ph not in self.metadata or self.metadata[ph] == "") and - (p4 not in self.metadata or self.metadata[p4] == "")): - return True - return False - - -datasources = [ - (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceMAAS.py b/cloudinit/handlers/DataSourceMAAS.py deleted file mode 100644 index 61a0038f..00000000 --- a/cloudinit/handlers/DataSourceMAAS.py +++ /dev/null @@ -1,345 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import oauth.oauth as oauth -import os.path -import urllib2 -import time - - -MD_VERSION = "2012-03-01" - - -class DataSourceMAAS(DataSource.DataSource): - """ - DataSourceMAAS reads instance information from MAAS. - Given a config metadata_url, and oauth tokens, it expects to find - files under the root named: - instance-id - user-data - hostname - """ - seeddir = base_seeddir + '/maas' - baseurl = None - - def __str__(self): - return("DataSourceMAAS[%s]" % self.baseurl) - - def get_data(self): - mcfg = self.ds_cfg - - try: - (userdata, metadata) = read_maas_seed_dir(self.seeddir) - self.userdata_raw = userdata - self.metadata = metadata - self.baseurl = self.seeddir - return True - except MAASSeedDirNone: - pass - except MAASSeedDirMalformed as exc: - log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) - raise - - try: - # if there is no metadata_url, then we're not configured - url = mcfg.get('metadata_url', None) - if url == None: - return False - - if not self.wait_for_metadata_service(url): - return False - - self.baseurl = url - - (userdata, metadata) = read_maas_seed_url(self.baseurl, - self.md_headers) - self.userdata_raw = userdata - self.metadata = metadata - return True - except Exception: - util.logexc(log) - return False - - def md_headers(self, url): - mcfg = self.ds_cfg - - # if we are missing token_key, token_secret or consumer_key - # then just do non-authed requests - for required in ('token_key', 'token_secret', 'consumer_key'): - if required not in mcfg: - return({}) - - consumer_secret = mcfg.get('consumer_secret', "") - - return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], - token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], - consumer_secret=consumer_secret)) - - def wait_for_metadata_service(self, url): - mcfg = self.ds_cfg - - max_wait = 120 - try: - max_wait = int(mcfg.get("max_wait", max_wait)) - except Exception: - util.logexc(log) - log.warn("Failed to get max wait. using %s" % max_wait) - - if max_wait == 0: - return False - - timeout = 50 - try: - timeout = int(mcfg.get("timeout", timeout)) - except Exception: - util.logexc(log) - log.warn("Failed to get timeout, using %s" % timeout) - - starttime = time.time() - check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=log.warn, - headers_cb=self.md_headers) - - if url: - log.debug("Using metadata source: '%s'" % url) - else: - log.critical("giving up on md after %i seconds\n" % - int(time.time() - starttime)) - - return (bool(url)) - - -def read_maas_seed_dir(seed_d): - """ - Return user-data and metadata for a maas seed dir in seed_d. - Expected format of seed_d are the following files: - * instance-id - * local-hostname - * user-data - """ - files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') - md = {} - - if not os.path.isdir(seed_d): - raise MAASSeedDirNone("%s: not a directory") - - for fname in files: - try: - with open(os.path.join(seed_d, fname)) as fp: - md[fname] = fp.read() - fp.close() - except IOError as e: - if e.errno != errno.ENOENT: - raise - - return(check_seed_contents(md, seed_d)) - - -def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): - """ - Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() - - Expected format of seed_url is are the following files: - * //meta-data/instance-id - * //meta-data/local-hostname - * //user-data - """ - files = ('meta-data/local-hostname', - 'meta-data/instance-id', - 'meta-data/public-keys', - 'user-data') - - base_url = "%s/%s" % (seed_url, version) - md = {} - for fname in files: - url = "%s/%s" % (base_url, fname) - if header_cb: - headers = header_cb(url) - else: - headers = {} - - try: - req = urllib2.Request(url, data=None, headers=headers) - resp = urllib2.urlopen(req, timeout=timeout) - md[os.path.basename(fname)] = resp.read() - except urllib2.HTTPError as e: - if e.code != 404: - raise - - return(check_seed_contents(md, seed_url)) - - -def check_seed_contents(content, seed): - """Validate if content is Is the content a dict that is valid as a - return for a datasource. - Either return a (userdata, metadata) tuple or - Raise MAASSeedDirMalformed or MAASSeedDirNone - """ - md_required = ('instance-id', 'local-hostname') - found = content.keys() - - if len(content) == 0: - raise MAASSeedDirNone("%s: no data files found" % seed) - - missing = [k for k in md_required if k not in found] - if len(missing): - raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) - - userdata = content.get('user-data', "") - md = {} - for (key, val) in content.iteritems(): - if key == 'user-data': - continue - md[key] = val - - return(userdata, md) - - -def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): - consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) - token = oauth.OAuthToken(token_key, token_secret) - params = { - 'oauth_version': "1.0", - 'oauth_nonce': oauth.generate_nonce(), - 'oauth_timestamp': int(time.time()), - 'oauth_token': token.key, - 'oauth_consumer_key': consumer.key, - } - req = oauth.OAuthRequest(http_url=url, parameters=params) - req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), - consumer, token) - return(req.to_header()) - - -class MAASSeedDirNone(Exception): - pass - - -class MAASSeedDirMalformed(Exception): - pass - - -datasources = [ - (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -] - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - """ - Call with single argument of directory or http or https url. - If url is given additional arguments are allowed, which will be - interpreted as consumer_key, token_key, token_secret, consumer_secret - """ - import argparse - import pprint - - parser = argparse.ArgumentParser(description='Interact with MAAS DS') - parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) - parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) - parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) - parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") - parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) - parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) - - subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") - subcmds.add_parser('crawl', help="crawl the datasource") - subcmds.add_parser('get', help="do a single GET of provided url") - subcmds.add_parser('check-seed', help="read andn verify seed at url") - - parser.add_argument("url", help="the data source to query") - - args = parser.parse_args() - - creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} - - if args.config: - import yaml - with open(args.config) as fp: - cfg = yaml.load(fp) - if 'datasource' in cfg: - cfg = cfg['datasource']['MAAS'] - for key in creds.keys(): - if key in cfg and creds[key] == None: - creds[key] = cfg[key] - - def geturl(url, headers_cb): - req = urllib2.Request(url, data=None, headers=headers_cb(url)) - return(urllib2.urlopen(req).read()) - - def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) - - def crawl(url, headers_cb=None): - if url.endswith("/"): - for line in geturl(url, headers_cb).splitlines(): - if line.endswith("/"): - crawl("%s%s" % (url, line), headers_cb) - else: - printurl("%s%s" % (url, line), headers_cb) - else: - printurl(url, headers_cb) - - def my_headers(url): - headers = {} - if creds.get('consumer_key', None) != None: - headers = oauth_headers(url, **creds) - return headers - - if args.subcmd == "check-seed": - if args.url.startswith("http"): - (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) - else: - (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" - pprint.pprint(metadata) - - elif args.subcmd == "get": - printurl(args.url, my_headers) - - elif args.subcmd == "crawl": - if not args.url.endswith("/"): - args.url = "%s/" % args.url - crawl(args.url, my_headers) - - main() diff --git a/cloudinit/handlers/DataSourceNoCloud.py b/cloudinit/handlers/DataSourceNoCloud.py deleted file mode 100644 index e8c56b8f..00000000 --- a/cloudinit/handlers/DataSourceNoCloud.py +++ /dev/null @@ -1,232 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import errno -import subprocess - - -class DataSourceNoCloud(DataSource.DataSource): - metadata = None - userdata = None - userdata_raw = None - supported_seed_starts = ("/", "file://") - dsmode = "local" - seed = None - cmdline_id = "ds=nocloud" - seeddir = base_seeddir + '/nocloud' - - def __str__(self): - mstr = "DataSourceNoCloud" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - defaults = { - "instance-id": "nocloud", "dsmode": self.dsmode - } - - found = [] - md = {} - ud = "" - - try: - # parse the kernel command line, getting data passed in - if parse_cmdline_data(self.cmdline_id, md): - found.append("cmdline") - except: - util.logexc(log) - return False - - # check to see if the seeddir has data. - seedret = {} - if util.read_optional_seed(seedret, base=self.seeddir + "/"): - md = util.mergedict(md, seedret['meta-data']) - ud = seedret['user-data'] - found.append(self.seeddir) - log.debug("using seeded cache data in %s" % self.seeddir) - - # if the datasource config had a 'seedfrom' entry, then that takes - # precedence over a 'seedfrom' that was found in a filesystem - # but not over external medi - if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: - found.append("ds_config") - md["seedfrom"] = self.ds_cfg['seedfrom'] - - fslist = util.find_devs_with("TYPE=vfat") - fslist.extend(util.find_devs_with("TYPE=iso9660")) - - label_list = util.find_devs_with("LABEL=cidata") - devlist = list(set(fslist) & set(label_list)) - devlist.sort(reverse=True) - - for dev in devlist: - try: - (newmd, newud) = util.mount_callback_umount(dev, - util.read_seeded) - md = util.mergedict(newmd, md) - ud = newud - - # for seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. - # If they want dsmode of local, then they must - # specify that. - if 'dsmode' not in md: - md['dsmode'] = "net" - - log.debug("using data from %s" % dev) - found.append(dev) - break - except OSError, e: - if e.errno != errno.ENOENT: - raise - except util.mountFailedError: - log.warn("Failed to mount %s when looking for seed" % dev) - - # there was no indication on kernel cmdline or data - # in the seeddir suggesting this handler should be used. - if len(found) == 0: - return False - - seeded_interfaces = None - - # the special argument "seedfrom" indicates we should - # attempt to seed the userdata / metadata from its value - # its primarily value is in allowing the user to type less - # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg - if "seedfrom" in md: - seedfrom = md["seedfrom"] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - if 'network-interfaces' in md: - seeded_interfaces = self.dsmode - - # this could throw errors, but the user told us to do it - # so if errors are raised, let them raise - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - # values in the command line override those from the seed - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - - # update the network-interfaces if metadata had 'network-interfaces' - # entry and this is the local datasource, or 'seedfrom' was used - # and the source of the seed was self.dsmode - # ('local' for NoCloud, 'net' for NoCloudNet') - if ('network-interfaces' in md and - (self.dsmode in ("local", seeded_interfaces))): - log.info("updating network interfaces from nocloud") - - util.write_file("/etc/network/interfaces", - md['network-interfaces']) - try: - (out, err) = util.subp(['ifup', '--all']) - if len(out) or len(err): - log.warn("ifup --all had stderr: %s" % err) - - except subprocess.CalledProcessError as exc: - log.warn("ifup --all failed: %s" % (exc.output[1])) - - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - - if md['dsmode'] == self.dsmode: - return True - - log.debug("%s: not claiming datasource, dsmode=%s" % - (self, md['dsmode'])) - return False - - -# returns true or false indicating if cmdline indicated -# that this module should be used -# example cmdline: -# root=LABEL=uec-rootfs ro ds=nocloud -def parse_cmdline_data(ds_id, fill, cmdline=None): - if cmdline is None: - cmdline = util.get_cmdline() - cmdline = " %s " % cmdline - - if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): - return False - - argline = "" - # cmdline can contain: - # ds=nocloud[;key=val;key=val] - for tok in cmdline.split(): - if tok.startswith(ds_id): - argline = tok.split("=", 1) - - # argline array is now 'nocloud' followed optionally by - # a ';' and then key=value pairs also terminated with ';' - tmp = argline[1].split(";") - if len(tmp) > 1: - kvpairs = tmp[1:] - else: - kvpairs = () - - # short2long mapping to save cmdline typing - s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} - for item in kvpairs: - try: - (k, v) = item.split("=", 1) - except: - k = item - v = None - if k in s2l: - k = s2l[k] - fill[k] = v - - return(True) - - -class DataSourceNoCloudNet(DataSourceNoCloud): - cmdline_id = "ds=nocloud-net" - supported_seed_starts = ("http://", "https://", "ftp://") - seeddir = base_seeddir + '/nocloud-net' - dsmode = "net" - - -datasources = ( - (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), - (DataSourceNoCloudNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/handlers/DataSourceOVF.py b/cloudinit/handlers/DataSourceOVF.py deleted file mode 100644 index a0b1b518..00000000 --- a/cloudinit/handlers/DataSourceOVF.py +++ /dev/null @@ -1,332 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Hafliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.DataSource as DataSource - -from cloudinit import seeddir as base_seeddir -from cloudinit import log -import cloudinit.util as util -import os.path -import os -from xml.dom import minidom -import base64 -import re -import tempfile -import subprocess - - -class DataSourceOVF(DataSource.DataSource): - seed = None - seeddir = base_seeddir + '/ovf' - environment = None - cfg = {} - userdata_raw = None - metadata = None - supported_seed_starts = ("/", "file://") - - def __str__(self): - mstr = "DataSourceOVF" - mstr = mstr + " [seed=%s]" % self.seed - return(mstr) - - def get_data(self): - found = [] - md = {} - ud = "" - - defaults = { - "instance-id": "iid-dsovf" - } - - (seedfile, contents) = get_ovf_env(base_seeddir) - if seedfile: - # found a seed dir - seed = "%s/%s" % (base_seeddir, seedfile) - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - - found.append(seed) - else: - np = {'iso': transport_iso9660, - 'vmware-guestd': transport_vmware_guestd, } - name = None - for name, transfunc in np.iteritems(): - (contents, _dev, _fname) = transfunc() - if contents: - break - - if contents: - (md, ud, cfg) = read_ovf_environment(contents) - self.environment = contents - found.append(name) - - # There was no OVF transports found - if len(found) == 0: - return False - - if 'seedfrom' in md and md['seedfrom']: - seedfrom = md['seedfrom'] - seedfound = False - for proto in self.supported_seed_starts: - if seedfrom.startswith(proto): - seedfound = proto - break - if not seedfound: - log.debug("seed from %s not supported by %s" % - (seedfrom, self.__class__)) - return False - - (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) - log.debug("using seeded cache data from %s" % seedfrom) - - md = util.mergedict(md, md_seed) - found.append(seedfrom) - - md = util.mergedict(md, defaults) - self.seed = ",".join(found) - self.metadata = md - self.userdata_raw = ud - self.cfg = cfg - return True - - def get_public_ssh_keys(self): - if not 'public-keys' in self.metadata: - return([]) - return([self.metadata['public-keys'], ]) - - # the data sources' config_obj is a cloud-config formated - # object that came to it from ways other than cloud-config - # because cloud-config content would be handled elsewhere - def get_config_obj(self): - return(self.cfg) - - -class DataSourceOVFNet(DataSourceOVF): - seeddir = base_seeddir + '/ovf-net' - supported_seed_starts = ("http://", "https://", "ftp://") - - -# this will return a dict with some content -# meta-data, user-data -def read_ovf_environment(contents): - props = getProperties(contents) - md = {} - cfg = {} - ud = "" - cfg_props = ['password', ] - md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for prop, val in props.iteritems(): - if prop == 'hostname': - prop = "local-hostname" - if prop in md_props: - md[prop] = val - elif prop in cfg_props: - cfg[prop] = val - elif prop == "user-data": - try: - ud = base64.decodestring(val) - except: - ud = val - return(md, ud, cfg) - - -# returns tuple of filename (in 'dirname', and the contents of the file) -# on "not found", returns 'None' for filename and False for contents -def get_ovf_env(dirname): - env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") - for fname in env_names: - if os.path.isfile("%s/%s" % (dirname, fname)): - fp = open("%s/%s" % (dirname, fname)) - contents = fp.read() - fp.close() - return(fname, contents) - return(None, False) - - -# transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): - - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" - - devname_regex = os.environ.get(envname, default_regex) - cdmatch = re.compile(devname_regex) - - # go through mounts to see if it was already mounted - fp = open("/proc/mounts") - mounts = fp.readlines() - fp.close() - - mounted = {} - for mpline in mounts: - (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() - mounted[dev] = (dev, fstype, mp, False) - mp = mp.replace("\\040", " ") - if fstype != "iso9660" and require_iso: - continue - - if cdmatch.match(dev[5:]) == None: # take off '/dev/' - continue - - (fname, contents) = get_ovf_env(mp) - if contents is not False: - return(contents, dev, fname) - - tmpd = None - dvnull = None - - devs = os.listdir("/dev/") - devs.sort() - - for dev in devs: - fullp = "/dev/%s" % dev - - if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): - continue - - fp = None - try: - fp = open(fullp, "rb") - fp.read(512) - fp.close() - except: - if fp: - fp.close() - continue - - if tmpd is None: - tmpd = tempfile.mkdtemp() - if dvnull is None: - try: - dvnull = open("/dev/null") - except: - pass - - cmd = ["mount", "-o", "ro", fullp, tmpd] - if require_iso: - cmd.extend(('-t', 'iso9660')) - - rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) - if rc: - continue - - (fname, contents) = get_ovf_env(tmpd) - - subprocess.call(["umount", tmpd]) - - if contents is not False: - os.rmdir(tmpd) - return(contents, fullp, fname) - - if tmpd: - os.rmdir(tmpd) - - if dvnull: - dvnull.close() - - return(False, None, None) - - -def transport_vmware_guestd(): - # http://blogs.vmware.com/vapp/2009/07/ \ - # selfconfiguration-and-the-ovf-environment.html - # try: - # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] - # (out, err) = subp(cmd) - # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') - # except: - # # would need to error check here and see why this failed - # # to know if log/error should be raised - # return(False, None, None) - return(False, None, None) - - -def findChild(node, filter_func): - ret = [] - if not node.hasChildNodes(): - return ret - for child in node.childNodes: - if filter_func(child): - ret.append(child) - return(ret) - - -def getProperties(environString): - dom = minidom.parseString(environString) - if dom.documentElement.localName != "Environment": - raise Exception("No Environment Node") - - if not dom.documentElement.hasChildNodes(): - raise Exception("No Child Nodes") - - envNsURI = "http://schemas.dmtf.org/ovf/environment/1" - - # could also check here that elem.namespaceURI == - # "http://schemas.dmtf.org/ovf/environment/1" - propSections = findChild(dom.documentElement, - lambda n: n.localName == "PropertySection") - - if len(propSections) == 0: - raise Exception("No 'PropertySection's") - - props = {} - propElems = findChild(propSections[0], lambda n: n.localName == "Property") - - for elem in propElems: - key = elem.attributes.getNamedItemNS(envNsURI, "key").value - val = elem.attributes.getNamedItemNS(envNsURI, "value").value - props[key] = val - - return(props) - - -datasources = ( - (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), - (DataSourceOVFNet, - (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), -) - - -# return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return(DataSource.list_from_depends(depends, datasources)) - - -if __name__ == "__main__": - def main(): - import sys - envStr = open(sys.argv[1]).read() - props = getProperties(envStr) - import pprint - pprint.pprint(props) - - md, ud, cfg = read_ovf_environment(envStr) - print "=== md ===" - pprint.pprint(md) - print "=== ud ===" - pprint.pprint(ud) - print "=== cfg ===" - pprint.pprint(cfg) - - main() diff --git a/cloudinit/sources/DataSource.py b/cloudinit/sources/DataSource.py new file mode 100644 index 00000000..e2a9150d --- /dev/null +++ b/cloudinit/sources/DataSource.py @@ -0,0 +1,214 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +DEP_FILESYSTEM = "FILESYSTEM" +DEP_NETWORK = "NETWORK" + +import cloudinit.UserDataHandler as ud +import cloudinit.util as util +import socket + + +class DataSource: + userdata = None + metadata = None + userdata_raw = None + cfgname = "" + # system config (passed in from cloudinit, + # cloud-config before input from the DataSource) + sys_cfg = {} + # datasource config, the cloud-config['datasource']['__name__'] + ds_cfg = {} # datasource config + + def __init__(self, sys_cfg=None): + if not self.cfgname: + name = str(self.__class__).split(".")[-1] + if name.startswith("DataSource"): + name = name[len("DataSource"):] + self.cfgname = name + if sys_cfg: + self.sys_cfg = sys_cfg + + self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, + ("datasource", self.cfgname), self.ds_cfg) + + def get_userdata(self): + if self.userdata == None: + self.userdata = ud.preprocess_userdata(self.userdata_raw) + return self.userdata + + def get_userdata_raw(self): + return(self.userdata_raw) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return({}) + + def get_public_ssh_keys(self): + keys = [] + if 'public-keys' not in self.metadata: + return([]) + + if isinstance(self.metadata['public-keys'], str): + return(str(self.metadata['public-keys']).splitlines()) + + if isinstance(self.metadata['public-keys'], list): + return(self.metadata['public-keys']) + + for _keyname, klist in self.metadata['public-keys'].items(): + # lp:506332 uec metadata service responds with + # data that makes boto populate a string for 'klist' rather + # than a list. + if isinstance(klist, str): + klist = [klist] + for pkey in klist: + # there is an empty string at the end of the keylist, trim it + if pkey: + keys.append(pkey) + + return(keys) + + def device_name_to_device(self, _name): + # translate a 'name' to a device + # the primary function at this point is on ec2 + # to consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + return(None) + + def get_locale(self): + return('en_US.UTF-8') + + def get_local_mirror(self): + return None + + def get_instance_id(self): + if 'instance-id' not in self.metadata: + return "iid-datasource" + return(self.metadata['instance-id']) + + def get_hostname(self, fqdn=False): + defdomain = "localdomain" + defhost = "localhost" + + domain = defdomain + if not 'local-hostname' in self.metadata: + + # this is somewhat questionable really. + # the cloud datasource was asked for a hostname + # and didn't have one. raising error might be more appropriate + # but instead, basically look up the existing hostname + toks = [] + + hostname = socket.gethostname() + + fqdn = util.get_fqdn_from_hosts(hostname) + + if fqdn and fqdn.find(".") > 0: + toks = str(fqdn).split(".") + elif hostname: + toks = [hostname, defdomain] + else: + toks = [defhost, defdomain] + + else: + # if there is an ipv4 address in 'local-hostname', then + # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx + lhost = self.metadata['local-hostname'] + if is_ipv4(lhost): + toks = "ip-%s" % lhost.replace(".", "-") + else: + toks = lhost.split(".") + + if len(toks) > 1: + hostname = toks[0] + domain = '.'.join(toks[1:]) + else: + hostname = toks[0] + + if fqdn: + return "%s.%s" % (hostname, domain) + else: + return hostname + + +# return a list of classes that have the same depends as 'depends' +# iterate through cfg_list, loading "DataSourceCollections" modules +# and calling their "get_datasource_list". +# return an ordered list of classes that match +# +# - modules must be named "DataSource", where 'item' is an entry +# in cfg_list +# - if pkglist is given, it will iterate try loading from that package +# ie, pkglist=[ "foo", "" ] +# will first try to load foo.DataSource +# then DataSource +def list_sources(cfg_list, depends, pkglist=None): + if pkglist is None: + pkglist = [] + retlist = [] + for ds_coll in cfg_list: + for pkg in pkglist: + if pkg: + pkg = "%s." % pkg + try: + mod = __import__("%sDataSource%s" % (pkg, ds_coll)) + if pkg: + mod = getattr(mod, "DataSource%s" % ds_coll) + lister = getattr(mod, "get_datasource_list") + retlist.extend(lister(depends)) + break + except: + raise + return(retlist) + + +# depends is a list of dependencies (DEP_FILESYSTEM) +# dslist is a list of 2 item lists +# dslist = [ +# ( class, ( depends-that-this-class-needs ) ) +# } +# it returns a list of 'class' that matched these deps exactly +# it is a helper function for DataSourceCollections +def list_from_depends(depends, dslist): + retlist = [] + depset = set(depends) + for elem in dslist: + (cls, deps) = elem + if depset == set(deps): + retlist.append(cls) + return(retlist) + + +def is_ipv4(instr): + """ determine if input string is a ipv4 address. return boolean""" + toks = instr.split('.') + if len(toks) != 4: + return False + + try: + toks = [x for x in toks if (int(x) < 256 and int(x) > 0)] + except: + return False + + return (len(toks) == 4) diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py new file mode 100644 index 00000000..5afdf7b6 --- /dev/null +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -0,0 +1,92 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Cosmin Luta +# +# Author: Cosmin Luta +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +from socket import inet_ntoa +import time +import boto.utils as boto_utils +from struct import pack + + +class DataSourceCloudStack(DataSource.DataSource): + api_ver = 'latest' + seeddir = base_seeddir + '/cs' + metadata_address = None + + def __init__(self, sys_cfg=None): + DataSource.DataSource.__init__(self, sys_cfg) + # Cloudstack has its metadata/userdata URLs located at + # http:///latest/ + self.metadata_address = "http://%s/" % self.get_default_gateway() + + def get_default_gateway(self): + """ Returns the default gateway ip address in the dotted format + """ + with open("/proc/net/route", "r") as f: + for line in f.readlines(): + items = line.split("\t") + if items[1] == "00000000": + # found the default route, get the gateway + gw = inet_ntoa(pack(" +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +import json +import subprocess + +DEFAULT_IID = "iid-dsconfigdrive" + + +class DataSourceConfigDrive(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/config_drive' + cfg = {} + userdata_raw = None + metadata = None + dsmode = "local" + + def __str__(self): + mstr = "DataSourceConfigDrive[%s]" % self.dsmode + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = None + md = {} + ud = "" + + defaults = {"instance-id": DEFAULT_IID, "dsmode": "pass"} + + if os.path.isdir(self.seeddir): + try: + (md, ud) = read_config_drive_dir(self.seeddir) + found = self.seeddir + except nonConfigDriveDir: + pass + + if not found: + dev = cfg_drive_device() + if dev: + try: + (md, ud) = util.mount_callback_umount(dev, + read_config_drive_dir) + found = dev + except (nonConfigDriveDir, util.mountFailedError): + pass + + if not found: + return False + + if 'dsconfig' in md: + self.cfg = md['dscfg'] + + md = util.mergedict(md, defaults) + + # update interfaces and ifup only on the local datasource + # this way the DataSourceConfigDriveNet doesn't do it also. + if 'network-interfaces' in md and self.dsmode == "local": + if md['dsmode'] == "pass": + log.info("updating network interfaces from configdrive") + else: + log.debug("updating network interfaces from configdrive") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = found + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return(self.metadata['public-keys']) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceConfigDriveNet(DataSourceConfigDrive): + dsmode = "net" + + +class nonConfigDriveDir(Exception): + pass + + +def cfg_drive_device(): + """ get the config drive device. return a string like '/dev/vdb' + or None (if there is no non-root device attached). This does not + check the contents, only reports that if there *were* a config_drive + attached, it would be this device. + per config_drive documentation, this is + "associated as the last available disk on the instance" + """ + + if 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' in os.environ: + return(os.environ['CLOUD_INIT_CONFIG_DRIVE_DEVICE']) + + # we are looking for a raw block device (sda, not sda1) with a vfat + # filesystem on it. + + letters = "abcdefghijklmnopqrstuvwxyz" + devs = util.find_devs_with("TYPE=vfat") + + # filter out anything not ending in a letter (ignore partitions) + devs = [f for f in devs if f[-1] in letters] + + # sort them in reverse so "last" device is first + devs.sort(reverse=True) + + if len(devs): + return(devs[0]) + + return(None) + + +def read_config_drive_dir(source_dir): + """ + read_config_drive_dir(source_dir): + read source_dir, and return a tuple with metadata dict and user-data + string populated. If not a valid dir, raise a nonConfigDriveDir + """ + md = {} + ud = "" + + flist = ("etc/network/interfaces", "root/.ssh/authorized_keys", "meta.js") + found = [f for f in flist if os.path.isfile("%s/%s" % (source_dir, f))] + keydata = "" + + if len(found) == 0: + raise nonConfigDriveDir("%s: %s" % (source_dir, "no files found")) + + if "etc/network/interfaces" in found: + with open("%s/%s" % (source_dir, "/etc/network/interfaces")) as fp: + md['network-interfaces'] = fp.read() + + if "root/.ssh/authorized_keys" in found: + with open("%s/%s" % (source_dir, "root/.ssh/authorized_keys")) as fp: + keydata = fp.read() + + meta_js = {} + + if "meta.js" in found: + content = '' + with open("%s/%s" % (source_dir, "meta.js")) as fp: + content = fp.read() + md['meta_js'] = content + try: + meta_js = json.loads(content) + except ValueError: + raise nonConfigDriveDir("%s: %s" % + (source_dir, "invalid json in meta.js")) + + keydata = meta_js.get('public-keys', keydata) + + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + for copy in ('dsmode', 'instance-id', 'dscfg'): + if copy in meta_js: + md[copy] = meta_js[copy] + + if 'user-data' in meta_js: + ud = meta_js['user-data'] + + return(md, ud) + +datasources = ( + (DataSourceConfigDrive, (DataSource.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + +if __name__ == "__main__": + def main(): + import sys + import pprint + print cfg_drive_device() + (md, ud) = read_config_drive_dir(sys.argv[1]) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + print(ud) + + main() + +# vi: ts=4 expandtab diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py new file mode 100644 index 00000000..7051ecda --- /dev/null +++ b/cloudinit/sources/DataSourceEc2.py @@ -0,0 +1,217 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import socket +import time +import boto.utils as boto_utils +import os.path + + +class DataSourceEc2(DataSource.DataSource): + api_ver = '2009-04-04' + seeddir = base_seeddir + '/ec2' + metadata_address = "http://169.254.169.254" + + def __str__(self): + return("DataSourceEc2") + + def get_data(self): + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + self.userdata_raw = seedret['user-data'] + self.metadata = seedret['meta-data'] + log.debug("using seeded ec2 data in %s" % self.seeddir) + return True + + try: + if not self.wait_for_metadata_service(): + return False + start = time.time() + self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, + None, self.metadata_address) + self.metadata = boto_utils.get_instance_metadata(self.api_ver, + self.metadata_address) + log.debug("crawl of metadata service took %ds" % (time.time() - + start)) + return True + except Exception as e: + print e + return False + + def get_instance_id(self): + return(self.metadata['instance-id']) + + def get_availability_zone(self): + return(self.metadata['placement']['availability-zone']) + + def get_local_mirror(self): + return(self.get_mirror_from_availability_zone()) + + def get_mirror_from_availability_zone(self, availability_zone=None): + # availability is like 'us-west-1b' or 'eu-west-1a' + if availability_zone == None: + availability_zone = self.get_availability_zone() + + fallback = None + + if self.is_vpc(): + return fallback + + try: + host = "%s.ec2.archive.ubuntu.com" % availability_zone[:-1] + socket.getaddrinfo(host, None, 0, socket.SOCK_STREAM) + return 'http://%s/ubuntu/' % host + except: + return fallback + + def wait_for_metadata_service(self): + mcfg = self.ds_cfg + + if not hasattr(mcfg, "get"): + mcfg = {} + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + def_mdurls = ["http://169.254.169.254", "http://instance-data:8773"] + mdurls = mcfg.get("metadata_urls", def_mdurls) + + # Remove addresses from the list that wont resolve. + filtered = [x for x in mdurls if util.is_resolvable_url(x)] + + if set(filtered) != set(mdurls): + log.debug("removed the following from metadata urls: %s" % + list((set(mdurls) - set(filtered)))) + + if len(filtered): + mdurls = filtered + else: + log.warn("Empty metadata url list! using default list") + mdurls = def_mdurls + + urls = [] + url2base = {False: False} + for url in mdurls: + cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver) + urls.append(cur) + url2base[cur] = url + + starttime = time.time() + url = util.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=log.warn) + + if url: + log.debug("Using metadata source: '%s'" % url2base[url]) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + self.metadata_address = url2base[url] + return (bool(url)) + + def device_name_to_device(self, name): + # consult metadata service, that has + # ephemeral0: sdb + # and return 'sdb' for input 'ephemeral0' + if 'block-device-mapping' not in self.metadata: + return(None) + + found = None + for entname, device in self.metadata['block-device-mapping'].items(): + if entname == name: + found = device + break + # LP: #513842 mapping in Euca has 'ephemeral' not 'ephemeral0' + if entname == "ephemeral" and name == "ephemeral0": + found = device + if found == None: + log.debug("unable to convert %s to a device" % name) + return None + + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + ofound = found + short = os.path.basename(found) + + if not found.startswith("/"): + found = "/dev/%s" % found + + if os.path.exists(found): + return(found) + + for nfrom, tlist in mappings.items(): + if not short.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short[len(nfrom):]) + if os.path.exists(cand): + log.debug("remapped device name %s => %s" % (found, cand)) + return(cand) + + # on t1.micro, ephemeral0 will appear in block-device-mapping from + # metadata, but it will not exist on disk (and never will) + # at this pint, we've verified that the path did not exist + # in the special case of 'ephemeral0' return None to avoid bogus + # fstab entry (LP: #744019) + if name == "ephemeral0": + return None + return ofound + + def is_vpc(self): + # per comment in LP: #615545 + ph = "public-hostname" + p4 = "public-ipv4" + if ((ph not in self.metadata or self.metadata[ph] == "") and + (p4 not in self.metadata or self.metadata[p4] == "")): + return True + return False + + +datasources = [ + (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py new file mode 100644 index 00000000..61a0038f --- /dev/null +++ b/cloudinit/sources/DataSourceMAAS.py @@ -0,0 +1,345 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import oauth.oauth as oauth +import os.path +import urllib2 +import time + + +MD_VERSION = "2012-03-01" + + +class DataSourceMAAS(DataSource.DataSource): + """ + DataSourceMAAS reads instance information from MAAS. + Given a config metadata_url, and oauth tokens, it expects to find + files under the root named: + instance-id + user-data + hostname + """ + seeddir = base_seeddir + '/maas' + baseurl = None + + def __str__(self): + return("DataSourceMAAS[%s]" % self.baseurl) + + def get_data(self): + mcfg = self.ds_cfg + + try: + (userdata, metadata) = read_maas_seed_dir(self.seeddir) + self.userdata_raw = userdata + self.metadata = metadata + self.baseurl = self.seeddir + return True + except MAASSeedDirNone: + pass + except MAASSeedDirMalformed as exc: + log.warn("%s was malformed: %s\n" % (self.seeddir, exc)) + raise + + try: + # if there is no metadata_url, then we're not configured + url = mcfg.get('metadata_url', None) + if url == None: + return False + + if not self.wait_for_metadata_service(url): + return False + + self.baseurl = url + + (userdata, metadata) = read_maas_seed_url(self.baseurl, + self.md_headers) + self.userdata_raw = userdata + self.metadata = metadata + return True + except Exception: + util.logexc(log) + return False + + def md_headers(self, url): + mcfg = self.ds_cfg + + # if we are missing token_key, token_secret or consumer_key + # then just do non-authed requests + for required in ('token_key', 'token_secret', 'consumer_key'): + if required not in mcfg: + return({}) + + consumer_secret = mcfg.get('consumer_secret', "") + + return(oauth_headers(url=url, consumer_key=mcfg['consumer_key'], + token_key=mcfg['token_key'], token_secret=mcfg['token_secret'], + consumer_secret=consumer_secret)) + + def wait_for_metadata_service(self, url): + mcfg = self.ds_cfg + + max_wait = 120 + try: + max_wait = int(mcfg.get("max_wait", max_wait)) + except Exception: + util.logexc(log) + log.warn("Failed to get max wait. using %s" % max_wait) + + if max_wait == 0: + return False + + timeout = 50 + try: + timeout = int(mcfg.get("timeout", timeout)) + except Exception: + util.logexc(log) + log.warn("Failed to get timeout, using %s" % timeout) + + starttime = time.time() + check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) + url = util.wait_for_url(urls=[check_url], max_wait=max_wait, + timeout=timeout, status_cb=log.warn, + headers_cb=self.md_headers) + + if url: + log.debug("Using metadata source: '%s'" % url) + else: + log.critical("giving up on md after %i seconds\n" % + int(time.time() - starttime)) + + return (bool(url)) + + +def read_maas_seed_dir(seed_d): + """ + Return user-data and metadata for a maas seed dir in seed_d. + Expected format of seed_d are the following files: + * instance-id + * local-hostname + * user-data + """ + files = ('local-hostname', 'instance-id', 'user-data', 'public-keys') + md = {} + + if not os.path.isdir(seed_d): + raise MAASSeedDirNone("%s: not a directory") + + for fname in files: + try: + with open(os.path.join(seed_d, fname)) as fp: + md[fname] = fp.read() + fp.close() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + return(check_seed_contents(md, seed_d)) + + +def read_maas_seed_url(seed_url, header_cb=None, timeout=None, + version=MD_VERSION): + """ + Read the maas datasource at seed_url. + header_cb is a method that should return a headers dictionary that will + be given to urllib2.Request() + + Expected format of seed_url is are the following files: + * //meta-data/instance-id + * //meta-data/local-hostname + * //user-data + """ + files = ('meta-data/local-hostname', + 'meta-data/instance-id', + 'meta-data/public-keys', + 'user-data') + + base_url = "%s/%s" % (seed_url, version) + md = {} + for fname in files: + url = "%s/%s" % (base_url, fname) + if header_cb: + headers = header_cb(url) + else: + headers = {} + + try: + req = urllib2.Request(url, data=None, headers=headers) + resp = urllib2.urlopen(req, timeout=timeout) + md[os.path.basename(fname)] = resp.read() + except urllib2.HTTPError as e: + if e.code != 404: + raise + + return(check_seed_contents(md, seed_url)) + + +def check_seed_contents(content, seed): + """Validate if content is Is the content a dict that is valid as a + return for a datasource. + Either return a (userdata, metadata) tuple or + Raise MAASSeedDirMalformed or MAASSeedDirNone + """ + md_required = ('instance-id', 'local-hostname') + found = content.keys() + + if len(content) == 0: + raise MAASSeedDirNone("%s: no data files found" % seed) + + missing = [k for k in md_required if k not in found] + if len(missing): + raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing)) + + userdata = content.get('user-data', "") + md = {} + for (key, val) in content.iteritems(): + if key == 'user-data': + continue + md[key] = val + + return(userdata, md) + + +def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret): + consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) + token = oauth.OAuthToken(token_key, token_secret) + params = { + 'oauth_version': "1.0", + 'oauth_nonce': oauth.generate_nonce(), + 'oauth_timestamp': int(time.time()), + 'oauth_token': token.key, + 'oauth_consumer_key': consumer.key, + } + req = oauth.OAuthRequest(http_url=url, parameters=params) + req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(), + consumer, token) + return(req.to_header()) + + +class MAASSeedDirNone(Exception): + pass + + +class MAASSeedDirMalformed(Exception): + pass + + +datasources = [ + (DataSourceMAAS, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +] + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + """ + Call with single argument of directory or http or https url. + If url is given additional arguments are allowed, which will be + interpreted as consumer_key, token_key, token_secret, consumer_secret + """ + import argparse + import pprint + + parser = argparse.ArgumentParser(description='Interact with MAAS DS') + parser.add_argument("--config", metavar="file", + help="specify DS config file", default=None) + parser.add_argument("--ckey", metavar="key", + help="the consumer key to auth with", default=None) + parser.add_argument("--tkey", metavar="key", + help="the token key to auth with", default=None) + parser.add_argument("--csec", metavar="secret", + help="the consumer secret (likely '')", default="") + parser.add_argument("--tsec", metavar="secret", + help="the token secret to auth with", default=None) + parser.add_argument("--apiver", metavar="version", + help="the apiver to use ("" can be used)", default=MD_VERSION) + + subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") + subcmds.add_parser('crawl', help="crawl the datasource") + subcmds.add_parser('get', help="do a single GET of provided url") + subcmds.add_parser('check-seed', help="read andn verify seed at url") + + parser.add_argument("url", help="the data source to query") + + args = parser.parse_args() + + creds = {'consumer_key': args.ckey, 'token_key': args.tkey, + 'token_secret': args.tsec, 'consumer_secret': args.csec} + + if args.config: + import yaml + with open(args.config) as fp: + cfg = yaml.load(fp) + if 'datasource' in cfg: + cfg = cfg['datasource']['MAAS'] + for key in creds.keys(): + if key in cfg and creds[key] == None: + creds[key] = cfg[key] + + def geturl(url, headers_cb): + req = urllib2.Request(url, data=None, headers=headers_cb(url)) + return(urllib2.urlopen(req).read()) + + def printurl(url, headers_cb): + print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + + def crawl(url, headers_cb=None): + if url.endswith("/"): + for line in geturl(url, headers_cb).splitlines(): + if line.endswith("/"): + crawl("%s%s" % (url, line), headers_cb) + else: + printurl("%s%s" % (url, line), headers_cb) + else: + printurl(url, headers_cb) + + def my_headers(url): + headers = {} + if creds.get('consumer_key', None) != None: + headers = oauth_headers(url, **creds) + return headers + + if args.subcmd == "check-seed": + if args.url.startswith("http"): + (userdata, metadata) = read_maas_seed_url(args.url, + header_cb=my_headers, version=args.apiver) + else: + (userdata, metadata) = read_maas_seed_url(args.url) + print "=== userdata ===" + print userdata + print "=== metadata ===" + pprint.pprint(metadata) + + elif args.subcmd == "get": + printurl(args.url, my_headers) + + elif args.subcmd == "crawl": + if not args.url.endswith("/"): + args.url = "%s/" % args.url + crawl(args.url, my_headers) + + main() diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py new file mode 100644 index 00000000..e8c56b8f --- /dev/null +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -0,0 +1,232 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import errno +import subprocess + + +class DataSourceNoCloud(DataSource.DataSource): + metadata = None + userdata = None + userdata_raw = None + supported_seed_starts = ("/", "file://") + dsmode = "local" + seed = None + cmdline_id = "ds=nocloud" + seeddir = base_seeddir + '/nocloud' + + def __str__(self): + mstr = "DataSourceNoCloud" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + defaults = { + "instance-id": "nocloud", "dsmode": self.dsmode + } + + found = [] + md = {} + ud = "" + + try: + # parse the kernel command line, getting data passed in + if parse_cmdline_data(self.cmdline_id, md): + found.append("cmdline") + except: + util.logexc(log) + return False + + # check to see if the seeddir has data. + seedret = {} + if util.read_optional_seed(seedret, base=self.seeddir + "/"): + md = util.mergedict(md, seedret['meta-data']) + ud = seedret['user-data'] + found.append(self.seeddir) + log.debug("using seeded cache data in %s" % self.seeddir) + + # if the datasource config had a 'seedfrom' entry, then that takes + # precedence over a 'seedfrom' that was found in a filesystem + # but not over external medi + if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']: + found.append("ds_config") + md["seedfrom"] = self.ds_cfg['seedfrom'] + + fslist = util.find_devs_with("TYPE=vfat") + fslist.extend(util.find_devs_with("TYPE=iso9660")) + + label_list = util.find_devs_with("LABEL=cidata") + devlist = list(set(fslist) & set(label_list)) + devlist.sort(reverse=True) + + for dev in devlist: + try: + (newmd, newud) = util.mount_callback_umount(dev, + util.read_seeded) + md = util.mergedict(newmd, md) + ud = newud + + # for seed from a device, the default mode is 'net'. + # that is more likely to be what is desired. + # If they want dsmode of local, then they must + # specify that. + if 'dsmode' not in md: + md['dsmode'] = "net" + + log.debug("using data from %s" % dev) + found.append(dev) + break + except OSError, e: + if e.errno != errno.ENOENT: + raise + except util.mountFailedError: + log.warn("Failed to mount %s when looking for seed" % dev) + + # there was no indication on kernel cmdline or data + # in the seeddir suggesting this handler should be used. + if len(found) == 0: + return False + + seeded_interfaces = None + + # the special argument "seedfrom" indicates we should + # attempt to seed the userdata / metadata from its value + # its primarily value is in allowing the user to type less + # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg + if "seedfrom" in md: + seedfrom = md["seedfrom"] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + if 'network-interfaces' in md: + seeded_interfaces = self.dsmode + + # this could throw errors, but the user told us to do it + # so if errors are raised, let them raise + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + # values in the command line override those from the seed + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + + # update the network-interfaces if metadata had 'network-interfaces' + # entry and this is the local datasource, or 'seedfrom' was used + # and the source of the seed was self.dsmode + # ('local' for NoCloud, 'net' for NoCloudNet') + if ('network-interfaces' in md and + (self.dsmode in ("local", seeded_interfaces))): + log.info("updating network interfaces from nocloud") + + util.write_file("/etc/network/interfaces", + md['network-interfaces']) + try: + (out, err) = util.subp(['ifup', '--all']) + if len(out) or len(err): + log.warn("ifup --all had stderr: %s" % err) + + except subprocess.CalledProcessError as exc: + log.warn("ifup --all failed: %s" % (exc.output[1])) + + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + + if md['dsmode'] == self.dsmode: + return True + + log.debug("%s: not claiming datasource, dsmode=%s" % + (self, md['dsmode'])) + return False + + +# returns true or false indicating if cmdline indicated +# that this module should be used +# example cmdline: +# root=LABEL=uec-rootfs ro ds=nocloud +def parse_cmdline_data(ds_id, fill, cmdline=None): + if cmdline is None: + cmdline = util.get_cmdline() + cmdline = " %s " % cmdline + + if not (" %s " % ds_id in cmdline or " %s;" % ds_id in cmdline): + return False + + argline = "" + # cmdline can contain: + # ds=nocloud[;key=val;key=val] + for tok in cmdline.split(): + if tok.startswith(ds_id): + argline = tok.split("=", 1) + + # argline array is now 'nocloud' followed optionally by + # a ';' and then key=value pairs also terminated with ';' + tmp = argline[1].split(";") + if len(tmp) > 1: + kvpairs = tmp[1:] + else: + kvpairs = () + + # short2long mapping to save cmdline typing + s2l = {"h": "local-hostname", "i": "instance-id", "s": "seedfrom"} + for item in kvpairs: + try: + (k, v) = item.split("=", 1) + except: + k = item + v = None + if k in s2l: + k = s2l[k] + fill[k] = v + + return(True) + + +class DataSourceNoCloudNet(DataSourceNoCloud): + cmdline_id = "ds=nocloud-net" + supported_seed_starts = ("http://", "https://", "ftp://") + seeddir = base_seeddir + '/nocloud-net' + dsmode = "net" + + +datasources = ( + (DataSourceNoCloud, (DataSource.DEP_FILESYSTEM, )), + (DataSourceNoCloudNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py new file mode 100644 index 00000000..a0b1b518 --- /dev/null +++ b/cloudinit/sources/DataSourceOVF.py @@ -0,0 +1,332 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Hafliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.DataSource as DataSource + +from cloudinit import seeddir as base_seeddir +from cloudinit import log +import cloudinit.util as util +import os.path +import os +from xml.dom import minidom +import base64 +import re +import tempfile +import subprocess + + +class DataSourceOVF(DataSource.DataSource): + seed = None + seeddir = base_seeddir + '/ovf' + environment = None + cfg = {} + userdata_raw = None + metadata = None + supported_seed_starts = ("/", "file://") + + def __str__(self): + mstr = "DataSourceOVF" + mstr = mstr + " [seed=%s]" % self.seed + return(mstr) + + def get_data(self): + found = [] + md = {} + ud = "" + + defaults = { + "instance-id": "iid-dsovf" + } + + (seedfile, contents) = get_ovf_env(base_seeddir) + if seedfile: + # found a seed dir + seed = "%s/%s" % (base_seeddir, seedfile) + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + + found.append(seed) + else: + np = {'iso': transport_iso9660, + 'vmware-guestd': transport_vmware_guestd, } + name = None + for name, transfunc in np.iteritems(): + (contents, _dev, _fname) = transfunc() + if contents: + break + + if contents: + (md, ud, cfg) = read_ovf_environment(contents) + self.environment = contents + found.append(name) + + # There was no OVF transports found + if len(found) == 0: + return False + + if 'seedfrom' in md and md['seedfrom']: + seedfrom = md['seedfrom'] + seedfound = False + for proto in self.supported_seed_starts: + if seedfrom.startswith(proto): + seedfound = proto + break + if not seedfound: + log.debug("seed from %s not supported by %s" % + (seedfrom, self.__class__)) + return False + + (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) + log.debug("using seeded cache data from %s" % seedfrom) + + md = util.mergedict(md, md_seed) + found.append(seedfrom) + + md = util.mergedict(md, defaults) + self.seed = ",".join(found) + self.metadata = md + self.userdata_raw = ud + self.cfg = cfg + return True + + def get_public_ssh_keys(self): + if not 'public-keys' in self.metadata: + return([]) + return([self.metadata['public-keys'], ]) + + # the data sources' config_obj is a cloud-config formated + # object that came to it from ways other than cloud-config + # because cloud-config content would be handled elsewhere + def get_config_obj(self): + return(self.cfg) + + +class DataSourceOVFNet(DataSourceOVF): + seeddir = base_seeddir + '/ovf-net' + supported_seed_starts = ("http://", "https://", "ftp://") + + +# this will return a dict with some content +# meta-data, user-data +def read_ovf_environment(contents): + props = getProperties(contents) + md = {} + cfg = {} + ud = "" + cfg_props = ['password', ] + md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] + for prop, val in props.iteritems(): + if prop == 'hostname': + prop = "local-hostname" + if prop in md_props: + md[prop] = val + elif prop in cfg_props: + cfg[prop] = val + elif prop == "user-data": + try: + ud = base64.decodestring(val) + except: + ud = val + return(md, ud, cfg) + + +# returns tuple of filename (in 'dirname', and the contents of the file) +# on "not found", returns 'None' for filename and False for contents +def get_ovf_env(dirname): + env_names = ("ovf-env.xml", "ovf_env.xml", "OVF_ENV.XML", "OVF-ENV.XML") + for fname in env_names: + if os.path.isfile("%s/%s" % (dirname, fname)): + fp = open("%s/%s" % (dirname, fname)) + contents = fp.read() + fp.close() + return(fname, contents) + return(None, False) + + +# transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + + # default_regex matches values in + # /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + envname = "CLOUD_INIT_CDROM_DEV_REGEX" + default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + + devname_regex = os.environ.get(envname, default_regex) + cdmatch = re.compile(devname_regex) + + # go through mounts to see if it was already mounted + fp = open("/proc/mounts") + mounts = fp.readlines() + fp.close() + + mounted = {} + for mpline in mounts: + (dev, mp, fstype, _opts, _freq, _passno) = mpline.split() + mounted[dev] = (dev, fstype, mp, False) + mp = mp.replace("\\040", " ") + if fstype != "iso9660" and require_iso: + continue + + if cdmatch.match(dev[5:]) == None: # take off '/dev/' + continue + + (fname, contents) = get_ovf_env(mp) + if contents is not False: + return(contents, dev, fname) + + tmpd = None + dvnull = None + + devs = os.listdir("/dev/") + devs.sort() + + for dev in devs: + fullp = "/dev/%s" % dev + + if fullp in mounted or not cdmatch.match(dev) or os.path.isdir(fullp): + continue + + fp = None + try: + fp = open(fullp, "rb") + fp.read(512) + fp.close() + except: + if fp: + fp.close() + continue + + if tmpd is None: + tmpd = tempfile.mkdtemp() + if dvnull is None: + try: + dvnull = open("/dev/null") + except: + pass + + cmd = ["mount", "-o", "ro", fullp, tmpd] + if require_iso: + cmd.extend(('-t', 'iso9660')) + + rc = subprocess.call(cmd, stderr=dvnull, stdout=dvnull, stdin=dvnull) + if rc: + continue + + (fname, contents) = get_ovf_env(tmpd) + + subprocess.call(["umount", tmpd]) + + if contents is not False: + os.rmdir(tmpd) + return(contents, fullp, fname) + + if tmpd: + os.rmdir(tmpd) + + if dvnull: + dvnull.close() + + return(False, None, None) + + +def transport_vmware_guestd(): + # http://blogs.vmware.com/vapp/2009/07/ \ + # selfconfiguration-and-the-ovf-environment.html + # try: + # cmd = ['vmware-guestd', '--cmd', 'info-get guestinfo.ovfEnv'] + # (out, err) = subp(cmd) + # return(out, 'guestinfo.ovfEnv', 'vmware-guestd') + # except: + # # would need to error check here and see why this failed + # # to know if log/error should be raised + # return(False, None, None) + return(False, None, None) + + +def findChild(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return(ret) + + +def getProperties(environString): + dom = minidom.parseString(environString) + if dom.documentElement.localName != "Environment": + raise Exception("No Environment Node") + + if not dom.documentElement.hasChildNodes(): + raise Exception("No Child Nodes") + + envNsURI = "http://schemas.dmtf.org/ovf/environment/1" + + # could also check here that elem.namespaceURI == + # "http://schemas.dmtf.org/ovf/environment/1" + propSections = findChild(dom.documentElement, + lambda n: n.localName == "PropertySection") + + if len(propSections) == 0: + raise Exception("No 'PropertySection's") + + props = {} + propElems = findChild(propSections[0], lambda n: n.localName == "Property") + + for elem in propElems: + key = elem.attributes.getNamedItemNS(envNsURI, "key").value + val = elem.attributes.getNamedItemNS(envNsURI, "value").value + props[key] = val + + return(props) + + +datasources = ( + (DataSourceOVF, (DataSource.DEP_FILESYSTEM, )), + (DataSourceOVFNet, + (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)), +) + + +# return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return(DataSource.list_from_depends(depends, datasources)) + + +if __name__ == "__main__": + def main(): + import sys + envStr = open(sys.argv[1]).read() + props = getProperties(envStr) + import pprint + pprint.pprint(props) + + md, ud, cfg = read_ovf_environment(envStr) + print "=== md ===" + pprint.pprint(md) + print "=== ud ===" + pprint.pprint(ud) + print "=== cfg ===" + pprint.pprint(cfg) + + main() diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py new file mode 100644 index 00000000..e69de29b -- cgit v1.2.3 From 8cd2871e7d7885076cc4a760d44323e480eb1b9a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:04:56 -0700 Subject: Remove cloud config from here and move to cloud.py --- cloudinit/handlers/__init__.py | 60 +++++++----------------------------------- 1 file changed, 9 insertions(+), 51 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index a16bdde6..3b0cdd4e 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -19,65 +19,23 @@ # along with this program. If not, see . # -import yaml -import cloudinit -import cloudinit.util as util -import sys -import traceback import os import subprocess +import sys import time +import traceback -per_instance = cloudinit.per_instance -per_always = cloudinit.per_always -per_once = cloudinit.per_once - - -class CloudConfig(): - cfgfile = None - cfg = None - - def __init__(self, cfgfile, cloud=None, ds_deps=None): - if cloud == None: - self.cloud = cloudinit.CloudInit(ds_deps) - self.cloud.get_data_source() - else: - self.cloud = cloud - self.cfg = self.get_config_obj(cfgfile) - - def get_config_obj(self, cfgfile): - try: - cfg = util.read_conf(cfgfile) - except: - # TODO: this 'log' could/should be passed in - cloudinit.log.critical("Failed loading of cloud config '%s'. " - "Continuing with empty config\n" % cfgfile) - cloudinit.log.debug(traceback.format_exc() + "\n") - cfg = None - if cfg is None: - cfg = {} - - try: - ds_cfg = self.cloud.datasource.get_config_obj() - except: - ds_cfg = {} +import yaml - cfg = util.mergedict(cfg, ds_cfg) - return(util.mergedict(cfg, self.cloud.cfg)) +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) - def handle(self, name, args, freq=None): - try: - mod = __import__("cc_" + name.replace("-", "_"), globals()) - def_freq = getattr(mod, "frequency", per_instance) - handler = getattr(mod, "handle") +from cloudinit import log as logging +from cloudinit import util - if not freq: - freq = def_freq +LOG = logging.getLogger(__name__) - self.cloud.sem_and_run("config-" + name, freq, handler, - [name, self.cfg, self.cloud, cloudinit.log, args]) - except: - raise +DEF_HANDLER_VERSION = 1 +DEF_FREQ = PER_INSTANCE # reads a cloudconfig module list, returns -- cgit v1.2.3 From 65a2249c9ba9503afe8c2da06afd54a63cf68fbd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 8 Jun 2012 18:42:54 -0700 Subject: Move how handler module names are found to here as well as a fixup module function for bad modules. --- cloudinit/handlers/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 3b0cdd4e..ae74b683 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -36,6 +36,7 @@ LOG = logging.getLogger(__name__) DEF_HANDLER_VERSION = 1 DEF_FREQ = PER_INSTANCE +HANDLER_TPL = "cc_%s" # reads a cloudconfig module list, returns @@ -230,3 +231,25 @@ def update_package_sources(): def install_packages(pkglist): update_package_sources() apt_get("install", pkglist) + + +def form_module_name(name): + canon_name = name.replace("-", "_") + if canon_name.endswith(".py"): + canon_name = canon_name[0:(len(canon_name) - 3)] + canon_name = canon_name.strip() + if not canon_name: + return None + return HANDLER_TPL % (canon_name) + + +def fixup_module(mod): + freq = getattr(mod, "frequency", None) + if not freq: + setattr(mod, 'frequency', PER_INSTANCE) + handler = getattr(mod, "handle", None) + if not handler: + def empty_handle(_name, _cfg, _cloud, _log, _args): + pass + setattr(mod, 'handle', empty_handle) + return mod -- cgit v1.2.3 From 82243cdf875cd7d8b2fbd962fa92ab139c653043 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:15:16 -0700 Subject: Removing distro specifics from here, as well as running functionality (which is now elsewhere). --- cloudinit/handlers/__init__.py | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index ae74b683..5d70ac43 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -36,7 +36,6 @@ LOG = logging.getLogger(__name__) DEF_HANDLER_VERSION = 1 DEF_FREQ = PER_INSTANCE -HANDLER_TPL = "cc_%s" # reads a cloudconfig module list, returns @@ -198,41 +197,6 @@ def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): return -def run_per_instance(name, func, args, clear_on_fail=False): - semfile = "%s/%s" % (cloudinit.get_ipath_cur("data"), name) - if os.path.exists(semfile): - return - - util.write_file(semfile, str(time.time())) - try: - func(*args) - except: - if clear_on_fail: - os.unlink(semfile) - raise - - -# apt_get top level command (install, update...), and args to pass it -def apt_get(tlc, args=None): - if args is None: - args = [] - e = os.environ.copy() - e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', tlc] - cmd.extend(args) - subprocess.check_call(cmd, env=e) - - -def update_package_sources(): - run_per_instance("update-sources", apt_get, ("update",)) - - -def install_packages(pkglist): - update_package_sources() - apt_get("install", pkglist) - - def form_module_name(name): canon_name = name.replace("-", "_") if canon_name.endswith(".py"): @@ -240,7 +204,9 @@ def form_module_name(name): canon_name = canon_name.strip() if not canon_name: return None - return HANDLER_TPL % (canon_name) + if not canon_name.startswith("cc_"): + canon_name = 'cc_%s' % (canon_name) + return canon_name def fixup_module(mod): -- cgit v1.2.3 From 4d74f499a0257a4f27f7def4b76402bcbde63567 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:35:07 -0700 Subject: Moved the handlers from a user_data directory to a handler directory. --- cloudinit/handlers/__init__.py | 221 ---------------------- cloudinit/handlers/boot_hook.py | 66 +++++++ cloudinit/handlers/cc_apt_pipelining.py | 53 ------ cloudinit/handlers/cc_apt_update_upgrade.py | 241 ------------------------ cloudinit/handlers/cc_bootcmd.py | 48 ----- cloudinit/handlers/cc_byobu.py | 77 -------- cloudinit/handlers/cc_ca_certs.py | 90 --------- cloudinit/handlers/cc_chef.py | 119 ------------ cloudinit/handlers/cc_disable_ec2_metadata.py | 30 --- cloudinit/handlers/cc_final_message.py | 58 ------ cloudinit/handlers/cc_foo.py | 29 --- cloudinit/handlers/cc_grub_dpkg.py | 64 ------- cloudinit/handlers/cc_keys_to_console.py | 42 ----- cloudinit/handlers/cc_landscape.py | 75 -------- cloudinit/handlers/cc_locale.py | 54 ------ cloudinit/handlers/cc_mcollective.py | 99 ---------- cloudinit/handlers/cc_mounts.py | 179 ------------------ cloudinit/handlers/cc_phone_home.py | 106 ----------- cloudinit/handlers/cc_puppet.py | 108 ----------- cloudinit/handlers/cc_resizefs.py | 108 ----------- cloudinit/handlers/cc_rightscale_userdata.py | 78 -------- cloudinit/handlers/cc_rsyslog.py | 101 ---------- cloudinit/handlers/cc_runcmd.py | 32 ---- cloudinit/handlers/cc_salt_minion.py | 56 ------ cloudinit/handlers/cc_scripts_per_boot.py | 34 ---- cloudinit/handlers/cc_scripts_per_instance.py | 34 ---- cloudinit/handlers/cc_scripts_per_once.py | 34 ---- cloudinit/handlers/cc_scripts_user.py | 34 ---- cloudinit/handlers/cc_set_hostname.py | 42 ----- cloudinit/handlers/cc_set_passwords.py | 129 ------------- cloudinit/handlers/cc_ssh.py | 106 ----------- cloudinit/handlers/cc_ssh_import_id.py | 50 ----- cloudinit/handlers/cc_timezone.py | 67 ------- cloudinit/handlers/cc_update_etc_hosts.py | 87 --------- cloudinit/handlers/cc_update_hostname.py | 101 ---------- cloudinit/transforms/__init__.py | 221 ++++++++++++++++++++++ cloudinit/transforms/cc_apt_pipelining.py | 53 ++++++ cloudinit/transforms/cc_apt_update_upgrade.py | 241 ++++++++++++++++++++++++ cloudinit/transforms/cc_bootcmd.py | 48 +++++ cloudinit/transforms/cc_byobu.py | 77 ++++++++ cloudinit/transforms/cc_ca_certs.py | 90 +++++++++ cloudinit/transforms/cc_chef.py | 119 ++++++++++++ cloudinit/transforms/cc_disable_ec2_metadata.py | 30 +++ cloudinit/transforms/cc_final_message.py | 58 ++++++ cloudinit/transforms/cc_foo.py | 29 +++ cloudinit/transforms/cc_grub_dpkg.py | 64 +++++++ cloudinit/transforms/cc_keys_to_console.py | 42 +++++ cloudinit/transforms/cc_landscape.py | 75 ++++++++ cloudinit/transforms/cc_locale.py | 54 ++++++ cloudinit/transforms/cc_mcollective.py | 99 ++++++++++ cloudinit/transforms/cc_mounts.py | 179 ++++++++++++++++++ cloudinit/transforms/cc_phone_home.py | 106 +++++++++++ cloudinit/transforms/cc_puppet.py | 108 +++++++++++ cloudinit/transforms/cc_resizefs.py | 108 +++++++++++ cloudinit/transforms/cc_rightscale_userdata.py | 78 ++++++++ cloudinit/transforms/cc_rsyslog.py | 101 ++++++++++ cloudinit/transforms/cc_runcmd.py | 32 ++++ cloudinit/transforms/cc_salt_minion.py | 56 ++++++ cloudinit/transforms/cc_scripts_per_boot.py | 34 ++++ cloudinit/transforms/cc_scripts_per_instance.py | 34 ++++ cloudinit/transforms/cc_scripts_per_once.py | 34 ++++ cloudinit/transforms/cc_scripts_user.py | 34 ++++ cloudinit/transforms/cc_set_hostname.py | 42 +++++ cloudinit/transforms/cc_set_passwords.py | 129 +++++++++++++ cloudinit/transforms/cc_ssh.py | 106 +++++++++++ cloudinit/transforms/cc_ssh_import_id.py | 50 +++++ cloudinit/transforms/cc_timezone.py | 67 +++++++ cloudinit/transforms/cc_update_etc_hosts.py | 87 +++++++++ cloudinit/transforms/cc_update_hostname.py | 101 ++++++++++ cloudinit/user_data/boot_hook.py | 66 ------- 70 files changed, 2852 insertions(+), 2852 deletions(-) delete mode 100644 cloudinit/handlers/__init__.py create mode 100644 cloudinit/handlers/boot_hook.py delete mode 100644 cloudinit/handlers/cc_apt_pipelining.py delete mode 100644 cloudinit/handlers/cc_apt_update_upgrade.py delete mode 100644 cloudinit/handlers/cc_bootcmd.py delete mode 100644 cloudinit/handlers/cc_byobu.py delete mode 100644 cloudinit/handlers/cc_ca_certs.py delete mode 100644 cloudinit/handlers/cc_chef.py delete mode 100644 cloudinit/handlers/cc_disable_ec2_metadata.py delete mode 100644 cloudinit/handlers/cc_final_message.py delete mode 100644 cloudinit/handlers/cc_foo.py delete mode 100644 cloudinit/handlers/cc_grub_dpkg.py delete mode 100644 cloudinit/handlers/cc_keys_to_console.py delete mode 100644 cloudinit/handlers/cc_landscape.py delete mode 100644 cloudinit/handlers/cc_locale.py delete mode 100644 cloudinit/handlers/cc_mcollective.py delete mode 100644 cloudinit/handlers/cc_mounts.py delete mode 100644 cloudinit/handlers/cc_phone_home.py delete mode 100644 cloudinit/handlers/cc_puppet.py delete mode 100644 cloudinit/handlers/cc_resizefs.py delete mode 100644 cloudinit/handlers/cc_rightscale_userdata.py delete mode 100644 cloudinit/handlers/cc_rsyslog.py delete mode 100644 cloudinit/handlers/cc_runcmd.py delete mode 100644 cloudinit/handlers/cc_salt_minion.py delete mode 100644 cloudinit/handlers/cc_scripts_per_boot.py delete mode 100644 cloudinit/handlers/cc_scripts_per_instance.py delete mode 100644 cloudinit/handlers/cc_scripts_per_once.py delete mode 100644 cloudinit/handlers/cc_scripts_user.py delete mode 100644 cloudinit/handlers/cc_set_hostname.py delete mode 100644 cloudinit/handlers/cc_set_passwords.py delete mode 100644 cloudinit/handlers/cc_ssh.py delete mode 100644 cloudinit/handlers/cc_ssh_import_id.py delete mode 100644 cloudinit/handlers/cc_timezone.py delete mode 100644 cloudinit/handlers/cc_update_etc_hosts.py delete mode 100644 cloudinit/handlers/cc_update_hostname.py create mode 100644 cloudinit/transforms/__init__.py create mode 100644 cloudinit/transforms/cc_apt_pipelining.py create mode 100644 cloudinit/transforms/cc_apt_update_upgrade.py create mode 100644 cloudinit/transforms/cc_bootcmd.py create mode 100644 cloudinit/transforms/cc_byobu.py create mode 100644 cloudinit/transforms/cc_ca_certs.py create mode 100644 cloudinit/transforms/cc_chef.py create mode 100644 cloudinit/transforms/cc_disable_ec2_metadata.py create mode 100644 cloudinit/transforms/cc_final_message.py create mode 100644 cloudinit/transforms/cc_foo.py create mode 100644 cloudinit/transforms/cc_grub_dpkg.py create mode 100644 cloudinit/transforms/cc_keys_to_console.py create mode 100644 cloudinit/transforms/cc_landscape.py create mode 100644 cloudinit/transforms/cc_locale.py create mode 100644 cloudinit/transforms/cc_mcollective.py create mode 100644 cloudinit/transforms/cc_mounts.py create mode 100644 cloudinit/transforms/cc_phone_home.py create mode 100644 cloudinit/transforms/cc_puppet.py create mode 100644 cloudinit/transforms/cc_resizefs.py create mode 100644 cloudinit/transforms/cc_rightscale_userdata.py create mode 100644 cloudinit/transforms/cc_rsyslog.py create mode 100644 cloudinit/transforms/cc_runcmd.py create mode 100644 cloudinit/transforms/cc_salt_minion.py create mode 100644 cloudinit/transforms/cc_scripts_per_boot.py create mode 100644 cloudinit/transforms/cc_scripts_per_instance.py create mode 100644 cloudinit/transforms/cc_scripts_per_once.py create mode 100644 cloudinit/transforms/cc_scripts_user.py create mode 100644 cloudinit/transforms/cc_set_hostname.py create mode 100644 cloudinit/transforms/cc_set_passwords.py create mode 100644 cloudinit/transforms/cc_ssh.py create mode 100644 cloudinit/transforms/cc_ssh_import_id.py create mode 100644 cloudinit/transforms/cc_timezone.py create mode 100644 cloudinit/transforms/cc_update_etc_hosts.py create mode 100644 cloudinit/transforms/cc_update_hostname.py delete mode 100644 cloudinit/user_data/boot_hook.py (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py deleted file mode 100644 index 5d70ac43..00000000 --- a/cloudinit/handlers/__init__.py +++ /dev/null @@ -1,221 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2008-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Chuck Short -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -import os -import subprocess -import sys -import time -import traceback - -import yaml - -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - -DEF_HANDLER_VERSION = 1 -DEF_FREQ = PER_INSTANCE - - -# reads a cloudconfig module list, returns -# a 2 dimensional array suitable to pass to run_cc_modules -def read_cc_modules(cfg, name): - if name not in cfg: - return([]) - module_list = [] - # create 'module_list', an array of arrays - # where array[0] = config - # array[1] = freq - # array[2:] = arguemnts - for item in cfg[name]: - if isinstance(item, str): - module_list.append((item,)) - elif isinstance(item, list): - module_list.append(item) - else: - raise TypeError("failed to read '%s' item in config") - return(module_list) - - -def run_cc_modules(cc, module_list, log): - failures = [] - for cfg_mod in module_list: - name = cfg_mod[0] - freq = None - run_args = [] - if len(cfg_mod) > 1: - freq = cfg_mod[1] - if len(cfg_mod) > 2: - run_args = cfg_mod[2:] - - try: - log.debug("handling %s with freq=%s and args=%s" % - (name, freq, run_args)) - cc.handle(name, run_args, freq=freq) - except: - log.warn(traceback.format_exc()) - log.error("config handling of %s, %s, %s failed\n" % - (name, freq, run_args)) - failures.append(name) - - return(failures) - - -# always returns well formated values -# cfg is expected to have an entry 'output' in it, which is a dictionary -# that includes entries for 'init', 'config', 'final' or 'all' -# init: /var/log/cloud.out -# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] -# final: -# output: "| logger -p" -# error: "> /dev/null" -# this returns the specific 'mode' entry, cleanly formatted, with value -# None if if none is given -def get_output_cfg(cfg, mode="init"): - ret = [None, None] - if not 'output' in cfg: - return ret - - outcfg = cfg['output'] - if mode in outcfg: - modecfg = outcfg[mode] - else: - if 'all' not in outcfg: - return ret - # if there is a 'all' item in the output list - # then it applies to all users of this (init, config, final) - modecfg = outcfg['all'] - - # if value is a string, it specifies stdout and stderr - if isinstance(modecfg, str): - ret = [modecfg, modecfg] - - # if its a list, then we expect (stdout, stderr) - if isinstance(modecfg, list): - if len(modecfg) > 0: - ret[0] = modecfg[0] - if len(modecfg) > 1: - ret[1] = modecfg[1] - - # if it is a dictionary, expect 'out' and 'error' - # items, which indicate out and error - if isinstance(modecfg, dict): - if 'output' in modecfg: - ret[0] = modecfg['output'] - if 'error' in modecfg: - ret[1] = modecfg['error'] - - # if err's entry == "&1", then make it same as stdout - # as in shell syntax of "echo foo >/dev/null 2>&1" - if ret[1] == "&1": - ret[1] = ret[0] - - swlist = [">>", ">", "|"] - for i in range(len(ret)): - if not ret[i]: - continue - val = ret[i].lstrip() - found = False - for s in swlist: - if val.startswith(s): - val = "%s %s" % (s, val[len(s):].strip()) - found = True - break - if not found: - # default behavior is append - val = "%s %s" % (">>", val.strip()) - ret[i] = val - - return(ret) - - -# redirect_output(outfmt, errfmt, orig_out, orig_err) -# replace orig_out and orig_err with filehandles specified in outfmt or errfmt -# fmt can be: -# > FILEPATH -# >> FILEPATH -# | program [ arg1 [ arg2 [ ... ] ] ] -# -# with a '|', arguments are passed to shell, so one level of -# shell escape is required. -def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): - if outfmt: - (mode, arg) = outfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_out: - os.dup2(new_fp.fileno(), o_out.fileno()) - if errfmt == outfmt: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - if errfmt: - (mode, arg) = errfmt.split(" ", 1) - if mode == ">" or mode == ">>": - owith = "ab" - if mode == ">": - owith = "wb" - new_fp = open(arg, owith) - elif mode == "|": - proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) - new_fp = proc.stdin - else: - raise TypeError("invalid type for outfmt: %s" % outfmt) - - if o_err: - os.dup2(new_fp.fileno(), o_err.fileno()) - return - - -def form_module_name(name): - canon_name = name.replace("-", "_") - if canon_name.endswith(".py"): - canon_name = canon_name[0:(len(canon_name) - 3)] - canon_name = canon_name.strip() - if not canon_name: - return None - if not canon_name.startswith("cc_"): - canon_name = 'cc_%s' % (canon_name) - return canon_name - - -def fixup_module(mod): - freq = getattr(mod, "frequency", None) - if not freq: - setattr(mod, 'frequency', PER_INSTANCE) - handler = getattr(mod, "handle", None) - if not handler: - def empty_handle(_name, _cfg, _cloud, _log, _args): - pass - setattr(mod, 'handle', empty_handle) - return mod diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py new file mode 100644 index 00000000..c75aeb72 --- /dev/null +++ b/cloudinit/handlers/boot_hook.py @@ -0,0 +1,66 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class BootHookPartHandler(ud.PartHandler): + def __init__(self, boothook_dir, instance_id): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.boothook_dir = boothook_dir + self.instance_id = instance_id + + def list_types(self): + return [ + ud.type_from_starts_with("#cloud-boothook"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in ud.CONTENT_SIGNALS: + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + prefix = "#cloud-boothook" + start = 0 + if payload.startswith(prefix): + start = len(prefix) + 1 + + filepath = os.path.join(self.boothook_dir, filename) + util.write_file(filepath, payload[start:], 0700) + try: + env = os.environ.copy() + env['INSTANCE_ID'] = str(self.instance_id) + util.subp([filepath], env=env) + except util.ProcessExecutionError as e: + util.logexc(LOG, "Boothooks script %s execution error", filepath) + except Exception as e: + util.logexc(LOG, ("Boothooks unknown " + "error when running %s"), filepath) diff --git a/cloudinit/handlers/cc_apt_pipelining.py b/cloudinit/handlers/cc_apt_pipelining.py deleted file mode 100644 index 0286a9ae..00000000 --- a/cloudinit/handlers/cc_apt_pipelining.py +++ /dev/null @@ -1,53 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# -# Author: Ben Howard -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance -default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" - - -def handle(_name, cfg, _cloud, log, _args): - - apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) - apt_pipe_value = str(apt_pipe_value).lower() - - if apt_pipe_value == "false": - write_apt_snippet("0", log) - - elif apt_pipe_value in ("none", "unchanged", "os"): - return - - elif apt_pipe_value in str(range(0, 6)): - write_apt_snippet(apt_pipe_value, log) - - else: - log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) - - -def write_apt_snippet(setting, log, f_name=default_file): - """ Writes f_name with apt pipeline depth 'setting' """ - - acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' - file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" - + (acquire_pipeline_depth % setting)) - - util.write_file(f_name, file_contents) - - log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/handlers/cc_apt_update_upgrade.py b/cloudinit/handlers/cc_apt_update_upgrade.py deleted file mode 100644 index a7049bce..00000000 --- a/cloudinit/handlers/cc_apt_update_upgrade.py +++ /dev/null @@ -1,241 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback -import os -import glob -import cloudinit.CloudConfig as cc - - -def handle(_name, cfg, cloud, log, _args): - update = util.get_cfg_option_bool(cfg, 'apt_update', False) - upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) - - release = get_release() - - mirror = find_apt_mirror(cloud, cfg) - - log.debug("selected mirror at: %s" % mirror) - - if not util.get_cfg_option_bool(cfg, \ - 'apt_preserve_sources_list', False): - generate_sources_list(release, mirror) - old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ - "archive.ubuntu.com/ubuntu") - rename_apt_lists(old_mir, mirror) - - # set up proxy - proxy = cfg.get("apt_proxy", None) - proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" - if proxy: - try: - contents = "Acquire::HTTP::Proxy \"%s\";\n" - with open(proxy_filename, "w") as fp: - fp.write(contents % proxy) - except Exception as e: - log.warn("Failed to write proxy to %s" % proxy_filename) - elif os.path.isfile(proxy_filename): - os.unlink(proxy_filename) - - # process 'apt_sources' - if 'apt_sources' in cfg: - errors = add_sources(cfg['apt_sources'], - {'MIRROR': mirror, 'RELEASE': release}) - for e in errors: - log.warn("Source Error: %s\n" % ':'.join(e)) - - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections") - log.debug(traceback.format_exc()) - - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) - - errors = [] - if update or len(pkglist) or upgrade: - try: - cc.update_package_sources() - except subprocess.CalledProcessError as e: - log.warn("apt-get update failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if upgrade: - try: - cc.apt_get("upgrade") - except subprocess.CalledProcessError as e: - log.warn("apt upgrade failed") - log.debug(traceback.format_exc()) - errors.append(e) - - if len(pkglist): - try: - cc.install_packages(pkglist) - except subprocess.CalledProcessError as e: - log.warn("Failed to install packages: %s " % pkglist) - log.debug(traceback.format_exc()) - errors.append(e) - - if len(errors): - raise errors[0] - - return(True) - - -def mirror2lists_fileprefix(mirror): - string = mirror - # take of http:// or ftp:// - if string.endswith("/"): - string = string[0:-1] - pos = string.find("://") - if pos >= 0: - string = string[pos + 3:] - string = string.replace("/", "_") - return string - - -def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): - oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) - nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) - if(oprefix == nprefix): - return - olen = len(oprefix) - for filename in glob.glob("%s_*" % oprefix): - os.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], - stdout=subprocess.PIPE).communicate() - return(str(stdout).strip()) - - -def generate_sources_list(codename, mirror): - util.render_to_file('sources.list', '/etc/apt/sources.list', \ - {'mirror': mirror, 'codename': codename}) - - -def add_sources(srclist, searchList=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList - """ - if searchList is None: - searchList = {} - elst = [] - - for ent in srclist: - if 'source' not in ent: - elst.append(["", "missing source"]) - continue - - source = ent['source'] - if source.startswith("ppa:"): - try: - util.subp(["add-apt-repository", source]) - except: - elst.append([source, "add-apt-repository failed"]) - continue - - source = util.render_string(source, searchList) - - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = "%s/%s" % \ - ("/etc/apt/sources.list.d/", ent['filename']) - - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = util.getkeybyid(ent['keyid'], ks) - except: - elst.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - elst.append([source, "failed add key"]) - - try: - util.write_file(ent['filename'], source + "\n", omode="ab") - except: - elst.append([source, "failed write to file %s" % ent['filename']]) - - return(elst) - - -def find_apt_mirror(cloud, cfg): - """ find an apt_mirror given the cloud and cfg provided """ - - # TODO: distro and defaults should be configurable - distro = "ubuntu" - defaults = { - 'ubuntu': "http://archive.ubuntu.com/ubuntu", - 'debian': "http://archive.debian.org/debian", - } - mirror = None - - cfg_mirror = cfg.get("apt_mirror", None) - if cfg_mirror: - mirror = cfg["apt_mirror"] - elif "apt_mirror_search" in cfg: - mirror = util.search_for_mirror(cfg['apt_mirror_search']) - else: - if cloud: - mirror = cloud.get_mirror() - - mydom = "" - - doms = [] - - if not mirror and cloud: - # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - mydom = ".".join(fqdn.split(".")[1:]) - if mydom: - doms.append(".%s" % mydom) - - if not mirror: - doms.extend((".localdomain", "",)) - - mirror_list = [] - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) - for post in doms: - mirror_list.append(mirrorfmt % post) - - mirror = util.search_for_mirror(mirror_list) - - if not mirror: - mirror = defaults[distro] - - return mirror diff --git a/cloudinit/handlers/cc_bootcmd.py b/cloudinit/handlers/cc_bootcmd.py deleted file mode 100644 index f584da02..00000000 --- a/cloudinit/handlers/cc_bootcmd.py +++ /dev/null @@ -1,48 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -import tempfile -import os -from cloudinit.CloudConfig import per_always -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if "bootcmd" not in cfg: - return - - try: - content = util.shellify(cfg["bootcmd"]) - tmpf = tempfile.TemporaryFile() - tmpf.write(content) - tmpf.seek(0) - except: - log.warn("failed to shellify bootcmd") - raise - - try: - env = os.environ.copy() - env['INSTANCE_ID'] = cloud.get_instance_id() - subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) - tmpf.close() - except: - log.warn("failed to run commands from bootcmd") - raise diff --git a/cloudinit/handlers/cc_byobu.py b/cloudinit/handlers/cc_byobu.py deleted file mode 100644 index e821b261..00000000 --- a/cloudinit/handlers/cc_byobu.py +++ /dev/null @@ -1,77 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - value = args[0] - else: - value = util.get_cfg_option_str(cfg, "byobu_by_default", "") - - if not value: - return - - if value == "user" or value == "system": - value = "enable-%s" % value - - valid = ("enable-user", "enable-system", "enable", - "disable-user", "disable-system", "disable") - if not value in valid: - log.warn("Unknown value %s for byobu_by_default" % value) - - mod_user = value.endswith("-user") - mod_sys = value.endswith("-system") - if value.startswith("enable"): - bl_inst = "install" - dc_val = "byobu byobu/launch-by-default boolean true" - mod_sys = True - else: - if value == "disable": - mod_user = True - mod_sys = True - bl_inst = "uninstall" - dc_val = "byobu byobu/launch-by-default boolean false" - - shcmd = "" - if mod_user: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) - shcmd += " || X=$(($X+1)); " - if mod_sys: - shcmd += "echo \"%s\" | debconf-set-selections" % dc_val - shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" - shcmd += " || X=$(($X+1)); " - - cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - - log.debug("setting byobu to %s" % value) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_ca_certs.py b/cloudinit/handlers/cc_ca_certs.py deleted file mode 100644 index 3af6238a..00000000 --- a/cloudinit/handlers/cc_ca_certs.py +++ /dev/null @@ -1,90 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Mike Milner -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import os -from subprocess import check_call -from cloudinit.util import (write_file, get_cfg_option_list_or_str, - delete_dir_contents, subp) - -CA_CERT_PATH = "/usr/share/ca-certificates/" -CA_CERT_FILENAME = "cloud-init-ca-certs.crt" -CA_CERT_CONFIG = "/etc/ca-certificates.conf" -CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" - - -def update_ca_certs(): - """ - Updates the CA certificate cache on the current machine. - """ - check_call(["update-ca-certificates"]) - - -def add_ca_certs(certs): - """ - Adds certificates to the system. To actually apply the new certificates - you must also call L{update_ca_certs}. - - @param certs: A list of certificate strings. - """ - if certs: - cert_file_contents = "\n".join(certs) - cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) - write_file(cert_file_fullpath, cert_file_contents, mode=0644) - # Append cert filename to CA_CERT_CONFIG file. - write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") - - -def remove_default_ca_certs(): - """ - Removes all default trusted CA certificates from the system. To actually - apply the change you must also call L{update_ca_certs}. - """ - delete_dir_contents(CA_CERT_PATH) - delete_dir_contents(CA_CERT_SYSTEM_PATH) - write_file(CA_CERT_CONFIG, "", mode=0644) - debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" - subp(('debconf-set-selections', '-'), debconf_sel) - - -def handle(_name, cfg, _cloud, log, _args): - """ - Call to handle ca-cert sections in cloud-config file. - - @param name: The module name "ca-cert" from cloud.cfg - @param cfg: A nested dict containing the entire cloud config contents. - @param cloud: The L{CloudInit} object in use. - @param log: Pre-initialized Python logger object to use for logging. - @param args: Any module arguments from cloud.cfg - """ - # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: - return - ca_cert_cfg = cfg['ca-certs'] - - # If there is a remove-defaults option set to true, remove the system - # default trusted CA certs first. - if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") - remove_default_ca_certs() - - # If we are given any new trusted CA certs to add, add them. - if "trusted" in ca_cert_cfg: - trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") - if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) - add_ca_certs(trusted_certs) - - # Update the system with the new cert configuration. - update_ca_certs() diff --git a/cloudinit/handlers/cc_chef.py b/cloudinit/handlers/cc_chef.py deleted file mode 100644 index 941e04fe..00000000 --- a/cloudinit/handlers/cc_chef.py +++ /dev/null @@ -1,119 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Avishai Ish-Shalom -# Author: Mike Moulton -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import json -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -ruby_version_default = "1.8" - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a chef key in the configuration don't do anything - if 'chef' not in cfg: - return - chef_cfg = cfg['chef'] - - # ensure the chef directories we use exist - mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', - '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) - - # set the validation key based on the presence of either 'validation_key' - # or 'validation_cert'. In the case where both exist, 'validation_key' - # takes precedence - for key in ('validation_key', 'validation_cert'): - if key in chef_cfg and chef_cfg[key]: - with open('/etc/chef/validation.pem', 'w') as validation_key_fh: - validation_key_fh.write(chef_cfg[key]) - break - - # create the chef config from template - util.render_to_file('chef_client.rb', '/etc/chef/client.rb', - {'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', - cloud.datasource.get_instance_id()), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name']}) - - # set the firstboot json - with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in initial_attributes.keys(): - initial_json[k] = initial_attributes[k] - firstboot_json_fh.write(json.dumps(initial_json)) - - # If chef is not installed, we install chef based on 'install_type' - if not os.path.isfile('/usr/bin/chef-client'): - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - ruby_version_default) - install_chef_from_gems(ruby_version, chef_version) - # and finally, run chef-client - log.debug('running chef-client') - subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', - '-s', '20']) - else: - # this will install and run the chef-client from packages - cc.install_packages(('chef',)) - - -def get_ruby_packages(version): - # return a list of packages needed to install ruby at version - pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] - if version == "1.8": - pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) - return(pkgs) - - -def install_chef_from_gems(ruby_version, chef_version=None): - cc.install_packages(get_ruby_packages(ruby_version)) - if not os.path.exists('/usr/bin/gem'): - os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') - if not os.path.exists('/usr/bin/ruby'): - os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') - if chef_version: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q']) - else: - subprocess.check_call(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q']) - - -def ensure_dir(d): - if not os.path.exists(d): - os.makedirs(d) - - -def mkdirs(dirs): - for d in dirs: - ensure_dir(d) diff --git a/cloudinit/handlers/cc_disable_ec2_metadata.py b/cloudinit/handlers/cc_disable_ec2_metadata.py deleted file mode 100644 index 6b31ea8e..00000000 --- a/cloudinit/handlers/cc_disable_ec2_metadata.py +++ /dev/null @@ -1,30 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import cloudinit.util as util -import subprocess -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): - fwall = "route add -host 169.254.169.254 reject" - subprocess.call(fwall.split(' ')) diff --git a/cloudinit/handlers/cc_final_message.py b/cloudinit/handlers/cc_final_message.py deleted file mode 100644 index abb4ca32..00000000 --- a/cloudinit/handlers/cc_final_message.py +++ /dev/null @@ -1,58 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_always -import sys -from cloudinit import util, boot_finished -import time - -frequency = per_always - -final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - msg_in = args[0] - else: - msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) - - try: - uptimef = open("/proc/uptime") - uptime = uptimef.read().split(" ")[0] - uptimef.close() - except IOError as e: - log.warn("unable to open /proc/uptime\n") - uptime = "na" - - try: - ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: - ts = "na" - - try: - subs = {'UPTIME': uptime, 'TIMESTAMP': ts} - sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) - except Exception as e: - log.warn("failed to render string to stdout: %s" % e) - - fp = open(boot_finished, "wb") - fp.write(uptime + "\n") - fp.close() diff --git a/cloudinit/handlers/cc_foo.py b/cloudinit/handlers/cc_foo.py deleted file mode 100644 index 35ec3fa7..00000000 --- a/cloudinit/handlers/cc_foo.py +++ /dev/null @@ -1,29 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -#import cloudinit -#import cloudinit.util as util -from cloudinit.CloudConfig import per_instance - -frequency = per_instance - - -def handle(_name, _cfg, _cloud, _log, _args): - print "hi" diff --git a/cloudinit/handlers/cc_grub_dpkg.py b/cloudinit/handlers/cc_grub_dpkg.py deleted file mode 100644 index 9f3a7eaf..00000000 --- a/cloudinit/handlers/cc_grub_dpkg.py +++ /dev/null @@ -1,64 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import traceback -import os - - -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None - - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) - - if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): - if idevs == None: - idevs = "" - if idevs_empty == None: - idevs_empty = "true" - else: - if idevs_empty == None: - idevs_empty = "false" - if idevs == None: - idevs = "/dev/sda" - for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): - if os.path.exists(dev): - idevs = dev - break - - # now idevs and idevs_empty are set to determined values - # or, those set by user - - dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ - "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty - log.debug("setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) - - try: - util.subp(('debconf-set-selections'), dconf_sel) - except: - log.error("Failed to run debconf-set-selections for grub-dpkg") - log.debug(traceback.format_exc()) diff --git a/cloudinit/handlers/cc_keys_to_console.py b/cloudinit/handlers/cc_keys_to_console.py deleted file mode 100644 index 73a477c0..00000000 --- a/cloudinit/handlers/cc_keys_to_console.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -import subprocess - -frequency = per_instance - - -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, - "ssh_key_console_blacklist", ["ssh-dss"]) - try: - confp = open('/dev/console', "wb") - cmd.append(','.join(fp_blacklist)) - cmd.append(','.join(key_blacklist)) - subprocess.call(cmd, stdout=confp) - confp.close() - except: - log.warn("writing keys to console value") - raise diff --git a/cloudinit/handlers/cc_landscape.py b/cloudinit/handlers/cc_landscape.py deleted file mode 100644 index a4113cbe..00000000 --- a/cloudinit/handlers/cc_landscape.py +++ /dev/null @@ -1,75 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -from cloudinit.CloudConfig import per_instance -from configobj import ConfigObj - -frequency = per_instance - -lsc_client_cfg_file = "/etc/landscape/client.conf" - -# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 -lsc_builtincfg = { - 'client': { - 'log_level': "info", - 'url': "https://landscape.canonical.com/message-system", - 'ping_url': "http://landscape.canonical.com/ping", - 'data_path': "/var/lib/landscape/client", - } -} - - -def handle(_name, cfg, _cloud, log, _args): - """ - Basically turn a top level 'landscape' entry with a 'client' dict - and render it to ConfigObj format under '[client]' section in - /etc/landscape/client.conf - """ - - ls_cloudcfg = cfg.get("landscape", {}) - - if not isinstance(ls_cloudcfg, dict): - raise(Exception("'landscape' existed in config, but not a dict")) - - merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) - - if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): - os.makedirs(os.path.dirname(lsc_client_cfg_file)) - - with open(lsc_client_cfg_file, "w") as fp: - merged.write(fp) - - log.debug("updated %s" % lsc_client_cfg_file) - - -def mergeTogether(objs): - """ - merge together ConfigObj objects or things that ConfigObj() will take in - later entries override earlier - """ - cfg = ConfigObj({}) - for obj in objs: - if isinstance(obj, ConfigObj): - cfg.merge(obj) - else: - cfg.merge(ConfigObj(obj)) - return cfg diff --git a/cloudinit/handlers/cc_locale.py b/cloudinit/handlers/cc_locale.py deleted file mode 100644 index 2bb22fdb..00000000 --- a/cloudinit/handlers/cc_locale.py +++ /dev/null @@ -1,54 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os.path -import subprocess -import traceback - - -def apply_locale(locale, cfgfile): - if os.path.exists('/usr/sbin/locale-gen'): - subprocess.Popen(['locale-gen', locale]).communicate() - if os.path.exists('/usr/sbin/update-locale'): - subprocess.Popen(['update-locale', locale]).communicate() - - util.render_to_file('default-locale', cfgfile, {'locale': locale}) - - -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - locale = args[0] - else: - locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - - locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", - "/etc/default/locale") - - if not locale: - return - - log.debug("setting locale to %s" % locale) - - try: - apply_locale(locale, locale_cfgfile) - except Exception as e: - log.debug(traceback.format_exc(e)) - raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/handlers/cc_mcollective.py b/cloudinit/handlers/cc_mcollective.py deleted file mode 100644 index a2a6230c..00000000 --- a/cloudinit/handlers/cc_mcollective.py +++ /dev/null @@ -1,99 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Marc Cluet -# Based on code by Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - -pubcert_file = "/etc/mcollective/ssl/server-public.pem" -pricert_file = "/etc/mcollective/ssl/server-private.pem" - - -# Our fake header section -class FakeSecHead(object): - def __init__(self, fp): - self.fp = fp - self.sechead = '[nullsection]\n' - - def readline(self): - if self.sechead: - try: - return self.sechead - finally: - self.sechead = None - else: - return self.fp.readline() - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a mcollective key in the configuration don't do anything - if 'mcollective' not in cfg: - return - mcollective_cfg = cfg['mcollective'] - # Start by installing the mcollective package ... - cc.install_packages(("mcollective",)) - - # ... and then update the mcollective configuration - if 'conf' in mcollective_cfg: - # Create object for reading server.cfg values - mcollective_config = ConfigParser.ConfigParser() - # Read server.cfg values from original file in order to be able to mix - # the rest up - mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' - 'server.cfg'))) - for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): - if cfg_name == 'public-cert': - util.write_file(pubcert_file, cfg, mode=0644) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_public', pubcert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - elif cfg_name == 'private-cert': - util.write_file(pricert_file, cfg, mode=0600) - mcollective_config.set(cfg_name, - 'plugin.ssl_server_private', pricert_file) - mcollective_config.set(cfg_name, 'securityprovider', 'ssl') - else: - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - mcollective_config.set(cfg_name, o, v) - # We got all our config as wanted we'll rename - # the previous server.cfg and create our new one - os.rename('/etc/mcollective/server.cfg', - '/etc/mcollective/server.cfg.old') - outputfile = StringIO.StringIO() - mcollective_config.write(outputfile) - # Now we got the whole file, write to disk except first line - # Note below, that we've just used ConfigParser because it generally - # works. Below, we remove the initial 'nullsection' header - # and then change 'key = value' to 'key: value'. The global - # search and replace of '=' with ':' could be problematic though. - # this most likely needs fixing. - util.write_file('/etc/mcollective/server.cfg', - outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', - ':'), - mode=0644) - - # Start mcollective - subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/handlers/cc_mounts.py b/cloudinit/handlers/cc_mounts.py deleted file mode 100644 index 6cdd74e8..00000000 --- a/cloudinit/handlers/cc_mounts.py +++ /dev/null @@ -1,179 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import os -import re -from string import whitespace # pylint: disable=W0402 - - -def is_mdname(name): - # return true if this is a metadata service name - if name in ["ami", "root", "swap"]: - return True - # names 'ephemeral0' or 'ephemeral1' - # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' - for enumname in ("ephemeral", "ebs"): - if name.startswith(enumname) and name.find(":") == -1: - return True - return False - - -def handle(_name, cfg, cloud, log, _args): - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] - defvals = cfg.get("mount_default_fields", defvals) - - # these are our default set of mounts - defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], - ["swap", "none", "swap", "sw", "0", "0"]] - - cfgmnt = [] - if "mounts" in cfg: - cfgmnt = cfg["mounts"] - - # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 - shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" - shortname = re.compile(shortname_filter) - - for i in range(len(cfgmnt)): - # skip something that wasn't a list - if not isinstance(cfgmnt[i], list): - continue - - # workaround, allow user to specify 'ephemeral' - # rather than more ec2 correct 'ephemeral0' - if cfgmnt[i][0] == "ephemeral": - cfgmnt[i][0] = "ephemeral0" - - if is_mdname(cfgmnt[i][0]): - newname = cloud.device_name_to_device(cfgmnt[i][0]) - if not newname: - log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) - cfgmnt[i][1] = None - else: - if newname.startswith("/"): - cfgmnt[i][0] = newname - else: - cfgmnt[i][0] = "/dev/%s" % newname - else: - if shortname.match(cfgmnt[i][0]): - cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] - - # in case the user did not quote a field (likely fs-freq, fs_passno) - # but do not convert None to 'None' (LP: #898365) - for j in range(len(cfgmnt[i])): - if isinstance(cfgmnt[i][j], int): - cfgmnt[i][j] = str(cfgmnt[i][j]) - - for i in range(len(cfgmnt)): - # fill in values with defaults from defvals above - for j in range(len(defvals)): - if len(cfgmnt[i]) <= j: - cfgmnt[i].append(defvals[j]) - elif cfgmnt[i][j] is None: - cfgmnt[i][j] = defvals[j] - - # if the second entry in the list is 'None' this - # clears all previous entries of that same 'fs_spec' - # (fs_spec is the first field in /etc/fstab, ie, that device) - if cfgmnt[i][1] is None: - for j in range(i): - if cfgmnt[j][0] == cfgmnt[i][0]: - cfgmnt[j][1] = None - - # for each of the "default" mounts, add them only if no other - # entry has the same device name - for defmnt in defmnts: - devname = cloud.device_name_to_device(defmnt[0]) - if devname is None: - continue - if devname.startswith("/"): - defmnt[0] = devname - else: - defmnt[0] = "/dev/%s" % devname - - cfgmnt_has = False - for cfgm in cfgmnt: - if cfgm[0] == defmnt[0]: - cfgmnt_has = True - break - - if cfgmnt_has: - continue - cfgmnt.append(defmnt) - - # now, each entry in the cfgmnt list has all fstab values - # if the second field is None (not the string, the value) we skip it - actlist = [x for x in cfgmnt if x[1] is not None] - - if len(actlist) == 0: - return - - comment = "comment=cloudconfig" - cc_lines = [] - needswap = False - dirs = [] - for line in actlist: - # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] - if line[2] == "swap": - needswap = True - if line[1].startswith("/"): - dirs.append(line[1]) - cc_lines.append('\t'.join(line)) - - fstab_lines = [] - fstab = open("/etc/fstab", "r+") - ws = re.compile("[%s]+" % whitespace) - for line in fstab.read().splitlines(): - try: - toks = ws.split(line) - if toks[3].find(comment) != -1: - continue - except: - pass - fstab_lines.append(line) - - fstab_lines.extend(cc_lines) - - fstab.seek(0) - fstab.write("%s\n" % '\n'.join(fstab_lines)) - fstab.truncate() - fstab.close() - - if needswap: - try: - util.subp(("swapon", "-a")) - except: - log.warn("Failed to enable swap") - - for d in dirs: - if os.path.exists(d): - continue - try: - os.makedirs(d) - except: - log.warn("Failed to make '%s' config-mount\n", d) - - try: - util.subp(("mount", "-a")) - except: - log.warn("'mount -a' failed") diff --git a/cloudinit/handlers/cc_phone_home.py b/cloudinit/handlers/cc_phone_home.py deleted file mode 100644 index a7ff74e1..00000000 --- a/cloudinit/handlers/cc_phone_home.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -from cloudinit.CloudConfig import per_instance -import cloudinit.util as util -from time import sleep - -frequency = per_instance -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', - 'hostname'] - - -# phone_home: -# url: http://my.foo.bar/$INSTANCE/ -# post: all -# tries: 10 -# -# phone_home: -# url: http://my.foo.bar/$INSTANCE_ID/ -# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id -# -def handle(_name, cfg, cloud, log, args): - if len(args) != 0: - ph_cfg = util.read_conf(args[0]) - else: - if not 'phone_home' in cfg: - return - ph_cfg = cfg['phone_home'] - - if 'url' not in ph_cfg: - log.warn("no 'url' token in phone_home") - return - - url = ph_cfg['url'] - post_list = ph_cfg.get('post', 'all') - tries = ph_cfg.get('tries', 10) - try: - tries = int(tries) - except: - log.warn("tries is not an integer. using 10") - tries = 10 - - if post_list == "all": - post_list = post_list_all - - all_keys = {} - all_keys['instance_id'] = cloud.get_instance_id() - all_keys['hostname'] = cloud.get_hostname() - - pubkeys = { - 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', - 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', - 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', - } - - for n, path in pubkeys.iteritems(): - try: - fp = open(path, "rb") - all_keys[n] = fp.read() - fp.close() - except: - log.warn("%s: failed to open in phone_home" % path) - - submit_keys = {} - for k in post_list: - if k in all_keys: - submit_keys[k] = all_keys[k] - else: - submit_keys[k] = "N/A" - log.warn("requested key %s from 'post' list not available") - - url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) - - null_exc = object() - last_e = null_exc - for i in range(0, tries): - try: - util.readurl(url, submit_keys) - log.debug("succeeded submit to %s on try %i" % (url, i + 1)) - return - except Exception as e: - log.debug("failed to post to %s on try %i" % (url, i + 1)) - last_e = e - sleep(3) - - log.warn("failed to post to %s in %i tries" % (url, tries)) - if last_e is not null_exc: - raise(last_e) - - return diff --git a/cloudinit/handlers/cc_puppet.py b/cloudinit/handlers/cc_puppet.py deleted file mode 100644 index 6fc475f6..00000000 --- a/cloudinit/handlers/cc_puppet.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import pwd -import socket -import subprocess -import StringIO -import ConfigParser -import cloudinit.CloudConfig as cc -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - # If there isn't a puppet key in the configuration don't do anything - if 'puppet' not in cfg: - return - puppet_cfg = cfg['puppet'] - # Start by installing the puppet package ... - cc.install_packages(("puppet",)) - - # ... and then update the puppet configuration - if 'conf' in puppet_cfg: - # Add all sections from the conf object to puppet.conf - puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') - # Create object for reading puppet.conf values - puppet_config = ConfigParser.ConfigParser() - # Read puppet.conf values from original file in order to be able to - # mix the rest up - puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in - puppet_conf_fh.readlines()))) - # Close original file, no longer needed - puppet_conf_fh.close() - for cfg_name, cfg in puppet_cfg['conf'].iteritems(): - # ca_cert configuration is a special case - # Dump the puppetmaster ca certificate in the correct place - if cfg_name == 'ca_cert': - # Puppet ssl sub-directory isn't created yet - # Create it with the proper permissions and ownership - os.makedirs('/var/lib/puppet/ssl') - os.chmod('/var/lib/puppet/ssl', 0771) - os.chown('/var/lib/puppet/ssl', - pwd.getpwnam('puppet').pw_uid, 0) - os.makedirs('/var/lib/puppet/ssl/certs/') - os.chown('/var/lib/puppet/ssl/certs/', - pwd.getpwnam('puppet').pw_uid, 0) - ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') - ca_fh.write(cfg) - ca_fh.close() - os.chown('/var/lib/puppet/ssl/certs/ca.pem', - pwd.getpwnam('puppet').pw_uid, 0) - util.restorecon_if_possible('/var/lib/puppet', recursive=True) - else: - #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) - # If puppet.conf already has this section we don't want to - # write it again - if puppet_config.has_section(cfg_name) == False: - puppet_config.add_section(cfg_name) - # Iterate throug the config items, we'll use ConfigParser.set - # to overwrite or create new items as needed - for o, v in cfg.iteritems(): - if o == 'certname': - # Expand %f as the fqdn - v = v.replace("%f", socket.getfqdn()) - # Expand %i as the instance id - v = v.replace("%i", - cloud.datasource.get_instance_id()) - # certname needs to be downcase - v = v.lower() - puppet_config.set(cfg_name, o, v) - #puppet_conf_fh.write("%s=%s\n" % (o, v)) - # We got all our config as wanted we'll rename - # the previous puppet.conf and create our new one - os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') - with open('/etc/puppet/puppet.conf', 'wb') as configfile: - puppet_config.write(configfile) - util.restorecon_if_possible('/etc/puppet/puppet.conf') - # Set puppet to automatically start - if os.path.exists('/etc/default/puppet'): - subprocess.check_call(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet']) - elif os.path.exists('/bin/systemctl'): - subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) - elif os.path.exists('/sbin/chkconfig'): - subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) - else: - log.warn("Do not know how to enable puppet service on this system") - # Start puppetd - subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/handlers/cc_resizefs.py b/cloudinit/handlers/cc_resizefs.py deleted file mode 100644 index 2dc66def..00000000 --- a/cloudinit/handlers/cc_resizefs.py +++ /dev/null @@ -1,108 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import os -import stat -import sys -import time -import tempfile -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - resize_root = False - if str(args[0]).lower() in ['true', '1', 'on', 'yes']: - resize_root = True - else: - resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) - - if str(resize_root).lower() in ['false', '0']: - return - - # we use mktemp rather than mkstemp because early in boot nothing - # else should be able to race us for this, and we need to mknod. - devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") - - try: - st_dev = os.stat("/").st_dev - dev = os.makedev(os.major(st_dev), os.minor(st_dev)) - os.mknod(devpth, 0400 | stat.S_IFBLK, dev) - except: - if util.is_container(): - log.debug("inside container, ignoring mknod failure in resizefs") - return - log.warn("Failed to make device node to resize /") - raise - - cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] - try: - (fstype, _err) = util.subp(cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % - (os.major(st_dev), os.minor(st_dev), cmd)) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - if str(fstype).startswith("ext"): - resize_cmd = ['resize2fs', devpth] - elif fstype == "xfs": - resize_cmd = ['xfs_growfs', devpth] - else: - os.unlink(devpth) - log.debug("not resizing unknown filesystem %s" % fstype) - return - - if resize_root == "noblock": - fid = os.fork() - if fid == 0: - try: - do_resize(resize_cmd, devpth, log) - os._exit(0) # pylint: disable=W0212 - except Exception as exc: - sys.stderr.write("Failed: %s" % exc) - os._exit(1) # pylint: disable=W0212 - else: - do_resize(resize_cmd, devpth, log) - - log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % - (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), - resize_root)) - - return - - -def do_resize(resize_cmd, devpth, log): - try: - start = time.time() - util.subp(resize_cmd) - except subprocess.CalledProcessError as e: - log.warn("Failed to resize filesystem (%s)" % resize_cmd) - log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) - os.unlink(devpth) - raise - - os.unlink(devpth) - log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/handlers/cc_rightscale_userdata.py b/cloudinit/handlers/cc_rightscale_userdata.py deleted file mode 100644 index 5ed0848f..00000000 --- a/cloudinit/handlers/cc_rightscale_userdata.py +++ /dev/null @@ -1,78 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -## -## The purpose of this script is to allow cloud-init to consume -## rightscale style userdata. rightscale user data is key-value pairs -## in a url-query-string like format. -## -## for cloud-init support, there will be a key named -## 'CLOUD_INIT_REMOTE_HOOK'. -## -## This cloud-config module will -## - read the blob of data from raw user data, and parse it as key/value -## - for each key that is found, download the content to -## the local instance/scripts directory and set them executable. -## - the files in that directory will be run by the user-scripts module -## Therefore, this must run before that. -## -## - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur -from urlparse import parse_qs - -frequency = per_instance -my_name = "cc_rightscale_userdata" -my_hookname = 'CLOUD_INIT_REMOTE_HOOK' - - -def handle(_name, _cfg, cloud, log, _args): - try: - ud = cloud.get_userdata_raw() - except: - log.warn("failed to get raw userdata in %s" % my_name) - return - - try: - mdict = parse_qs(ud) - if not my_hookname in mdict: - return - except: - log.warn("failed to urlparse.parse_qa(userdata_raw())") - raise - - scripts_d = get_ipath_cur('scripts') - i = 0 - first_e = None - for url in mdict[my_hookname]: - fname = "%s/rightscale-%02i" % (scripts_d, i) - i = i + 1 - try: - content = util.readurl(url) - util.write_file(fname, content, mode=0700) - except Exception as e: - if not first_e: - first_e = None - log.warn("%s failed to read %s: %s" % (my_name, url, e)) - - if first_e: - raise(e) diff --git a/cloudinit/handlers/cc_rsyslog.py b/cloudinit/handlers/cc_rsyslog.py deleted file mode 100644 index ac7f2c74..00000000 --- a/cloudinit/handlers/cc_rsyslog.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab syntax=python -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit -import logging -import cloudinit.util as util -import traceback - -DEF_FILENAME = "20-cloud-config.conf" -DEF_DIR = "/etc/rsyslog.d" - - -def handle(_name, cfg, _cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - - # process 'rsyslog' - if not 'rsyslog' in cfg: - return - - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) - - files = [] - elst = [] - for ent in cfg['rsyslog']: - if isinstance(ent, dict): - if not "content" in ent: - elst.append((ent, "no 'content' entry")) - continue - content = ent['content'] - filename = ent.get("filename", def_fname) - else: - content = ent - filename = def_fname - - if not filename.startswith("/"): - filename = "%s/%s" % (def_dir, filename) - - omode = "ab" - # truncate filename first time you see it - if filename not in files: - omode = "wb" - files.append(filename) - - try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: - log.debug(traceback.format_exc(e)) - elst.append((content, "failed to write to %s" % filename)) - - # need to restart syslogd - restarted = False - try: - # if this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set - log.debug("restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - - except Exception as e: - elst.append(("restart", str(e))) - - if restarted: - # this only needs to run if we *actually* restarted - # syslog above. - cloudinit.logging_set_from_cfg_file() - log = logging.getLogger() - log.debug("rsyslog configured %s" % files) - - for e in elst: - log.warn("rsyslog error: %s\n" % ':'.join(e)) - - return diff --git a/cloudinit/handlers/cc_runcmd.py b/cloudinit/handlers/cc_runcmd.py deleted file mode 100644 index f7e8c671..00000000 --- a/cloudinit/handlers/cc_runcmd.py +++ /dev/null @@ -1,32 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if "runcmd" not in cfg: - return - outfile = "%s/runcmd" % cloud.get_ipath('scripts') - try: - content = util.shellify(cfg["runcmd"]) - util.write_file(outfile, content, 0700) - except: - log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/handlers/cc_salt_minion.py b/cloudinit/handlers/cc_salt_minion.py deleted file mode 100644 index 1a3b5039..00000000 --- a/cloudinit/handlers/cc_salt_minion.py +++ /dev/null @@ -1,56 +0,0 @@ -# vi: ts=4 expandtab -# -# Author: Jeff Bauer -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import os.path -import subprocess -import cloudinit.CloudConfig as cc -import yaml - - -def handle(_name, cfg, _cloud, _log, _args): - # If there isn't a salt key in the configuration don't do anything - if 'salt_minion' not in cfg: - return - salt_cfg = cfg['salt_minion'] - # Start by installing the salt package ... - cc.install_packages(("salt",)) - config_dir = '/etc/salt' - if not os.path.isdir(config_dir): - os.makedirs(config_dir) - # ... and then update the salt configuration - if 'conf' in salt_cfg: - # Add all sections from the conf object to /etc/salt/minion - minion_config = os.path.join(config_dir, 'minion') - yaml.dump(salt_cfg['conf'], - file(minion_config, 'w'), - default_flow_style=False) - # ... copy the key pair if specified - if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = '/etc/salt/pki' - cumask = os.umask(077) - if not os.path.isdir(pki_dir): - os.makedirs(pki_dir) - pub_name = os.path.join(pki_dir, 'minion.pub') - pem_name = os.path.join(pki_dir, 'minion.pem') - with open(pub_name, 'w') as f: - f.write(salt_cfg['public_key']) - with open(pem_name, 'w') as f: - f.write(salt_cfg['private_key']) - os.umask(cumask) - - # Start salt-minion - subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/handlers/cc_scripts_per_boot.py b/cloudinit/handlers/cc_scripts_per_boot.py deleted file mode 100644 index 41a74754..00000000 --- a/cloudinit/handlers/cc_scripts_per_boot.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -from cloudinit import get_cpath - -frequency = per_always -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_scripts_per_instance.py b/cloudinit/handlers/cc_scripts_per_instance.py deleted file mode 100644 index a2981eab..00000000 --- a/cloudinit/handlers/cc_scripts_per_instance.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_cpath - -frequency = per_instance -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_scripts_per_once.py b/cloudinit/handlers/cc_scripts_per_once.py deleted file mode 100644 index a69151da..00000000 --- a/cloudinit/handlers/cc_scripts_per_once.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_once -from cloudinit import get_cpath - -frequency = per_once -runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_scripts_user.py b/cloudinit/handlers/cc_scripts_user.py deleted file mode 100644 index 933aa4e0..00000000 --- a/cloudinit/handlers/cc_scripts_user.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_instance -from cloudinit import get_ipath_cur - -frequency = per_instance -runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") - - -def handle(_name, _cfg, _cloud, log, _args): - try: - util.runparts(runparts_path) - except: - log.warn("failed to run-parts in %s" % runparts_path) - raise diff --git a/cloudinit/handlers/cc_set_hostname.py b/cloudinit/handlers/cc_set_hostname.py deleted file mode 100644 index acea74d9..00000000 --- a/cloudinit/handlers/cc_set_hostname.py +++ /dev/null @@ -1,42 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not setting hostname") - return(True) - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - set_hostname(hostname, log) - except Exception: - util.logexc(log) - log.warn("failed to set hostname to %s\n", hostname) - - return(True) - - -def set_hostname(hostname, log): - util.subp(['hostname', hostname]) - util.write_file("/etc/hostname", "%s\n" % hostname, 0644) - log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/handlers/cc_set_passwords.py b/cloudinit/handlers/cc_set_passwords.py deleted file mode 100644 index 9d0bbdb8..00000000 --- a/cloudinit/handlers/cc_set_passwords.py +++ /dev/null @@ -1,129 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import sys -import random -from string import letters, digits # pylint: disable=W0402 - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - # if run from command line, and give args, wipe the chpasswd['list'] - password = args[0] - if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: - del cfg['chpasswd']['list'] - else: - password = util.get_cfg_option_str(cfg, "password", None) - - expire = True - pw_auth = "no" - change_pwauth = False - plist = None - - if 'chpasswd' in cfg: - chfg = cfg['chpasswd'] - plist = util.get_cfg_option_str(chfg, 'list', plist) - expire = util.get_cfg_option_bool(chfg, 'expire', expire) - - if not plist and password: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - plist = "%s:%s" % (user, password) - - errors = [] - if plist: - plist_in = [] - randlist = [] - users = [] - for line in plist.splitlines(): - u, p = line.split(':', 1) - if p == "R" or p == "RANDOM": - p = rand_user_password() - randlist.append("%s:%s" % (u, p)) - plist_in.append("%s:%s" % (u, p)) - users.append(u) - - ch_in = '\n'.join(plist_in) - try: - util.subp(['chpasswd'], ch_in) - log.debug("changed password for %s:" % users) - except Exception as e: - errors.append(e) - log.warn("failed to set passwords with chpasswd: %s" % e) - - if len(randlist): - sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", - '\n'.join(randlist))) - - if expire: - enum = len(errors) - for u in users: - try: - util.subp(['passwd', '--expire', u]) - except Exception as e: - errors.append(e) - log.warn("failed to expire account for %s" % u) - if enum == len(errors): - log.debug("expired passwords for: %s" % u) - - if 'ssh_pwauth' in cfg: - val = str(cfg['ssh_pwauth']).lower() - if val in ("true", "1", "yes"): - pw_auth = "yes" - change_pwauth = True - elif val in ("false", "0", "no"): - pw_auth = "no" - change_pwauth = True - else: - change_pwauth = False - - if change_pwauth: - pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" - msg = "set PasswordAuthentication to '%s'" % pw_auth - try: - cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), - '/etc/ssh/sshd_config'] - util.subp(cmd) - log.debug(msg) - except Exception as e: - log.warn("failed %s" % msg) - errors.append(e) - - try: - p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), - 'restart']) - log.debug("restarted sshd") - except: - log.warn("restart of ssh failed") - - if len(errors): - raise(errors[0]) - - return - - -def rand_str(strlen=32, select_from=letters + digits): - return("".join([random.choice(select_from) for _x in range(0, strlen)])) - - -def rand_user_password(pwlen=9): - selfrom = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) - return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/handlers/cc_ssh.py b/cloudinit/handlers/cc_ssh.py deleted file mode 100644 index 48eb58bc..00000000 --- a/cloudinit/handlers/cc_ssh.py +++ /dev/null @@ -1,106 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import cloudinit.SshUtil as sshutil -import os -import glob -import subprocess - -DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ -"rather than the user \\\"root\\\".\';echo;sleep 10\"" - - -def handle(_name, cfg, cloud, log, _args): - - # remove the static keys from the pristine image - if cfg.get("ssh_deletekeys", True): - for f in glob.glob("/etc/ssh/ssh_host_*key*"): - try: - os.unlink(f) - except: - pass - - if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them - key2file = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), - } - - for key, val in cfg["ssh_keys"].items(): - if key in key2file: - util.write_file(key2file[key][0], val, key2file[key][1]) - - priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', } - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' - for priv, pub in priv2pub.iteritems(): - if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: - continue - pair = (key2file[priv][0], key2file[pub][0]) - subprocess.call(('sh', '-xc', cmd % pair)) - log.debug("generated %s from %s" % pair) - else: - # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', - ['rsa', 'dsa', 'ecdsa']): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype - if not os.path.exists(keyfile): - subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', - '-f', keyfile]) - - util.restorecon_if_possible('/etc/ssh', recursive=True) - - try: - user = util.get_cfg_option_str(cfg, 'user') - disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) - disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) - keys = cloud.get_public_ssh_keys() - - if "ssh_authorized_keys" in cfg: - cfgkeys = cfg["ssh_authorized_keys"] - keys.extend(cfgkeys) - - apply_credentials(keys, user, disable_root, disable_root_opts, log) - except: - util.logexc(log) - log.warn("applying credentials failed!\n") - - -def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): - keys = set(keys) - if user: - sshutil.setup_user_keys(keys, user, '', log) - - if disable_root: - key_prefix = disable_root_opts.replace('$USER', user) - else: - key_prefix = '' - - sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/handlers/cc_ssh_import_id.py b/cloudinit/handlers/cc_ssh_import_id.py deleted file mode 100644 index bbf5bd83..00000000 --- a/cloudinit/handlers/cc_ssh_import_id.py +++ /dev/null @@ -1,50 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import traceback - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - user = args[0] - ids = [] - if len(args) > 1: - ids = args[1:] - else: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) - - if len(ids) == 0: - return - - cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids - - log.debug("importing ssh ids. cmd = %s" % cmd) - - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) - except OSError as e: - log.debug(traceback.format_exc(e)) - raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/handlers/cc_timezone.py b/cloudinit/handlers/cc_timezone.py deleted file mode 100644 index e5c9901b..00000000 --- a/cloudinit/handlers/cc_timezone.py +++ /dev/null @@ -1,67 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.CloudConfig import per_instance -from cloudinit import util -import os.path -import shutil - -frequency = per_instance -tz_base = "/usr/share/zoneinfo" - - -def handle(_name, cfg, _cloud, log, args): - if len(args) != 0: - timezone = args[0] - else: - timezone = util.get_cfg_option_str(cfg, "timezone", False) - - if not timezone: - return - - tz_file = "%s/%s" % (tz_base, timezone) - - if not os.path.isfile(tz_file): - log.debug("Invalid timezone %s" % tz_file) - raise Exception("Invalid timezone %s" % tz_file) - - try: - fp = open("/etc/timezone", "wb") - fp.write("%s\n" % timezone) - fp.close() - except: - log.debug("failed to write to /etc/timezone") - raise - if os.path.exists("/etc/sysconfig/clock"): - try: - with open("/etc/sysconfig/clock", "w") as fp: - fp.write('ZONE="%s"\n' % timezone) - except: - log.debug("failed to write to /etc/sysconfig/clock") - raise - - try: - shutil.copy(tz_file, "/etc/localtime") - except: - log.debug("failed to copy %s to /etc/localtime" % tz_file) - raise - - log.debug("set timezone to %s" % timezone) - return diff --git a/cloudinit/handlers/cc_update_etc_hosts.py b/cloudinit/handlers/cc_update_etc_hosts.py deleted file mode 100644 index 6ad2fca8..00000000 --- a/cloudinit/handlers/cc_update_etc_hosts.py +++ /dev/null @@ -1,87 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -from cloudinit.CloudConfig import per_always -import StringIO - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - - manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) - if manage_hosts in ("True", "true", True, "template"): - # render from template file - try: - if not hostname: - log.info("manage_etc_hosts was set, but no hostname found") - return - - util.render_to_file('hosts', '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) - except Exception: - log.warn("failed to update /etc/hosts") - raise - elif manage_hosts == "localhost": - log.debug("managing 127.0.1.1 in /etc/hosts") - update_etc_hosts(hostname, fqdn, log) - return - else: - if manage_hosts not in ("False", False): - log.warn("Unknown value for manage_etc_hosts. Assuming False") - else: - log.debug("not managing /etc/hosts") - - -def update_etc_hosts(hostname, fqdn, _log): - with open('/etc/hosts', 'r') as etchosts: - header = "# Added by cloud-init\n" - hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) - need_write = False - need_change = True - new_etchosts = StringIO.StringIO() - for line in etchosts: - split_line = [s.strip() for s in line.split()] - if len(split_line) < 2: - new_etchosts.write(line) - continue - if line == header: - continue - ip, hosts = split_line[0], split_line[1:] - if ip == "127.0.1.1": - if sorted([hostname, fqdn]) == sorted(hosts): - need_change = False - if need_change == True: - line = "%s%s" % (header, hosts_line) - need_change = False - need_write = True - new_etchosts.write(line) - etchosts.close() - if need_change == True: - new_etchosts.write("%s%s" % (header, hosts_line)) - need_write = True - if need_write == True: - new_etcfile = open('/etc/hosts', 'wb') - new_etcfile.write(new_etchosts.getvalue()) - new_etcfile.close() - new_etchosts.close() - return diff --git a/cloudinit/handlers/cc_update_hostname.py b/cloudinit/handlers/cc_update_hostname.py deleted file mode 100644 index b9d1919a..00000000 --- a/cloudinit/handlers/cc_update_hostname.py +++ /dev/null @@ -1,101 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import cloudinit.util as util -import subprocess -import errno -from cloudinit.CloudConfig import per_always - -frequency = per_always - - -def handle(_name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): - log.debug("preserve_hostname is set. not updating hostname") - return - - (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) - try: - prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") - update_hostname(hostname, prev, log) - except Exception: - log.warn("failed to set hostname\n") - raise - - -# read hostname from a 'hostname' file -# allow for comments and stripping line endings. -# if file doesn't exist, or no contents, return default -def read_hostname(filename, default=None): - try: - fp = open(filename, "r") - lines = fp.readlines() - fp.close() - for line in lines: - hpos = line.find("#") - if hpos != -1: - line = line[0:hpos] - line = line.rstrip() - if line: - return line - except IOError as e: - if e.errno != errno.ENOENT: - raise - return default - - -def update_hostname(hostname, prev_file, log): - etc_file = "/etc/hostname" - - hostname_prev = None - hostname_in_etc = None - - try: - hostname_prev = read_hostname(prev_file) - except Exception as e: - log.warn("Failed to open %s: %s" % (prev_file, e)) - - try: - hostname_in_etc = read_hostname(etc_file) - except: - log.warn("Failed to open %s" % etc_file) - - update_files = [] - if not hostname_prev or hostname_prev != hostname: - update_files.append(prev_file) - - if (not hostname_in_etc or - (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): - update_files.append(etc_file) - - try: - for fname in update_files: - util.write_file(fname, "%s\n" % hostname, 0644) - log.debug("wrote %s to %s" % (hostname, fname)) - except: - log.warn("failed to write hostname to %s" % fname) - - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - log.debug("%s differs from %s. assuming user maintained" % - (prev_file, etc_file)) - - if etc_file in update_files: - log.debug("setting hostname to %s" % hostname) - subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py new file mode 100644 index 00000000..5d70ac43 --- /dev/null +++ b/cloudinit/transforms/__init__.py @@ -0,0 +1,221 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2008-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Chuck Short +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import os +import subprocess +import sys +import time +import traceback + +import yaml + +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) + +from cloudinit import log as logging +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DEF_HANDLER_VERSION = 1 +DEF_FREQ = PER_INSTANCE + + +# reads a cloudconfig module list, returns +# a 2 dimensional array suitable to pass to run_cc_modules +def read_cc_modules(cfg, name): + if name not in cfg: + return([]) + module_list = [] + # create 'module_list', an array of arrays + # where array[0] = config + # array[1] = freq + # array[2:] = arguemnts + for item in cfg[name]: + if isinstance(item, str): + module_list.append((item,)) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("failed to read '%s' item in config") + return(module_list) + + +def run_cc_modules(cc, module_list, log): + failures = [] + for cfg_mod in module_list: + name = cfg_mod[0] + freq = None + run_args = [] + if len(cfg_mod) > 1: + freq = cfg_mod[1] + if len(cfg_mod) > 2: + run_args = cfg_mod[2:] + + try: + log.debug("handling %s with freq=%s and args=%s" % + (name, freq, run_args)) + cc.handle(name, run_args, freq=freq) + except: + log.warn(traceback.format_exc()) + log.error("config handling of %s, %s, %s failed\n" % + (name, freq, run_args)) + failures.append(name) + + return(failures) + + +# always returns well formated values +# cfg is expected to have an entry 'output' in it, which is a dictionary +# that includes entries for 'init', 'config', 'final' or 'all' +# init: /var/log/cloud.out +# config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ] +# final: +# output: "| logger -p" +# error: "> /dev/null" +# this returns the specific 'mode' entry, cleanly formatted, with value +# None if if none is given +def get_output_cfg(cfg, mode="init"): + ret = [None, None] + if not 'output' in cfg: + return ret + + outcfg = cfg['output'] + if mode in outcfg: + modecfg = outcfg[mode] + else: + if 'all' not in outcfg: + return ret + # if there is a 'all' item in the output list + # then it applies to all users of this (init, config, final) + modecfg = outcfg['all'] + + # if value is a string, it specifies stdout and stderr + if isinstance(modecfg, str): + ret = [modecfg, modecfg] + + # if its a list, then we expect (stdout, stderr) + if isinstance(modecfg, list): + if len(modecfg) > 0: + ret[0] = modecfg[0] + if len(modecfg) > 1: + ret[1] = modecfg[1] + + # if it is a dictionary, expect 'out' and 'error' + # items, which indicate out and error + if isinstance(modecfg, dict): + if 'output' in modecfg: + ret[0] = modecfg['output'] + if 'error' in modecfg: + ret[1] = modecfg['error'] + + # if err's entry == "&1", then make it same as stdout + # as in shell syntax of "echo foo >/dev/null 2>&1" + if ret[1] == "&1": + ret[1] = ret[0] + + swlist = [">>", ">", "|"] + for i in range(len(ret)): + if not ret[i]: + continue + val = ret[i].lstrip() + found = False + for s in swlist: + if val.startswith(s): + val = "%s %s" % (s, val[len(s):].strip()) + found = True + break + if not found: + # default behavior is append + val = "%s %s" % (">>", val.strip()) + ret[i] = val + + return(ret) + + +# redirect_output(outfmt, errfmt, orig_out, orig_err) +# replace orig_out and orig_err with filehandles specified in outfmt or errfmt +# fmt can be: +# > FILEPATH +# >> FILEPATH +# | program [ arg1 [ arg2 [ ... ] ] ] +# +# with a '|', arguments are passed to shell, so one level of +# shell escape is required. +def redirect_output(outfmt, errfmt, o_out=sys.stdout, o_err=sys.stderr): + if outfmt: + (mode, arg) = outfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_out: + os.dup2(new_fp.fileno(), o_out.fileno()) + if errfmt == outfmt: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + if errfmt: + (mode, arg) = errfmt.split(" ", 1) + if mode == ">" or mode == ">>": + owith = "ab" + if mode == ">": + owith = "wb" + new_fp = open(arg, owith) + elif mode == "|": + proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE) + new_fp = proc.stdin + else: + raise TypeError("invalid type for outfmt: %s" % outfmt) + + if o_err: + os.dup2(new_fp.fileno(), o_err.fileno()) + return + + +def form_module_name(name): + canon_name = name.replace("-", "_") + if canon_name.endswith(".py"): + canon_name = canon_name[0:(len(canon_name) - 3)] + canon_name = canon_name.strip() + if not canon_name: + return None + if not canon_name.startswith("cc_"): + canon_name = 'cc_%s' % (canon_name) + return canon_name + + +def fixup_module(mod): + freq = getattr(mod, "frequency", None) + if not freq: + setattr(mod, 'frequency', PER_INSTANCE) + handler = getattr(mod, "handle", None) + if not handler: + def empty_handle(_name, _cfg, _cloud, _log, _args): + pass + setattr(mod, 'handle', empty_handle) + return mod diff --git a/cloudinit/transforms/cc_apt_pipelining.py b/cloudinit/transforms/cc_apt_pipelining.py new file mode 100644 index 00000000..0286a9ae --- /dev/null +++ b/cloudinit/transforms/cc_apt_pipelining.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance +default_file = "/etc/apt/apt.conf.d/90cloud-init-pipelining" + + +def handle(_name, cfg, _cloud, log, _args): + + apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) + apt_pipe_value = str(apt_pipe_value).lower() + + if apt_pipe_value == "false": + write_apt_snippet("0", log) + + elif apt_pipe_value in ("none", "unchanged", "os"): + return + + elif apt_pipe_value in str(range(0, 6)): + write_apt_snippet(apt_pipe_value, log) + + else: + log.warn("Invalid option for apt_pipeling: %s" % apt_pipe_value) + + +def write_apt_snippet(setting, log, f_name=default_file): + """ Writes f_name with apt pipeline depth 'setting' """ + + acquire_pipeline_depth = 'Acquire::http::Pipeline-Depth "%s";\n' + file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" + + (acquire_pipeline_depth % setting)) + + util.write_file(f_name, file_contents) + + log.debug("Wrote %s with APT pipeline setting" % f_name) diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py new file mode 100644 index 00000000..a7049bce --- /dev/null +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -0,0 +1,241 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback +import os +import glob +import cloudinit.CloudConfig as cc + + +def handle(_name, cfg, cloud, log, _args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + release = get_release() + + mirror = find_apt_mirror(cloud, cfg) + + log.debug("selected mirror at: %s" % mirror) + + if not util.get_cfg_option_bool(cfg, \ + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', \ + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # set up proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + contents = "Acquire::HTTP::Proxy \"%s\";\n" + with open(proxy_filename, "w") as fp: + fp.write(contents % proxy) + except Exception as e: + log.warn("Failed to write proxy to %s" % proxy_filename) + elif os.path.isfile(proxy_filename): + os.unlink(proxy_filename) + + # process 'apt_sources' + if 'apt_sources' in cfg: + errors = add_sources(cfg['apt_sources'], + {'MIRROR': mirror, 'RELEASE': release}) + for e in errors: + log.warn("Source Error: %s\n" % ':'.join(e)) + + dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) + if dconf_sel: + log.debug("setting debconf selections per cloud config") + try: + util.subp(('debconf-set-selections', '-'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections") + log.debug(traceback.format_exc()) + + pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cc.update_package_sources() + except subprocess.CalledProcessError as e: + log.warn("apt-get update failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if upgrade: + try: + cc.apt_get("upgrade") + except subprocess.CalledProcessError as e: + log.warn("apt upgrade failed") + log.debug(traceback.format_exc()) + errors.append(e) + + if len(pkglist): + try: + cc.install_packages(pkglist) + except subprocess.CalledProcessError as e: + log.warn("Failed to install packages: %s " % pkglist) + log.debug(traceback.format_exc()) + errors.append(e) + + if len(errors): + raise errors[0] + + return(True) + + +def mirror2lists_fileprefix(mirror): + string = mirror + # take of http:// or ftp:// + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if(oprefix == nprefix): + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + os.rename(filename, "%s%s" % (nprefix, filename[olen:])) + + +def get_release(): + stdout, _stderr = subprocess.Popen(['lsb_release', '-cs'], + stdout=subprocess.PIPE).communicate() + return(str(stdout).strip()) + + +def generate_sources_list(codename, mirror): + util.render_to_file('sources.list', '/etc/apt/sources.list', \ + {'mirror': mirror, 'codename': codename}) + + +def add_sources(srclist, searchList=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if searchList is None: + searchList = {} + elst = [] + + for ent in srclist: + if 'source' not in ent: + elst.append(["", "missing source"]) + continue + + source = ent['source'] + if source.startswith("ppa:"): + try: + util.subp(["add-apt-repository", source]) + except: + elst.append([source, "add-apt-repository failed"]) + continue + + source = util.render_string(source, searchList) + + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = "%s/%s" % \ + ("/etc/apt/sources.list.d/", ent['filename']) + + if ('keyid' in ent and 'key' not in ent): + ks = "keyserver.ubuntu.com" + if 'keyserver' in ent: + ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + elst.append([source, "failed to get key from %s" % ks]) + continue + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + elst.append([source, "failed add key"]) + + try: + util.write_file(ent['filename'], source + "\n", omode="ab") + except: + elst.append([source, "failed write to file %s" % ent['filename']]) + + return(elst) + + +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + # TODO: distro and defaults should be configurable + distro = "ubuntu" + defaults = { + 'ubuntu': "http://archive.ubuntu.com/ubuntu", + 'debian': "http://archive.debian.org/debian", + } + mirror = None + + cfg_mirror = cfg.get("apt_mirror", None) + if cfg_mirror: + mirror = cfg["apt_mirror"] + elif "apt_mirror_search" in cfg: + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + if cloud: + mirror = cloud.get_mirror() + + mydom = "" + + doms = [] + + if not mirror and cloud: + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + if not mirror: + doms.extend((".localdomain", "",)) + + mirror_list = [] + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % post) + + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = defaults[distro] + + return mirror diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py new file mode 100644 index 00000000..f584da02 --- /dev/null +++ b/cloudinit/transforms/cc_bootcmd.py @@ -0,0 +1,48 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +import tempfile +import os +from cloudinit.CloudConfig import per_always +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if "bootcmd" not in cfg: + return + + try: + content = util.shellify(cfg["bootcmd"]) + tmpf = tempfile.TemporaryFile() + tmpf.write(content) + tmpf.seek(0) + except: + log.warn("failed to shellify bootcmd") + raise + + try: + env = os.environ.copy() + env['INSTANCE_ID'] = cloud.get_instance_id() + subprocess.check_call(['/bin/sh'], env=env, stdin=tmpf) + tmpf.close() + except: + log.warn("failed to run commands from bootcmd") + raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py new file mode 100644 index 00000000..e821b261 --- /dev/null +++ b/cloudinit/transforms/cc_byobu.py @@ -0,0 +1,77 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + value = args[0] + else: + value = util.get_cfg_option_str(cfg, "byobu_by_default", "") + + if not value: + return + + if value == "user" or value == "system": + value = "enable-%s" % value + + valid = ("enable-user", "enable-system", "enable", + "disable-user", "disable-system", "disable") + if not value in valid: + log.warn("Unknown value %s for byobu_by_default" % value) + + mod_user = value.endswith("-user") + mod_sys = value.endswith("-system") + if value.startswith("enable"): + bl_inst = "install" + dc_val = "byobu byobu/launch-by-default boolean true" + mod_sys = True + else: + if value == "disable": + mod_user = True + mod_sys = True + bl_inst = "uninstall" + dc_val = "byobu byobu/launch-by-default boolean false" + + shcmd = "" + if mod_user: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += " || X=$(($X+1)); " + if mod_sys: + shcmd += "echo \"%s\" | debconf-set-selections" % dc_val + shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" + shcmd += " || X=$(($X+1)); " + + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] + + log.debug("setting byobu to %s" % value) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py new file mode 100644 index 00000000..3af6238a --- /dev/null +++ b/cloudinit/transforms/cc_ca_certs.py @@ -0,0 +1,90 @@ +# vi: ts=4 expandtab +# +# Author: Mike Milner +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +from subprocess import check_call +from cloudinit.util import (write_file, get_cfg_option_list_or_str, + delete_dir_contents, subp) + +CA_CERT_PATH = "/usr/share/ca-certificates/" +CA_CERT_FILENAME = "cloud-init-ca-certs.crt" +CA_CERT_CONFIG = "/etc/ca-certificates.conf" +CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" + + +def update_ca_certs(): + """ + Updates the CA certificate cache on the current machine. + """ + check_call(["update-ca-certificates"]) + + +def add_ca_certs(certs): + """ + Adds certificates to the system. To actually apply the new certificates + you must also call L{update_ca_certs}. + + @param certs: A list of certificate strings. + """ + if certs: + cert_file_contents = "\n".join(certs) + cert_file_fullpath = os.path.join(CA_CERT_PATH, CA_CERT_FILENAME) + write_file(cert_file_fullpath, cert_file_contents, mode=0644) + # Append cert filename to CA_CERT_CONFIG file. + write_file(CA_CERT_CONFIG, "\n%s" % CA_CERT_FILENAME, omode="a") + + +def remove_default_ca_certs(): + """ + Removes all default trusted CA certificates from the system. To actually + apply the change you must also call L{update_ca_certs}. + """ + delete_dir_contents(CA_CERT_PATH) + delete_dir_contents(CA_CERT_SYSTEM_PATH) + write_file(CA_CERT_CONFIG, "", mode=0644) + debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" + subp(('debconf-set-selections', '-'), debconf_sel) + + +def handle(_name, cfg, _cloud, log, _args): + """ + Call to handle ca-cert sections in cloud-config file. + + @param name: The module name "ca-cert" from cloud.cfg + @param cfg: A nested dict containing the entire cloud config contents. + @param cloud: The L{CloudInit} object in use. + @param log: Pre-initialized Python logger object to use for logging. + @param args: Any module arguments from cloud.cfg + """ + # If there isn't a ca-certs section in the configuration don't do anything + if "ca-certs" not in cfg: + return + ca_cert_cfg = cfg['ca-certs'] + + # If there is a remove-defaults option set to true, remove the system + # default trusted CA certs first. + if ca_cert_cfg.get("remove-defaults", False): + log.debug("removing default certificates") + remove_default_ca_certs() + + # If we are given any new trusted CA certs to add, add them. + if "trusted" in ca_cert_cfg: + trusted_certs = get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + if trusted_certs: + log.debug("adding %d certificates" % len(trusted_certs)) + add_ca_certs(trusted_certs) + + # Update the system with the new cert configuration. + update_ca_certs() diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py new file mode 100644 index 00000000..941e04fe --- /dev/null +++ b/cloudinit/transforms/cc_chef.py @@ -0,0 +1,119 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Avishai Ish-Shalom +# Author: Mike Moulton +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import json +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +ruby_version_default = "1.8" + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a chef key in the configuration don't do anything + if 'chef' not in cfg: + return + chef_cfg = cfg['chef'] + + # ensure the chef directories we use exist + mkdirs(['/etc/chef', '/var/log/chef', '/var/lib/chef', + '/var/cache/chef', '/var/backups/chef', '/var/run/chef']) + + # set the validation key based on the presence of either 'validation_key' + # or 'validation_cert'. In the case where both exist, 'validation_key' + # takes precedence + for key in ('validation_key', 'validation_cert'): + if key in chef_cfg and chef_cfg[key]: + with open('/etc/chef/validation.pem', 'w') as validation_key_fh: + validation_key_fh.write(chef_cfg[key]) + break + + # create the chef config from template + util.render_to_file('chef_client.rb', '/etc/chef/client.rb', + {'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + cloud.datasource.get_instance_id()), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name']}) + + # set the firstboot json + with open('/etc/chef/firstboot.json', 'w') as firstboot_json_fh: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in initial_attributes.keys(): + initial_json[k] = initial_attributes[k] + firstboot_json_fh.write(json.dumps(initial_json)) + + # If chef is not installed, we install chef based on 'install_type' + if not os.path.isfile('/usr/bin/chef-client'): + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # this will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + ruby_version_default) + install_chef_from_gems(ruby_version, chef_version) + # and finally, run chef-client + log.debug('running chef-client') + subprocess.check_call(['/usr/bin/chef-client', '-d', '-i', '1800', + '-s', '20']) + else: + # this will install and run the chef-client from packages + cc.install_packages(('chef',)) + + +def get_ruby_packages(version): + # return a list of packages needed to install ruby at version + pkgs = ['ruby%s' % version, 'ruby%s-dev' % version] + if version == "1.8": + pkgs.extend(('libopenssl-ruby1.8', 'rubygems1.8')) + return(pkgs) + + +def install_chef_from_gems(ruby_version, chef_version=None): + cc.install_packages(get_ruby_packages(ruby_version)) + if not os.path.exists('/usr/bin/gem'): + os.symlink('/usr/bin/gem%s' % ruby_version, '/usr/bin/gem') + if not os.path.exists('/usr/bin/ruby'): + os.symlink('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') + if chef_version: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q']) + else: + subprocess.check_call(['/usr/bin/gem', 'install', 'chef', + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q']) + + +def ensure_dir(d): + if not os.path.exists(d): + os.makedirs(d) + + +def mkdirs(dirs): + for d in dirs: + ensure_dir(d) diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py new file mode 100644 index 00000000..6b31ea8e --- /dev/null +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -0,0 +1,30 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import cloudinit.util as util +import subprocess +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, _log, _args): + if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + fwall = "route add -host 169.254.169.254 reject" + subprocess.call(fwall.split(' ')) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py new file mode 100644 index 00000000..abb4ca32 --- /dev/null +++ b/cloudinit/transforms/cc_final_message.py @@ -0,0 +1,58 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_always +import sys +from cloudinit import util, boot_finished +import time + +frequency = per_always + +final_message = "cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + msg_in = args[0] + else: + msg_in = util.get_cfg_option_str(cfg, "final_message", final_message) + + try: + uptimef = open("/proc/uptime") + uptime = uptimef.read().split(" ")[0] + uptimef.close() + except IOError as e: + log.warn("unable to open /proc/uptime\n") + uptime = "na" + + try: + ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) + except: + ts = "na" + + try: + subs = {'UPTIME': uptime, 'TIMESTAMP': ts} + sys.stdout.write("%s\n" % util.render_string(msg_in, subs)) + except Exception as e: + log.warn("failed to render string to stdout: %s" % e) + + fp = open(boot_finished, "wb") + fp.write(uptime + "\n") + fp.close() diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py new file mode 100644 index 00000000..35ec3fa7 --- /dev/null +++ b/cloudinit/transforms/cc_foo.py @@ -0,0 +1,29 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#import cloudinit +#import cloudinit.util as util +from cloudinit.CloudConfig import per_instance + +frequency = per_instance + + +def handle(_name, _cfg, _cloud, _log, _args): + print "hi" diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py new file mode 100644 index 00000000..9f3a7eaf --- /dev/null +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import traceback +import os + + +def handle(_name, cfg, _cloud, log, _args): + idevs = None + idevs_empty = None + + if "grub-dpkg" in cfg: + idevs = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], + "grub-pc/install_devices_empty", None) + + if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): + if idevs == None: + idevs = "" + if idevs_empty == None: + idevs_empty = "true" + else: + if idevs_empty == None: + idevs_empty = "false" + if idevs == None: + idevs = "/dev/sda" + for dev in ("/dev/sda", "/dev/vda", "/dev/sda1", "/dev/vda1"): + if os.path.exists(dev): + idevs = dev + break + + # now idevs and idevs_empty are set to determined values + # or, those set by user + + dconf_sel = "grub-pc grub-pc/install_devices string %s\n" % idevs + \ + "grub-pc grub-pc/install_devices_empty boolean %s\n" % idevs_empty + log.debug("setting grub debconf-set-selections with '%s','%s'" % + (idevs, idevs_empty)) + + try: + util.subp(('debconf-set-selections'), dconf_sel) + except: + log.error("Failed to run debconf-set-selections for grub-dpkg") + log.debug(traceback.format_exc()) diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py new file mode 100644 index 00000000..73a477c0 --- /dev/null +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +import subprocess + +frequency = per_instance + + +def handle(_name, cfg, _cloud, log, _args): + cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] + fp_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_fp_console_blacklist", []) + key_blacklist = util.get_cfg_option_list_or_str(cfg, + "ssh_key_console_blacklist", ["ssh-dss"]) + try: + confp = open('/dev/console', "wb") + cmd.append(','.join(fp_blacklist)) + cmd.append(','.join(key_blacklist)) + subprocess.call(cmd, stdout=confp) + confp.close() + except: + log.warn("writing keys to console value") + raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py new file mode 100644 index 00000000..a4113cbe --- /dev/null +++ b/cloudinit/transforms/cc_landscape.py @@ -0,0 +1,75 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +from cloudinit.CloudConfig import per_instance +from configobj import ConfigObj + +frequency = per_instance + +lsc_client_cfg_file = "/etc/landscape/client.conf" + +# defaults taken from stock client.conf in landscape-client 11.07.1.1-0ubuntu2 +lsc_builtincfg = { + 'client': { + 'log_level': "info", + 'url': "https://landscape.canonical.com/message-system", + 'ping_url': "http://landscape.canonical.com/ping", + 'data_path': "/var/lib/landscape/client", + } +} + + +def handle(_name, cfg, _cloud, log, _args): + """ + Basically turn a top level 'landscape' entry with a 'client' dict + and render it to ConfigObj format under '[client]' section in + /etc/landscape/client.conf + """ + + ls_cloudcfg = cfg.get("landscape", {}) + + if not isinstance(ls_cloudcfg, dict): + raise(Exception("'landscape' existed in config, but not a dict")) + + merged = mergeTogether([lsc_builtincfg, lsc_client_cfg_file, ls_cloudcfg]) + + if not os.path.isdir(os.path.dirname(lsc_client_cfg_file)): + os.makedirs(os.path.dirname(lsc_client_cfg_file)) + + with open(lsc_client_cfg_file, "w") as fp: + merged.write(fp) + + log.debug("updated %s" % lsc_client_cfg_file) + + +def mergeTogether(objs): + """ + merge together ConfigObj objects or things that ConfigObj() will take in + later entries override earlier + """ + cfg = ConfigObj({}) + for obj in objs: + if isinstance(obj, ConfigObj): + cfg.merge(obj) + else: + cfg.merge(ConfigObj(obj)) + return cfg diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py new file mode 100644 index 00000000..2bb22fdb --- /dev/null +++ b/cloudinit/transforms/cc_locale.py @@ -0,0 +1,54 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os.path +import subprocess +import traceback + + +def apply_locale(locale, cfgfile): + if os.path.exists('/usr/sbin/locale-gen'): + subprocess.Popen(['locale-gen', locale]).communicate() + if os.path.exists('/usr/sbin/update-locale'): + subprocess.Popen(['update-locale', locale]).communicate() + + util.render_to_file('default-locale', cfgfile, {'locale': locale}) + + +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + locale = args[0] + else: + locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) + + locale_cfgfile = util.get_cfg_option_str(cfg, "locale_configfile", + "/etc/default/locale") + + if not locale: + return + + log.debug("setting locale to %s" % locale) + + try: + apply_locale(locale, locale_cfgfile) + except Exception as e: + log.debug(traceback.format_exc(e)) + raise Exception("failed to apply locale %s" % locale) diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py new file mode 100644 index 00000000..a2a6230c --- /dev/null +++ b/cloudinit/transforms/cc_mcollective.py @@ -0,0 +1,99 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Marc Cluet +# Based on code by Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + +pubcert_file = "/etc/mcollective/ssl/server-public.pem" +pricert_file = "/etc/mcollective/ssl/server-private.pem" + + +# Our fake header section +class FakeSecHead(object): + def __init__(self, fp): + self.fp = fp + self.sechead = '[nullsection]\n' + + def readline(self): + if self.sechead: + try: + return self.sechead + finally: + self.sechead = None + else: + return self.fp.readline() + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a mcollective key in the configuration don't do anything + if 'mcollective' not in cfg: + return + mcollective_cfg = cfg['mcollective'] + # Start by installing the mcollective package ... + cc.install_packages(("mcollective",)) + + # ... and then update the mcollective configuration + if 'conf' in mcollective_cfg: + # Create object for reading server.cfg values + mcollective_config = ConfigParser.ConfigParser() + # Read server.cfg values from original file in order to be able to mix + # the rest up + mcollective_config.readfp(FakeSecHead(open('/etc/mcollective/' + 'server.cfg'))) + for cfg_name, cfg in mcollective_cfg['conf'].iteritems(): + if cfg_name == 'public-cert': + util.write_file(pubcert_file, cfg, mode=0644) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_public', pubcert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + elif cfg_name == 'private-cert': + util.write_file(pricert_file, cfg, mode=0600) + mcollective_config.set(cfg_name, + 'plugin.ssl_server_private', pricert_file) + mcollective_config.set(cfg_name, 'securityprovider', 'ssl') + else: + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + mcollective_config.set(cfg_name, o, v) + # We got all our config as wanted we'll rename + # the previous server.cfg and create our new one + os.rename('/etc/mcollective/server.cfg', + '/etc/mcollective/server.cfg.old') + outputfile = StringIO.StringIO() + mcollective_config.write(outputfile) + # Now we got the whole file, write to disk except first line + # Note below, that we've just used ConfigParser because it generally + # works. Below, we remove the initial 'nullsection' header + # and then change 'key = value' to 'key: value'. The global + # search and replace of '=' with ':' could be problematic though. + # this most likely needs fixing. + util.write_file('/etc/mcollective/server.cfg', + outputfile.getvalue().replace('[nullsection]\n', '').replace(' =', + ':'), + mode=0644) + + # Start mcollective + subprocess.check_call(['service', 'mcollective', 'start']) diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py new file mode 100644 index 00000000..6cdd74e8 --- /dev/null +++ b/cloudinit/transforms/cc_mounts.py @@ -0,0 +1,179 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import os +import re +from string import whitespace # pylint: disable=W0402 + + +def is_mdname(name): + # return true if this is a metadata service name + if name in ["ami", "root", "swap"]: + return True + # names 'ephemeral0' or 'ephemeral1' + # 'ebs[0-9]' appears when '--block-device-mapping sdf=snap-d4d90bbc' + for enumname in ("ephemeral", "ebs"): + if name.startswith(enumname) and name.find(":") == -1: + return True + return False + + +def handle(_name, cfg, cloud, log, _args): + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno + defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + defvals = cfg.get("mount_default_fields", defvals) + + # these are our default set of mounts + defmnts = [["ephemeral0", "/mnt", "auto", defvals[3], "0", "2"], + ["swap", "none", "swap", "sw", "0", "0"]] + + cfgmnt = [] + if "mounts" in cfg: + cfgmnt = cfg["mounts"] + + # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 + shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" + shortname = re.compile(shortname_filter) + + for i in range(len(cfgmnt)): + # skip something that wasn't a list + if not isinstance(cfgmnt[i], list): + continue + + # workaround, allow user to specify 'ephemeral' + # rather than more ec2 correct 'ephemeral0' + if cfgmnt[i][0] == "ephemeral": + cfgmnt[i][0] = "ephemeral0" + + if is_mdname(cfgmnt[i][0]): + newname = cloud.device_name_to_device(cfgmnt[i][0]) + if not newname: + log.debug("ignoring nonexistant named mount %s" % cfgmnt[i][0]) + cfgmnt[i][1] = None + else: + if newname.startswith("/"): + cfgmnt[i][0] = newname + else: + cfgmnt[i][0] = "/dev/%s" % newname + else: + if shortname.match(cfgmnt[i][0]): + cfgmnt[i][0] = "/dev/%s" % cfgmnt[i][0] + + # in case the user did not quote a field (likely fs-freq, fs_passno) + # but do not convert None to 'None' (LP: #898365) + for j in range(len(cfgmnt[i])): + if isinstance(cfgmnt[i][j], int): + cfgmnt[i][j] = str(cfgmnt[i][j]) + + for i in range(len(cfgmnt)): + # fill in values with defaults from defvals above + for j in range(len(defvals)): + if len(cfgmnt[i]) <= j: + cfgmnt[i].append(defvals[j]) + elif cfgmnt[i][j] is None: + cfgmnt[i][j] = defvals[j] + + # if the second entry in the list is 'None' this + # clears all previous entries of that same 'fs_spec' + # (fs_spec is the first field in /etc/fstab, ie, that device) + if cfgmnt[i][1] is None: + for j in range(i): + if cfgmnt[j][0] == cfgmnt[i][0]: + cfgmnt[j][1] = None + + # for each of the "default" mounts, add them only if no other + # entry has the same device name + for defmnt in defmnts: + devname = cloud.device_name_to_device(defmnt[0]) + if devname is None: + continue + if devname.startswith("/"): + defmnt[0] = devname + else: + defmnt[0] = "/dev/%s" % devname + + cfgmnt_has = False + for cfgm in cfgmnt: + if cfgm[0] == defmnt[0]: + cfgmnt_has = True + break + + if cfgmnt_has: + continue + cfgmnt.append(defmnt) + + # now, each entry in the cfgmnt list has all fstab values + # if the second field is None (not the string, the value) we skip it + actlist = [x for x in cfgmnt if x[1] is not None] + + if len(actlist) == 0: + return + + comment = "comment=cloudconfig" + cc_lines = [] + needswap = False + dirs = [] + for line in actlist: + # write 'comment' in the fs_mntops, entry, claiming this + line[3] = "%s,comment=cloudconfig" % line[3] + if line[2] == "swap": + needswap = True + if line[1].startswith("/"): + dirs.append(line[1]) + cc_lines.append('\t'.join(line)) + + fstab_lines = [] + fstab = open("/etc/fstab", "r+") + ws = re.compile("[%s]+" % whitespace) + for line in fstab.read().splitlines(): + try: + toks = ws.split(line) + if toks[3].find(comment) != -1: + continue + except: + pass + fstab_lines.append(line) + + fstab_lines.extend(cc_lines) + + fstab.seek(0) + fstab.write("%s\n" % '\n'.join(fstab_lines)) + fstab.truncate() + fstab.close() + + if needswap: + try: + util.subp(("swapon", "-a")) + except: + log.warn("Failed to enable swap") + + for d in dirs: + if os.path.exists(d): + continue + try: + os.makedirs(d) + except: + log.warn("Failed to make '%s' config-mount\n", d) + + try: + util.subp(("mount", "-a")) + except: + log.warn("'mount -a' failed") diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py new file mode 100644 index 00000000..a7ff74e1 --- /dev/null +++ b/cloudinit/transforms/cc_phone_home.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from cloudinit.CloudConfig import per_instance +import cloudinit.util as util +from time import sleep + +frequency = per_instance +post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', + 'hostname'] + + +# phone_home: +# url: http://my.foo.bar/$INSTANCE/ +# post: all +# tries: 10 +# +# phone_home: +# url: http://my.foo.bar/$INSTANCE_ID/ +# post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id +# +def handle(_name, cfg, cloud, log, args): + if len(args) != 0: + ph_cfg = util.read_conf(args[0]) + else: + if not 'phone_home' in cfg: + return + ph_cfg = cfg['phone_home'] + + if 'url' not in ph_cfg: + log.warn("no 'url' token in phone_home") + return + + url = ph_cfg['url'] + post_list = ph_cfg.get('post', 'all') + tries = ph_cfg.get('tries', 10) + try: + tries = int(tries) + except: + log.warn("tries is not an integer. using 10") + tries = 10 + + if post_list == "all": + post_list = post_list_all + + all_keys = {} + all_keys['instance_id'] = cloud.get_instance_id() + all_keys['hostname'] = cloud.get_hostname() + + pubkeys = { + 'pub_key_dsa': '/etc/ssh/ssh_host_dsa_key.pub', + 'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub', + 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', + } + + for n, path in pubkeys.iteritems(): + try: + fp = open(path, "rb") + all_keys[n] = fp.read() + fp.close() + except: + log.warn("%s: failed to open in phone_home" % path) + + submit_keys = {} + for k in post_list: + if k in all_keys: + submit_keys[k] = all_keys[k] + else: + submit_keys[k] = "N/A" + log.warn("requested key %s from 'post' list not available") + + url = util.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + + null_exc = object() + last_e = null_exc + for i in range(0, tries): + try: + util.readurl(url, submit_keys) + log.debug("succeeded submit to %s on try %i" % (url, i + 1)) + return + except Exception as e: + log.debug("failed to post to %s on try %i" % (url, i + 1)) + last_e = e + sleep(3) + + log.warn("failed to post to %s in %i tries" % (url, tries)) + if last_e is not null_exc: + raise(last_e) + + return diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py new file mode 100644 index 00000000..6fc475f6 --- /dev/null +++ b/cloudinit/transforms/cc_puppet.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import pwd +import socket +import subprocess +import StringIO +import ConfigParser +import cloudinit.CloudConfig as cc +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + # If there isn't a puppet key in the configuration don't do anything + if 'puppet' not in cfg: + return + puppet_cfg = cfg['puppet'] + # Start by installing the puppet package ... + cc.install_packages(("puppet",)) + + # ... and then update the puppet configuration + if 'conf' in puppet_cfg: + # Add all sections from the conf object to puppet.conf + puppet_conf_fh = open('/etc/puppet/puppet.conf', 'r') + # Create object for reading puppet.conf values + puppet_config = ConfigParser.ConfigParser() + # Read puppet.conf values from original file in order to be able to + # mix the rest up + puppet_config.readfp(StringIO.StringIO(''.join(i.lstrip() for i in + puppet_conf_fh.readlines()))) + # Close original file, no longer needed + puppet_conf_fh.close() + for cfg_name, cfg in puppet_cfg['conf'].iteritems(): + # ca_cert configuration is a special case + # Dump the puppetmaster ca certificate in the correct place + if cfg_name == 'ca_cert': + # Puppet ssl sub-directory isn't created yet + # Create it with the proper permissions and ownership + os.makedirs('/var/lib/puppet/ssl') + os.chmod('/var/lib/puppet/ssl', 0771) + os.chown('/var/lib/puppet/ssl', + pwd.getpwnam('puppet').pw_uid, 0) + os.makedirs('/var/lib/puppet/ssl/certs/') + os.chown('/var/lib/puppet/ssl/certs/', + pwd.getpwnam('puppet').pw_uid, 0) + ca_fh = open('/var/lib/puppet/ssl/certs/ca.pem', 'w') + ca_fh.write(cfg) + ca_fh.close() + os.chown('/var/lib/puppet/ssl/certs/ca.pem', + pwd.getpwnam('puppet').pw_uid, 0) + util.restorecon_if_possible('/var/lib/puppet', recursive=True) + else: + #puppet_conf_fh.write("\n[%s]\n" % (cfg_name)) + # If puppet.conf already has this section we don't want to + # write it again + if puppet_config.has_section(cfg_name) == False: + puppet_config.add_section(cfg_name) + # Iterate throug the config items, we'll use ConfigParser.set + # to overwrite or create new items as needed + for o, v in cfg.iteritems(): + if o == 'certname': + # Expand %f as the fqdn + v = v.replace("%f", socket.getfqdn()) + # Expand %i as the instance id + v = v.replace("%i", + cloud.datasource.get_instance_id()) + # certname needs to be downcase + v = v.lower() + puppet_config.set(cfg_name, o, v) + #puppet_conf_fh.write("%s=%s\n" % (o, v)) + # We got all our config as wanted we'll rename + # the previous puppet.conf and create our new one + os.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + with open('/etc/puppet/puppet.conf', 'wb') as configfile: + puppet_config.write(configfile) + util.restorecon_if_possible('/etc/puppet/puppet.conf') + # Set puppet to automatically start + if os.path.exists('/etc/default/puppet'): + subprocess.check_call(['sed', '-i', + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet']) + elif os.path.exists('/bin/systemctl'): + subprocess.check_call(['/bin/systemctl', 'enable', 'puppet.service']) + elif os.path.exists('/sbin/chkconfig'): + subprocess.check_call(['/sbin/chkconfig', 'puppet', 'on']) + else: + log.warn("Do not know how to enable puppet service on this system") + # Start puppetd + subprocess.check_call(['service', 'puppet', 'start']) diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py new file mode 100644 index 00000000..2dc66def --- /dev/null +++ b/cloudinit/transforms/cc_resizefs.py @@ -0,0 +1,108 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import os +import stat +import sys +import time +import tempfile +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + resize_root = False + if str(args[0]).lower() in ['true', '1', 'on', 'yes']: + resize_root = True + else: + resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) + + if str(resize_root).lower() in ['false', '0']: + return + + # we use mktemp rather than mkstemp because early in boot nothing + # else should be able to race us for this, and we need to mknod. + devpth = tempfile.mktemp(prefix="cloudinit.resizefs.", dir="/run") + + try: + st_dev = os.stat("/").st_dev + dev = os.makedev(os.major(st_dev), os.minor(st_dev)) + os.mknod(devpth, 0400 | stat.S_IFBLK, dev) + except: + if util.is_container(): + log.debug("inside container, ignoring mknod failure in resizefs") + return + log.warn("Failed to make device node to resize /") + raise + + cmd = ['blkid', '-c', '/dev/null', '-sTYPE', '-ovalue', devpth] + try: + (fstype, _err) = util.subp(cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to get filesystem type of maj=%s, min=%s via: %s" % + (os.major(st_dev), os.minor(st_dev), cmd)) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + if str(fstype).startswith("ext"): + resize_cmd = ['resize2fs', devpth] + elif fstype == "xfs": + resize_cmd = ['xfs_growfs', devpth] + else: + os.unlink(devpth) + log.debug("not resizing unknown filesystem %s" % fstype) + return + + if resize_root == "noblock": + fid = os.fork() + if fid == 0: + try: + do_resize(resize_cmd, devpth, log) + os._exit(0) # pylint: disable=W0212 + except Exception as exc: + sys.stderr.write("Failed: %s" % exc) + os._exit(1) # pylint: disable=W0212 + else: + do_resize(resize_cmd, devpth, log) + + log.debug("resizing root filesystem (type=%s, maj=%i, min=%i, val=%s)" % + (str(fstype).rstrip("\n"), os.major(st_dev), os.minor(st_dev), + resize_root)) + + return + + +def do_resize(resize_cmd, devpth, log): + try: + start = time.time() + util.subp(resize_cmd) + except subprocess.CalledProcessError as e: + log.warn("Failed to resize filesystem (%s)" % resize_cmd) + log.warn("output=%s\nerror=%s\n", e.output[0], e.output[1]) + os.unlink(devpth) + raise + + os.unlink(devpth) + log.debug("resize took %s seconds" % (time.time() - start)) diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py new file mode 100644 index 00000000..5ed0848f --- /dev/null +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -0,0 +1,78 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +## +## The purpose of this script is to allow cloud-init to consume +## rightscale style userdata. rightscale user data is key-value pairs +## in a url-query-string like format. +## +## for cloud-init support, there will be a key named +## 'CLOUD_INIT_REMOTE_HOOK'. +## +## This cloud-config module will +## - read the blob of data from raw user data, and parse it as key/value +## - for each key that is found, download the content to +## the local instance/scripts directory and set them executable. +## - the files in that directory will be run by the user-scripts module +## Therefore, this must run before that. +## +## + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur +from urlparse import parse_qs + +frequency = per_instance +my_name = "cc_rightscale_userdata" +my_hookname = 'CLOUD_INIT_REMOTE_HOOK' + + +def handle(_name, _cfg, cloud, log, _args): + try: + ud = cloud.get_userdata_raw() + except: + log.warn("failed to get raw userdata in %s" % my_name) + return + + try: + mdict = parse_qs(ud) + if not my_hookname in mdict: + return + except: + log.warn("failed to urlparse.parse_qa(userdata_raw())") + raise + + scripts_d = get_ipath_cur('scripts') + i = 0 + first_e = None + for url in mdict[my_hookname]: + fname = "%s/rightscale-%02i" % (scripts_d, i) + i = i + 1 + try: + content = util.readurl(url) + util.write_file(fname, content, mode=0700) + except Exception as e: + if not first_e: + first_e = None + log.warn("%s failed to read %s: %s" % (my_name, url, e)) + + if first_e: + raise(e) diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py new file mode 100644 index 00000000..ac7f2c74 --- /dev/null +++ b/cloudinit/transforms/cc_rsyslog.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab syntax=python +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit +import logging +import cloudinit.util as util +import traceback + +DEF_FILENAME = "20-cloud-config.conf" +DEF_DIR = "/etc/rsyslog.d" + + +def handle(_name, cfg, _cloud, log, _args): + # rsyslog: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + + # process 'rsyslog' + if not 'rsyslog' in cfg: + return + + def_dir = cfg.get('rsyslog_dir', DEF_DIR) + def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) + + files = [] + elst = [] + for ent in cfg['rsyslog']: + if isinstance(ent, dict): + if not "content" in ent: + elst.append((ent, "no 'content' entry")) + continue + content = ent['content'] + filename = ent.get("filename", def_fname) + else: + content = ent + filename = def_fname + + if not filename.startswith("/"): + filename = "%s/%s" % (def_dir, filename) + + omode = "ab" + # truncate filename first time you see it + if filename not in files: + omode = "wb" + files.append(filename) + + try: + util.write_file(filename, content + "\n", omode=omode) + except Exception as e: + log.debug(traceback.format_exc(e)) + elst.append((content, "failed to write to %s" % filename)) + + # need to restart syslogd + restarted = False + try: + # if this config module is running at cloud-init time + # (before rsyslog is running) we don't actually have to + # restart syslog. + # + # upstart actually does what we want here, in that it doesn't + # start a service that wasn't running already on 'restart' + # it will also return failure on the attempt, so 'restarted' + # won't get set + log.debug("restarting rsyslog") + util.subp(['service', 'rsyslog', 'restart']) + restarted = True + + except Exception as e: + elst.append(("restart", str(e))) + + if restarted: + # this only needs to run if we *actually* restarted + # syslog above. + cloudinit.logging_set_from_cfg_file() + log = logging.getLogger() + log.debug("rsyslog configured %s" % files) + + for e in elst: + log.warn("rsyslog error: %s\n" % ':'.join(e)) + + return diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py new file mode 100644 index 00000000..f7e8c671 --- /dev/null +++ b/cloudinit/transforms/cc_runcmd.py @@ -0,0 +1,32 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if "runcmd" not in cfg: + return + outfile = "%s/runcmd" % cloud.get_ipath('scripts') + try: + content = util.shellify(cfg["runcmd"]) + util.write_file(outfile, content, 0700) + except: + log.warn("failed to open %s for runcmd" % outfile) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py new file mode 100644 index 00000000..1a3b5039 --- /dev/null +++ b/cloudinit/transforms/cc_salt_minion.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Author: Jeff Bauer +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import subprocess +import cloudinit.CloudConfig as cc +import yaml + + +def handle(_name, cfg, _cloud, _log, _args): + # If there isn't a salt key in the configuration don't do anything + if 'salt_minion' not in cfg: + return + salt_cfg = cfg['salt_minion'] + # Start by installing the salt package ... + cc.install_packages(("salt",)) + config_dir = '/etc/salt' + if not os.path.isdir(config_dir): + os.makedirs(config_dir) + # ... and then update the salt configuration + if 'conf' in salt_cfg: + # Add all sections from the conf object to /etc/salt/minion + minion_config = os.path.join(config_dir, 'minion') + yaml.dump(salt_cfg['conf'], + file(minion_config, 'w'), + default_flow_style=False) + # ... copy the key pair if specified + if 'public_key' in salt_cfg and 'private_key' in salt_cfg: + pki_dir = '/etc/salt/pki' + cumask = os.umask(077) + if not os.path.isdir(pki_dir): + os.makedirs(pki_dir) + pub_name = os.path.join(pki_dir, 'minion.pub') + pem_name = os.path.join(pki_dir, 'minion.pem') + with open(pub_name, 'w') as f: + f.write(salt_cfg['public_key']) + with open(pem_name, 'w') as f: + f.write(salt_cfg['private_key']) + os.umask(cumask) + + # Start salt-minion + subprocess.check_call(['service', 'salt-minion', 'start']) diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py new file mode 100644 index 00000000..41a74754 --- /dev/null +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +from cloudinit import get_cpath + +frequency = per_always +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-boot") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py new file mode 100644 index 00000000..a2981eab --- /dev/null +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_cpath + +frequency = per_instance +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-instance") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py new file mode 100644 index 00000000..a69151da --- /dev/null +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_once +from cloudinit import get_cpath + +frequency = per_once +runparts_path = "%s/%s" % (get_cpath(), "scripts/per-once") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py new file mode 100644 index 00000000..933aa4e0 --- /dev/null +++ b/cloudinit/transforms/cc_scripts_user.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_instance +from cloudinit import get_ipath_cur + +frequency = per_instance +runparts_path = "%s/%s" % (get_ipath_cur(), "scripts") + + +def handle(_name, _cfg, _cloud, log, _args): + try: + util.runparts(runparts_path) + except: + log.warn("failed to run-parts in %s" % runparts_path) + raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py new file mode 100644 index 00000000..acea74d9 --- /dev/null +++ b/cloudinit/transforms/cc_set_hostname.py @@ -0,0 +1,42 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not setting hostname") + return(True) + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + set_hostname(hostname, log) + except Exception: + util.logexc(log) + log.warn("failed to set hostname to %s\n", hostname) + + return(True) + + +def set_hostname(hostname, log): + util.subp(['hostname', hostname]) + util.write_file("/etc/hostname", "%s\n" % hostname, 0644) + log.debug("populated /etc/hostname with %s on first boot", hostname) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py new file mode 100644 index 00000000..9d0bbdb8 --- /dev/null +++ b/cloudinit/transforms/cc_set_passwords.py @@ -0,0 +1,129 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import sys +import random +from string import letters, digits # pylint: disable=W0402 + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + # if run from command line, and give args, wipe the chpasswd['list'] + password = args[0] + if 'chpasswd' in cfg and 'list' in cfg['chpasswd']: + del cfg['chpasswd']['list'] + else: + password = util.get_cfg_option_str(cfg, "password", None) + + expire = True + pw_auth = "no" + change_pwauth = False + plist = None + + if 'chpasswd' in cfg: + chfg = cfg['chpasswd'] + plist = util.get_cfg_option_str(chfg, 'list', plist) + expire = util.get_cfg_option_bool(chfg, 'expire', expire) + + if not plist and password: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + plist = "%s:%s" % (user, password) + + errors = [] + if plist: + plist_in = [] + randlist = [] + users = [] + for line in plist.splitlines(): + u, p = line.split(':', 1) + if p == "R" or p == "RANDOM": + p = rand_user_password() + randlist.append("%s:%s" % (u, p)) + plist_in.append("%s:%s" % (u, p)) + users.append(u) + + ch_in = '\n'.join(plist_in) + try: + util.subp(['chpasswd'], ch_in) + log.debug("changed password for %s:" % users) + except Exception as e: + errors.append(e) + log.warn("failed to set passwords with chpasswd: %s" % e) + + if len(randlist): + sys.stdout.write("%s\n%s\n" % ("Set the following passwords\n", + '\n'.join(randlist))) + + if expire: + enum = len(errors) + for u in users: + try: + util.subp(['passwd', '--expire', u]) + except Exception as e: + errors.append(e) + log.warn("failed to expire account for %s" % u) + if enum == len(errors): + log.debug("expired passwords for: %s" % u) + + if 'ssh_pwauth' in cfg: + val = str(cfg['ssh_pwauth']).lower() + if val in ("true", "1", "yes"): + pw_auth = "yes" + change_pwauth = True + elif val in ("false", "0", "no"): + pw_auth = "no" + change_pwauth = True + else: + change_pwauth = False + + if change_pwauth: + pa_s = "\(#*\)\(PasswordAuthentication[[:space:]]\+\)\(yes\|no\)" + msg = "set PasswordAuthentication to '%s'" % pw_auth + try: + cmd = ['sed', '-i', 's,%s,\\2%s,' % (pa_s, pw_auth), + '/etc/ssh/sshd_config'] + util.subp(cmd) + log.debug(msg) + except Exception as e: + log.warn("failed %s" % msg) + errors.append(e) + + try: + p = util.subp(['service', cfg.get('ssh_svcname', 'ssh'), + 'restart']) + log.debug("restarted sshd") + except: + log.warn("restart of ssh failed") + + if len(errors): + raise(errors[0]) + + return + + +def rand_str(strlen=32, select_from=letters + digits): + return("".join([random.choice(select_from) for _x in range(0, strlen)])) + + +def rand_user_password(pwlen=9): + selfrom = (letters.translate(None, 'loLOI') + + digits.translate(None, '01')) + return(rand_str(pwlen, select_from=selfrom)) diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py new file mode 100644 index 00000000..48eb58bc --- /dev/null +++ b/cloudinit/transforms/cc_ssh.py @@ -0,0 +1,106 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import cloudinit.SshUtil as sshutil +import os +import glob +import subprocess + +DISABLE_ROOT_OPTS = "no-port-forwarding,no-agent-forwarding," \ +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " \ +"rather than the user \\\"root\\\".\';echo;sleep 10\"" + + +def handle(_name, cfg, cloud, log, _args): + + # remove the static keys from the pristine image + if cfg.get("ssh_deletekeys", True): + for f in glob.glob("/etc/ssh/ssh_host_*key*"): + try: + os.unlink(f) + except: + pass + + if "ssh_keys" in cfg: + # if there are keys in cloud-config, use them + key2file = { + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + } + + for key, val in cfg["ssh_keys"].items(): + if key in key2file: + util.write_file(key2file[key][0], val, key2file[key][1]) + + priv2pub = {'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', + 'ecdsa_private': 'ecdsa_public', } + + cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): + if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: + continue + pair = (key2file[priv][0], key2file[pub][0]) + subprocess.call(('sh', '-xc', cmd % pair)) + log.debug("generated %s from %s" % pair) + else: + # if not, generate them + for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', + ['rsa', 'dsa', 'ecdsa']): + keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + if not os.path.exists(keyfile): + subprocess.call(['ssh-keygen', '-t', keytype, '-N', '', + '-f', keyfile]) + + util.restorecon_if_possible('/etc/ssh', recursive=True) + + try: + user = util.get_cfg_option_str(cfg, 'user') + disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) + disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", + DISABLE_ROOT_OPTS) + keys = cloud.get_public_ssh_keys() + + if "ssh_authorized_keys" in cfg: + cfgkeys = cfg["ssh_authorized_keys"] + keys.extend(cfgkeys) + + apply_credentials(keys, user, disable_root, disable_root_opts, log) + except: + util.logexc(log) + log.warn("applying credentials failed!\n") + + +def apply_credentials(keys, user, disable_root, + disable_root_opts=DISABLE_ROOT_OPTS, log=None): + keys = set(keys) + if user: + sshutil.setup_user_keys(keys, user, '', log) + + if disable_root: + key_prefix = disable_root_opts.replace('$USER', user) + else: + key_prefix = '' + + sshutil.setup_user_keys(keys, 'root', key_prefix, log) diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py new file mode 100644 index 00000000..bbf5bd83 --- /dev/null +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import traceback + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + user = args[0] + ids = [] + if len(args) > 1: + ids = args[1:] + else: + user = util.get_cfg_option_str(cfg, "user", "ubuntu") + ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + + if len(ids) == 0: + return + + cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids + + log.debug("importing ssh ids. cmd = %s" % cmd) + + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd returned %s: %s" % (e.returncode, cmd)) + except OSError as e: + log.debug(traceback.format_exc(e)) + raise Exception("Cmd failed to execute: %s" % (cmd)) diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py new file mode 100644 index 00000000..e5c9901b --- /dev/null +++ b/cloudinit/transforms/cc_timezone.py @@ -0,0 +1,67 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.CloudConfig import per_instance +from cloudinit import util +import os.path +import shutil + +frequency = per_instance +tz_base = "/usr/share/zoneinfo" + + +def handle(_name, cfg, _cloud, log, args): + if len(args) != 0: + timezone = args[0] + else: + timezone = util.get_cfg_option_str(cfg, "timezone", False) + + if not timezone: + return + + tz_file = "%s/%s" % (tz_base, timezone) + + if not os.path.isfile(tz_file): + log.debug("Invalid timezone %s" % tz_file) + raise Exception("Invalid timezone %s" % tz_file) + + try: + fp = open("/etc/timezone", "wb") + fp.write("%s\n" % timezone) + fp.close() + except: + log.debug("failed to write to /etc/timezone") + raise + if os.path.exists("/etc/sysconfig/clock"): + try: + with open("/etc/sysconfig/clock", "w") as fp: + fp.write('ZONE="%s"\n' % timezone) + except: + log.debug("failed to write to /etc/sysconfig/clock") + raise + + try: + shutil.copy(tz_file, "/etc/localtime") + except: + log.debug("failed to copy %s to /etc/localtime" % tz_file) + raise + + log.debug("set timezone to %s" % timezone) + return diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py new file mode 100644 index 00000000..6ad2fca8 --- /dev/null +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -0,0 +1,87 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +from cloudinit.CloudConfig import per_always +import StringIO + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + + manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) + if manage_hosts in ("True", "true", True, "template"): + # render from template file + try: + if not hostname: + log.info("manage_etc_hosts was set, but no hostname found") + return + + util.render_to_file('hosts', '/etc/hosts', + {'hostname': hostname, 'fqdn': fqdn}) + except Exception: + log.warn("failed to update /etc/hosts") + raise + elif manage_hosts == "localhost": + log.debug("managing 127.0.1.1 in /etc/hosts") + update_etc_hosts(hostname, fqdn, log) + return + else: + if manage_hosts not in ("False", False): + log.warn("Unknown value for manage_etc_hosts. Assuming False") + else: + log.debug("not managing /etc/hosts") + + +def update_etc_hosts(hostname, fqdn, _log): + with open('/etc/hosts', 'r') as etchosts: + header = "# Added by cloud-init\n" + hosts_line = "127.0.1.1\t%s %s\n" % (fqdn, hostname) + need_write = False + need_change = True + new_etchosts = StringIO.StringIO() + for line in etchosts: + split_line = [s.strip() for s in line.split()] + if len(split_line) < 2: + new_etchosts.write(line) + continue + if line == header: + continue + ip, hosts = split_line[0], split_line[1:] + if ip == "127.0.1.1": + if sorted([hostname, fqdn]) == sorted(hosts): + need_change = False + if need_change == True: + line = "%s%s" % (header, hosts_line) + need_change = False + need_write = True + new_etchosts.write(line) + etchosts.close() + if need_change == True: + new_etchosts.write("%s%s" % (header, hosts_line)) + need_write = True + if need_write == True: + new_etcfile = open('/etc/hosts', 'wb') + new_etcfile.write(new_etchosts.getvalue()) + new_etcfile.close() + new_etchosts.close() + return diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py new file mode 100644 index 00000000..b9d1919a --- /dev/null +++ b/cloudinit/transforms/cc_update_hostname.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cloudinit.util as util +import subprocess +import errno +from cloudinit.CloudConfig import per_always + +frequency = per_always + + +def handle(_name, cfg, cloud, log, _args): + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): + log.debug("preserve_hostname is set. not updating hostname") + return + + (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) + try: + prev = "%s/%s" % (cloud.get_cpath('data'), "previous-hostname") + update_hostname(hostname, prev, log) + except Exception: + log.warn("failed to set hostname\n") + raise + + +# read hostname from a 'hostname' file +# allow for comments and stripping line endings. +# if file doesn't exist, or no contents, return default +def read_hostname(filename, default=None): + try: + fp = open(filename, "r") + lines = fp.readlines() + fp.close() + for line in lines: + hpos = line.find("#") + if hpos != -1: + line = line[0:hpos] + line = line.rstrip() + if line: + return line + except IOError as e: + if e.errno != errno.ENOENT: + raise + return default + + +def update_hostname(hostname, prev_file, log): + etc_file = "/etc/hostname" + + hostname_prev = None + hostname_in_etc = None + + try: + hostname_prev = read_hostname(prev_file) + except Exception as e: + log.warn("Failed to open %s: %s" % (prev_file, e)) + + try: + hostname_in_etc = read_hostname(etc_file) + except: + log.warn("Failed to open %s" % etc_file) + + update_files = [] + if not hostname_prev or hostname_prev != hostname: + update_files.append(prev_file) + + if (not hostname_in_etc or + (hostname_in_etc == hostname_prev and hostname_in_etc != hostname)): + update_files.append(etc_file) + + try: + for fname in update_files: + util.write_file(fname, "%s\n" % hostname, 0644) + log.debug("wrote %s to %s" % (hostname, fname)) + except: + log.warn("failed to write hostname to %s" % fname) + + if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: + log.debug("%s differs from %s. assuming user maintained" % + (prev_file, etc_file)) + + if etc_file in update_files: + log.debug("setting hostname to %s" % hostname) + subprocess.Popen(['hostname', hostname]).communicate() diff --git a/cloudinit/user_data/boot_hook.py b/cloudinit/user_data/boot_hook.py deleted file mode 100644 index 87e7a3ec..00000000 --- a/cloudinit/user_data/boot_hook.py +++ /dev/null @@ -1,66 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -class BootHookPartHandler(ud.PartHandler): - def __init__(self, boothook_dir, instance_id): - ud.PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir - self.instance_id = instance_id - - def list_types(self): - return [ - ud.type_from_starts_with("#cloud-boothook"), - ] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in ud.CONTENT_SIGNALS: - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 - - filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) - try: - env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) - util.subp([filepath], env=env) - except util.ProcessExecutionError as e: - LOG.error("Boothooks script %s execution error %s", filepath, e) - except Exception as e: - LOG.exception(("Boothooks unknown " - "error %s when running %s"), e, filepath) -- cgit v1.2.3 From 76a7160d4a4eb70e946cd29317ace1fe93ffe5d5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 17:35:38 -0700 Subject: Moved the user_data directory back to a user_data.py and made a handler subdir for just the internal handlers. --- cloudinit/handlers/cloud_config.py | 56 +++++++++++++++++++++++++++++++++++ cloudinit/handlers/shell_script.py | 53 +++++++++++++++++++++++++++++++++ cloudinit/handlers/upstart_job.py | 59 +++++++++++++++++++++++++++++++++++++ cloudinit/user_data/cloud_config.py | 56 ----------------------------------- cloudinit/user_data/shell_script.py | 53 --------------------------------- cloudinit/user_data/upstart_job.py | 59 ------------------------------------- 6 files changed, 168 insertions(+), 168 deletions(-) create mode 100644 cloudinit/handlers/cloud_config.py create mode 100644 cloudinit/handlers/shell_script.py create mode 100644 cloudinit/handlers/upstart_job.py delete mode 100644 cloudinit/user_data/cloud_config.py delete mode 100644 cloudinit/user_data/shell_script.py delete mode 100644 cloudinit/user_data/upstart_job.py (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py new file mode 100644 index 00000000..f0e88eeb --- /dev/null +++ b/cloudinit/handlers/cloud_config.py @@ -0,0 +1,56 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class CloudConfigPartHandler(ud.PartHandler): + def __init__(self, cloud_fn): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.cloud_buf = [] + self.cloud_fn = cloud_fn + + def list_types(self): + return [ + ud.type_from_starts_with("#cloud-config"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype == ud.CONTENT_START: + self.cloud_buf = [] + return + + if ctype == ud.CONTENT_END: + payload = "\n".join(self.cloud_buf) + util.write_file(self.cloud_fn, payload, 0600) + self.cloud_buf = [] + return + + filename = util.clean_filename(filename) + entry = "\n".join(["#%s" % (filename), str(payload)]) + self.cloud_buf.append(entry) diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py new file mode 100644 index 00000000..564e4623 --- /dev/null +++ b/cloudinit/handlers/shell_script.py @@ -0,0 +1,53 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_ALWAYS) + +LOG = logging.getLogger(__name__) + + +class ShellScriptPartHandler(ud.PartHandler): + + def __init__(self, script_dir): + ud.PartHandler.__init__(self, PER_ALWAYS) + self.script_dir = script_dir + + def list_types(self): + return [ + ud.type_from_starts_with("#!"), + ] + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in ud.CONTENT_SIGNALS: + # TODO: maybe delete existing things here + return + + filename = util.clean_filename(filename) + payload = util.dos2unix(payload) + path = os.path.join(self.script_dir, filename) + util.write_file(path, payload, 0700) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py new file mode 100644 index 00000000..568a644a --- /dev/null +++ b/cloudinit/handlers/upstart_job.py @@ -0,0 +1,59 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os + +from cloudinit import log as logging +from cloudinit import user_data as ud +from cloudinit import util + +from cloudinit.settings import (PER_INSTANCE) + +LOG = logging.getLogger(__name__) + + +class UpstartJobPartHandler(ud.PartHandler): + def __init__(self, upstart_dir): + ud.PartHandler.__init__(self, PER_INSTANCE) + self.upstart_dir = upstart_dir + + def list_types(self): + return [ + ud.type_from_starts_with("#upstart-job"), + ] + + def _handle_part(self, _data, ctype, filename, payload, frequency): + if ctype in ud.CONTENT_SIGNALS: + return + + filename = util.clean_filename(filename) + (_name, ext) = os.path.splitext(filename) + if not ext: + ext = '' + ext = ext.lower() + if ext != ".conf": + filename = filename + ".conf" + + payload = util.dos2unix(payload) + path = os.path.join(self.upstart_dir, filename) + util.write_file(path, payload, 0644) diff --git a/cloudinit/user_data/cloud_config.py b/cloudinit/user_data/cloud_config.py deleted file mode 100644 index f0e88eeb..00000000 --- a/cloudinit/user_data/cloud_config.py +++ /dev/null @@ -1,56 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -class CloudConfigPartHandler(ud.PartHandler): - def __init__(self, cloud_fn): - ud.PartHandler.__init__(self, PER_ALWAYS) - self.cloud_buf = [] - self.cloud_fn = cloud_fn - - def list_types(self): - return [ - ud.type_from_starts_with("#cloud-config"), - ] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype == ud.CONTENT_START: - self.cloud_buf = [] - return - - if ctype == ud.CONTENT_END: - payload = "\n".join(self.cloud_buf) - util.write_file(self.cloud_fn, payload, 0600) - self.cloud_buf = [] - return - - filename = util.clean_filename(filename) - entry = "\n".join(["#%s" % (filename), str(payload)]) - self.cloud_buf.append(entry) diff --git a/cloudinit/user_data/shell_script.py b/cloudinit/user_data/shell_script.py deleted file mode 100644 index 564e4623..00000000 --- a/cloudinit/user_data/shell_script.py +++ /dev/null @@ -1,53 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_ALWAYS) - -LOG = logging.getLogger(__name__) - - -class ShellScriptPartHandler(ud.PartHandler): - - def __init__(self, script_dir): - ud.PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir - - def list_types(self): - return [ - ud.type_from_starts_with("#!"), - ] - - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in ud.CONTENT_SIGNALS: - # TODO: maybe delete existing things here - return - - filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - path = os.path.join(self.script_dir, filename) - util.write_file(path, payload, 0700) diff --git a/cloudinit/user_data/upstart_job.py b/cloudinit/user_data/upstart_job.py deleted file mode 100644 index 568a644a..00000000 --- a/cloudinit/user_data/upstart_job.py +++ /dev/null @@ -1,59 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import os - -from cloudinit import log as logging -from cloudinit import user_data as ud -from cloudinit import util - -from cloudinit.settings import (PER_INSTANCE) - -LOG = logging.getLogger(__name__) - - -class UpstartJobPartHandler(ud.PartHandler): - def __init__(self, upstart_dir): - ud.PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir - - def list_types(self): - return [ - ud.type_from_starts_with("#upstart-job"), - ] - - def _handle_part(self, _data, ctype, filename, payload, frequency): - if ctype in ud.CONTENT_SIGNALS: - return - - filename = util.clean_filename(filename) - (_name, ext) = os.path.splitext(filename) - if not ext: - ext = '' - ext = ext.lower() - if ext != ".conf": - filename = filename + ".conf" - - payload = util.dos2unix(payload) - path = os.path.join(self.upstart_dir, filename) - util.write_file(path, payload, 0644) -- cgit v1.2.3 From 784edb7689d60b81a4334d5171603841cc83da98 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 19:03:28 -0700 Subject: Ensure that this directory is treated as a module. --- cloudinit/handlers/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 cloudinit/handlers/__init__.py (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py new file mode 100644 index 00000000..09163abb --- /dev/null +++ b/cloudinit/handlers/__init__.py @@ -0,0 +1,22 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + -- cgit v1.2.3 From 450261a1fcf1f8929a2f7a25c2c278ba40689289 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 21:33:55 -0700 Subject: Fixups to ensure that pylint does not find anything major wrong. --- cloudinit/cloud.py | 2 - cloudinit/distros/__init__.py | 21 +++--- cloudinit/distros/ubuntu.py | 23 +++--- cloudinit/handlers/boot_hook.py | 14 ++-- cloudinit/handlers/cloud_config.py | 4 +- cloudinit/handlers/shell_script.py | 5 +- cloudinit/handlers/upstart_job.py | 7 +- cloudinit/helpers.py | 41 ++--------- cloudinit/log.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 4 +- cloudinit/sources/DataSourceMAAS.py | 14 ++-- cloudinit/sources/DataSourceNoCloud.py | 3 +- cloudinit/sources/DataSourceOVF.py | 12 +-- cloudinit/sources/__init__.py | 23 +++--- cloudinit/stages.py | 98 ++++++++++++++++--------- cloudinit/transforms/__init__.py | 2 +- cloudinit/transforms/cc_apt_update_upgrade.py | 12 ++- cloudinit/transforms/cc_bootcmd.py | 8 +- cloudinit/transforms/cc_byobu.py | 2 +- cloudinit/transforms/cc_ca_certs.py | 13 +++- cloudinit/transforms/cc_chef.py | 3 +- cloudinit/transforms/cc_disable_ec2_metadata.py | 3 +- cloudinit/transforms/cc_final_message.py | 4 +- cloudinit/transforms/cc_foo.py | 6 +- cloudinit/transforms/cc_grub_dpkg.py | 4 +- cloudinit/transforms/cc_keys_to_console.py | 23 ++++-- cloudinit/transforms/cc_landscape.py | 3 +- cloudinit/transforms/cc_locale.py | 2 +- cloudinit/transforms/cc_mcollective.py | 9 +-- cloudinit/transforms/cc_mounts.py | 15 ++-- cloudinit/transforms/cc_phone_home.py | 36 ++++++--- cloudinit/transforms/cc_puppet.py | 17 +++-- cloudinit/transforms/cc_resizefs.py | 38 ++++++---- cloudinit/transforms/cc_rightscale_userdata.py | 16 ++-- cloudinit/transforms/cc_rsyslog.py | 14 ++-- cloudinit/transforms/cc_runcmd.py | 9 ++- cloudinit/transforms/cc_salt_minion.py | 7 +- cloudinit/transforms/cc_scripts_per_boot.py | 5 +- cloudinit/transforms/cc_scripts_per_instance.py | 5 +- cloudinit/transforms/cc_scripts_per_once.py | 5 +- cloudinit/transforms/cc_scripts_user.py | 9 ++- cloudinit/transforms/cc_set_hostname.py | 2 +- cloudinit/transforms/cc_set_passwords.py | 10 ++- cloudinit/transforms/cc_ssh.py | 22 +++--- cloudinit/transforms/cc_ssh_import_id.py | 8 +- cloudinit/transforms/cc_timezone.py | 8 +- cloudinit/transforms/cc_update_etc_hosts.py | 22 ++++-- cloudinit/transforms/cc_update_hostname.py | 4 +- cloudinit/transforms/cc_welcome.py | 6 +- cloudinit/user_data.py | 13 ++-- cloudinit/util.py | 48 ++++++------ 52 files changed, 388 insertions(+), 300 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index b2dfc749..8372d123 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -23,8 +23,6 @@ import copy import os -from cloudinit import distros -from cloudinit import helpers from cloudinit import log as logging LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 90607668..fd4c70c1 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -20,29 +20,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + import abc import copy from cloudinit import importer +from cloudinit import log as logging from cloudinit import util -from StringIO import StringIO - # TODO: Make this via config?? IFACE_ACTIONS = { 'up': ['ifup', '--all'], 'down': ['ifdown', '--all'], } +LOG = logging.getLogger(__name__) + class Distro(object): __metaclass__ = abc.ABCMeta - def __init__(self, cfg, runner): + def __init__(self, name, cfg, runner): self._runner = runner - self._cfg = util.get_cfg_by_path(cfg, ('system_info', ), {}) - self.name = self._cfg.pop("distro", 'generic') + self._cfg = cfg + self.name = name @abc.abstractmethod def install_packages(self, pkglist): @@ -135,10 +138,9 @@ class Distro(object): action, cmd) (_out, err) = util.subp(cmd) if len(err): - LOG.warn("Running %s resulted in stderr output: %s", - IF_UP_CMD, err) + LOG.warn("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError as exc: + except util.ProcessExecutionError: util.logexc(LOG, "Running %s failed", cmd) return False @@ -152,7 +154,8 @@ def fetch(distro_name, mods=(__name__, )): except RuntimeError: pass if not mod: - raise RuntimeError("No distribution found for distro %s" % (distro_name)) + raise RuntimeError("No distribution found for distro %s" + % (distro_name)) distro_cls = getattr(mod, 'Distro') return distro_cls diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 6b0aff47..9252a1c4 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -20,17 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO - import os -import socket from cloudinit import distros from cloudinit import log as logging -from cloudinit import templater from cloudinit import util -from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -65,9 +61,11 @@ class Distro(distros.Distro): try: util.write_file(fn, "%s\n" % hostname, 0644) except: - util.logexc(LOG, "Failed to write hostname %s to %s", hostname, fn) - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - LOG.debug(("%s differs from /etc/hostname." + util.logexc(LOG, "Failed to write hostname %s to %s", + hostname, fn) + if (hostname_in_etc and hostname_prev and + hostname_in_etc != hostname_prev): + LOG.debug(("%s differs from /etc/hostname." " Assuming user maintained hostname."), prev_file) if "/etc/hostname" in update_files: LOG.debug("Setting hostname to %s", hostname) @@ -91,7 +89,8 @@ class Distro(distros.Distro): def set_timezone(self, tz): tz_file = os.path.join("/usr/share/zoneinfo", tz) if not os.path.isfile(tz_file): - raise Exception("Invalid timezone %s, no file found at %s" % (tz, tz_file)) + raise Exception(("Invalid timezone %s," + " no file found at %s") % (tz, tz_file)) tz_contents = "%s\n" % tz util.write_file("/etc/timezone", tz_contents) # TODO, this should be in a rhel distro subclass?? @@ -101,9 +100,6 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, "/etc/localtime") - def name(self): - return "ubuntu" - # apt_get top level command (install, update...), and args to pass it def _apt_get(self, tlc, args=None): e = os.environ.copy() @@ -116,4 +112,5 @@ class Distro(distros.Distro): util.subp(cmd, env=e, capture=False) def _update_package_sources(self): - self.runner.run("update-sources", self._apt_get, ["update"], freq=PER_INSTANCE) \ No newline at end of file + self._runner.run("update-sources", self._apt_get, + ["update"], freq=PER_INSTANCE) \ No newline at end of file diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index c75aeb72..b3aab366 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -32,9 +32,9 @@ LOG = logging.getLogger(__name__) class BootHookPartHandler(ud.PartHandler): - def __init__(self, boothook_dir, instance_id): + def __init__(self, paths, instance_id, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir + self.boothook_dir = paths.get_ipath("boothooks") self.instance_id = instance_id def list_types(self): @@ -54,13 +54,15 @@ class BootHookPartHandler(ud.PartHandler): start = len(prefix) + 1 filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) + contents = payload[start:] + util.write_file(filepath, contents, 0700) try: env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) + if self.instance_id: + env['INSTANCE_ID'] = str(self.instance_id) util.subp([filepath], env=env) - except util.ProcessExecutionError as e: + except util.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) - except Exception as e: + except Exception: util.logexc(LOG, ("Boothooks unknown " "error when running %s"), filepath) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index f0e88eeb..12d1bd96 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -30,10 +30,10 @@ LOG = logging.getLogger(__name__) class CloudConfigPartHandler(ud.PartHandler): - def __init__(self, cloud_fn): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) self.cloud_buf = [] - self.cloud_fn = cloud_fn + self.cloud_fn = paths.get_ipath("cloud_config") def list_types(self): return [ diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 564e4623..f6e2ef16 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -32,10 +32,9 @@ LOG = logging.getLogger(__name__) class ShellScriptPartHandler(ud.PartHandler): - - def __init__(self, script_dir): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir + self.script_dir = paths.get_ipath_cur('scripts') def list_types(self): return [ diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 568a644a..059a4851 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -33,9 +33,9 @@ LOG = logging.getLogger(__name__) class UpstartJobPartHandler(ud.PartHandler): - def __init__(self, upstart_dir): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir + self.upstart_dir = paths.upstart_conf_d def list_types(self): return [ @@ -46,6 +46,9 @@ class UpstartJobPartHandler(ud.PartHandler): if ctype in ud.CONTENT_SIGNALS: return + if not self.upstart_dir: + return + filename = util.clean_filename(filename) (_name, ext) = os.path.splitext(filename) if not ext: diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 2ecda3e9..c276a54c 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -30,11 +30,6 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) from cloudinit import log as logging from cloudinit import util -from cloudinit.user_data import boot_hook as bh_part -from cloudinit.user_data import cloud_config as cc_part -from cloudinit.user_data import shell_script as ss_part -from cloudinit.user_data import upstart_job as up_part - LOG = logging.getLogger(__name__) @@ -77,7 +72,7 @@ class FileSemaphores(object): sem_file = self._get_path(name, freq) try: util.del_file(sem_file) - except (IOError, OSError) as e: + except (IOError, OSError): util.logexc(LOG, "Failed deleting semaphore %s", sem_file) return False return True @@ -99,7 +94,7 @@ class FileSemaphores(object): contents = "%s: %s\n" % (os.getpid(), time()) try: util.write_file(sem_file, contents) - except (IOError, OSError) as e: + except (IOError, OSError): util.logexc(LOG, "Failed writing semaphore file %s", sem_file) return None return sem_file @@ -162,9 +157,10 @@ class Runners(object): class ContentHandlers(object): - def __init__(self, paths): + def __init__(self, paths, iid=None): self.paths = paths self.registered = {} + self.iid = iid def __contains__(self, item): return self.is_registered(item) @@ -191,34 +187,9 @@ class ContentHandlers(object): def iteritems(self): return self.registered.iteritems() - def _get_default_handlers(self): - def_handlers = [] - - cc_path = self.paths.get_ipath("cloud_config") - if cc_path: - cc_h = cc_part.CloudConfigPartHandler(cc_path) - def_handlers.append(cc_h) - - sc_path = self.paths.get_ipath_cur('scripts') - if sc_path: - ss_h = ss_part.ShellScriptPartHandler(sc_path) - def_handlers.append(ss_h) - - bh_path = self.paths.get_ipath("boothooks") - if bh_path: - bh_h = bh_part.BootHookPartHandler(bh_path) - def_handlers.append(bh_h) - - upstart_pth = self.paths.upstart_conf_d - if upstart_pth: - up_h = up_part.UpstartJobPartHandler(upstart_pth) - def_handlers.append(up_h) - - return def_handlers - - def register_defaults(self): + def register_defaults(self, defs): registered = set() - for mod in self._get_default_handlers(): + for mod in defs: for t in mod.list_types(): if not self.is_registered(t): self.registered[t] = mod diff --git a/cloudinit/log.py b/cloudinit/log.py index c247eb9e..5fcb77ef 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -56,7 +56,7 @@ def setupBasicLogging(): cfile = logging.FileHandler('/var/log/cloud-init.log') cfile.setFormatter(logging.Formatter(DEF_CON_FORMAT)) cfile.setLevel(DEBUG) - root.addHandle(cfile) + root.addHandler(cfile) except (IOError, OSError): # Likely that u can't write to that file... # Make console now have DEBUG?? diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 791df68f..27217e65 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -79,7 +79,7 @@ class DataSourceCloudStack(sources.DataSource): tot_time = (time.time() - start) LOG.debug("Crawl of metadata service took %s", int(tot_time)) return True - except Exception as e: + except Exception: util.logexc(LOG, ('Failed fetching from metadata ' 'service %s'), self.metadata_address) return False diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 176b62b0..7450572f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -51,8 +51,8 @@ class DataSourceConfigDrive(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') def __str__(self): - mstr = "%s[%s]" % (util.obj_name(self), self.dsmode) - mstr += " [seed=%s]" % (self.seed) + mstr = "%s [%s]" % (util.obj_name(self), self.dsmode) + mstr += "[seed=%s]" % (self.seed) return mstr def get_data(self): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 27196265..9e639649 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,9 +18,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os import errno import oauth.oauth as oauth +import os import time import urllib2 @@ -48,7 +48,7 @@ class DataSourceMAAS(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'maas') def __str__(self): - return "%s[%s]" % (util.obj_name(self), self.base_url) + return "%s [%s]" % (util.obj_name(self), self.base_url) def get_data(self): mcfg = self.ds_cfg @@ -122,9 +122,10 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn, - headers_cb=self.md_headers) + urls = [check_url] + url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn, + headers_cb=self.md_headers) if url: LOG.info("Using metadata source: '%s'", url) @@ -185,7 +186,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, headers = {} try: (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout) - md[name] = resp + if uhelp.ok_http_code(sc): + md[name] = resp except urllib2.HTTPError as e: if e.code != 404: raise diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 84d0f99d..2b016d1c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -106,7 +106,8 @@ class DataSourceNoCloud(sources.DataSource): if e.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, "Failed to mount %s when looking for seed", dev) + util.logexc(LOG, ("Failed to mount %s" + " when looking for data"), dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index bb0f46c2..258d8d03 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -21,10 +21,10 @@ # along with this program. If not, see . from xml.dom import minidom + import base64 import os import re -import tempfile from cloudinit import log as logging from cloudinit import sources @@ -51,7 +51,7 @@ class DataSourceOVF(sources.DataSource): ud = "" defaults = { - "instance-id": "iid-dsovf" + "instance-id": "iid-dsovf", } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) @@ -198,7 +198,7 @@ def transport_iso9660(require_iso=True): for dev in devs: fullp = os.path.join("/dev/", dev) - if (fullp in mounted or + if (fullp in mounts or not cdmatch.match(dev) or os.path.isdir(fullp)): continue @@ -210,7 +210,8 @@ def transport_iso9660(require_iso=True): continue try: - (fname, contents) = utils.mount_cb(fullp, get_ovf_env, mtype="iso9660") + (fname, contents) = util.mount_cb(fullp, + get_ovf_env, mtype="iso9660") except util.MountFailedError: util.logexc(LOG, "Failed mounting %s", fullp) continue @@ -265,7 +266,8 @@ def get_properties(contents): raise XmlError("No 'PropertySection's") props = {} - propElems = find_child(propSections[0], lambda n: n.localName == "Property") + propElems = find_child(propSections[0], + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 08669f5d..beb0f3d7 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -22,10 +22,9 @@ from cloudinit import importer from cloudinit import log as logging +from cloudinit import user_data as ud from cloudinit import util -from cloudinit.user_data import processor as ud_proc - DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -42,7 +41,6 @@ class DataSource(object): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths - self.userdata_proc = ud_proc.UserDataProcessor(paths) self.userdata = None self.metadata = None self.userdata_raw = None @@ -55,7 +53,7 @@ class DataSource(object): def get_userdata(self): if self.userdata is None: raw_data = self.get_userdata_raw() - self.userdata = self.userdata_proc.process(raw_data) + self.userdata = ud.UserDataProcessor(self.paths).process(raw_data) return self.userdata def get_userdata_raw(self): @@ -73,7 +71,7 @@ class DataSource(object): if not self.metadata or 'public-keys' not in self.metadata: return keys - if isinstance(self.metadata['public-keys'], (str)): + if isinstance(self.metadata['public-keys'], (basestring, str)): return str(self.metadata['public-keys']).splitlines() if isinstance(self.metadata['public-keys'], (list, set)): @@ -84,11 +82,12 @@ class DataSource(object): # lp:506332 uec metadata service responds with # data that makes boto populate a string for 'klist' rather # than a list. - if isinstance(klist, (str)): + if isinstance(klist, (str, basestring)): klist = [klist] if isinstance(klist, (list, set)): for pkey in klist: - # there is an empty string at the end of the keylist, trim it + # There is an empty string at + # the end of the keylist, trim it if pkey: keys.append(pkey) @@ -159,13 +158,14 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [util.obj_name(f) for f in ds_list] LOG.info("Searching for data source in: %s", ds_names) + for cls in ds_list: ds = util.obj_name(cls) try: s = cls(distro, sys_cfg, paths) if s.get_data(): return (s, ds) - except Exception as e: + except Exception: util.logexc(LOG, "Getting data from %s failed", ds) msg = "Did not find any data source, searched classes: %s" % (ds_names) @@ -178,7 +178,8 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): # return an ordered list of classes that match def list_sources(cfg_list, depends, pkg_list): src_list = [] - LOG.info("Looking for for data source in: %s, %s that match %s", cfg_list, pkg_list, depends) + LOG.info(("Looking for for data source in: %s," + " %s that matches %s"), cfg_list, pkg_list, depends) for ds_coll in cfg_list: ds_name = str(ds_coll) if not ds_name.startswith(DS_PREFIX): @@ -201,8 +202,8 @@ def list_sources(cfg_list, depends, pkg_list): if not cls_matches: continue src_list.extend(cls_matches) - LOG.debug("Found a match for data source %s in %s with matches %s", - ds_name, mod, cls_matches) + LOG.debug(("Found a match for data source %s" + " in %s with matches %s"), ds_name, mod, cls_matches) break return src_list diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2615d59f..b9076881 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -31,19 +31,23 @@ try: except ImportError: ConfigObj = None -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit.settings import (OLD_CLOUD_CONFIG) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) + +from cloudinit.handlers import boot_hook as bh_part +from cloudinit.handlers import cloud_config as cc_part +from cloudinit.handlers import shell_script as ss_part +from cloudinit.handlers import upstart_job as up_part from cloudinit import cloud from cloudinit import distros -from cloudinit import modules from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import sources -from cloudinit import util - +from cloudinit import transforms from cloudinit import user_data as ud +from cloudinit import util LOG = logging.getLogger(__name__) @@ -73,12 +77,19 @@ class Init(object): def distro(self): if not self._distro: d_cfg = util.get_cfg_by_path(self.cfg, ('system_info'), {}) + # Ensure its a dictionary + if not isinstance(d_cfg, (dict)): + d_cfg = {} # Ensure not modified indirectly d_cfg = copy.deepcopy(d_cfg) + # Remove this since its path config, not distro config d_cfg.pop('paths', None) - distro_cls = distros.fetch(sys_cfg.pop('distro', 'ubuntu')) + # Try to find the right class to use + distro_name = d_cfg.pop('distro', 'ubuntu') + distro_cls = distros.fetch(distro_name) LOG.debug("Using distro class %s", distro_cls) - distro = distro_cls(d_cfg, helpers.Runners(self.paths)) + distro = distro_cls(distro_name, d_cfg, + helpers.Runners(self.paths)) self._distro = distro return self._distro @@ -93,7 +104,8 @@ class Init(object): @property def paths(self): if not self._paths: - path_info = util.get_cfg_by_path(self.cfg, ('system_info', 'paths'), {}) + path_info = util.get_cfg_by_path(self.cfg, + ('system_info', 'paths'), {}) # Ensure not modified indirectly path_info = copy.deepcopy(path_info) self._paths = helpers.Paths(path_info, self.datasource) @@ -156,7 +168,7 @@ class Init(object): # by using the instance link, if purge_cache was called # the file wont exist return pickle.loads(util.load_file(pickled_fn)) - except Exception as e: + except Exception: util.logexc(LOG, "Failed loading pickled datasource from %s", pickled_fn) return None @@ -166,7 +178,7 @@ class Init(object): try: contents = pickle.dumps(self.datasource) util.write_file(pickled_fn, contents, mode=0400) - except Exception as e: + except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False @@ -192,7 +204,8 @@ class Init(object): # (which will affect user-data handlers down the line...) sys_cfg = copy.deepcopy(self.cfg) ds_deps = copy.deepcopy(self.ds_deps) - (ds, dsname) = sources.find_source(sys_cfg, self.distro, self.paths, + (ds, dsname) = sources.find_source(sys_cfg, self.distro, + self.paths, ds_deps, cfg_list, pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) self.datasource = ds @@ -270,6 +283,20 @@ class Init(object): processed_ud = "%s" % (self.datasource.get_userdata()) util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600) + def _default_userdata_handlers(self): + opts = { + 'paths': self.paths, + 'instance_id': self.datasource.get_instance_id(), + } + # TODO Hmmm, should we dynamically import these?? + def_handlers = [ + cc_part.CloudConfigPartHandler(**opts), + ss_part.ShellScriptPartHandler(**opts), + bh_part.BootHookPartHandler(**opts), + up_part.UpstartJobPartHandler(**opts), + ] + return def_handlers + def consume(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") idir = self.paths.get_ipath("handlers") @@ -279,8 +306,11 @@ class Init(object): sys.path.insert(0, cdir) sys.path.insert(0, idir) + # Ensure datasource fetched before activation (just incase) + ud_obj = self.datasource.get_userdata() + # This keeps track of all the active handlers - c_handlers = helpers.ContentHandlers(self.paths) + c_handlers = helpers.ContentHandlers(paths=self.paths) # Add handlers in cdir potential_handlers = util.find_modules(cdir) @@ -292,13 +322,10 @@ class Init(object): except: util.logexc(LOG, "Failed to register handler from %s", fname) - def_handlers = c_handlers.register_defaults() - if def_handlers: - LOG.debug("Registered default handlers for %s", def_handlers) - - - # Ensure userdata fetched before activation (just incase) - ud_obj = self.datasource.get_userdata() + def_handlers = self._default_userdata_handlers() + applied_def_handlers = c_handlers.register_defaults(def_handlers) + if applied_def_handlers: + LOG.debug("Registered default handlers: %s", applied_def_handlers) # Form our cloud interface data = self.cloudify() @@ -334,11 +361,11 @@ class Init(object): class Transforms(object): - def __init__(self, cloudobj, cfgfile=None): - self.datasource = cloudobj.datasource + def __init__(self, init, cfgfile=None): + self.datasource = init.fetch() self.cfgfile = cfgfile - self.basecfg = copy.deepcopy(cloudobj.cfg) - self.cloudobj = cloudobj + self.basecfg = copy.deepcopy(init.cfg) + self.init = init # Created on first use self._cachedcfg = None @@ -409,25 +436,28 @@ class Transforms(object): (item, util.obj_name(item))) return module_list - def _transforms_modules(self, raw_mods): + def _fixup_transforms(self, raw_mods): mostly_mods = [] for raw_mod in raw_mods: raw_name = raw_mod['mod'] freq = raw_mod.get('freq') run_args = raw_mod.get('args') or [] - mod_name = modules.form_module_name(raw_name) + mod_name = transforms.form_module_name(raw_name) if not mod_name: continue if freq and freq not in FREQUENCIES: - LOG.warn("Config specified module %s has an unknown frequency %s", raw_name, freq) + LOG.warn(("Config specified transform %s" + " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = modules.fixup_module(importer.import_module(mod_name)) + mod = transforms.fixup_module(importer.import_module(mod_name)) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods def _run_transforms(self, mostly_mods): failures = [] + d_name = self.init.distro.name + c_cloud = self.init.cloudify() for (mod, name, freq, args) in mostly_mods: try: # Try the modules frequency, otherwise fallback to a known one @@ -436,17 +466,17 @@ class Transforms(object): if not freq in FREQUENCIES: freq = PER_INSTANCE worked_distros = mod.distros - if worked_distros and self.cloud.distro.name() not in worked_distros: - LOG.warn(("Module %s is verified on %s distros" + if (worked_distros and d_name not in worked_distros): + LOG.warn(("Transform %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, - self.cloud.distro.name()) + " correctly."), name, worked_distros, d_name) # Deep copy the config so that modules can't alter it + # Use the transforms logger and not our own func_args = [name, copy.deepcopy(self.cfg), - self.cloudobj, LOG, args] - # This name will affect the semphapore name created + c_cloud, transforms.LOG, args] + # This name will affect the semaphore name created run_name = "config-%s" % (name) - self.cloudobj.run(run_name, mod.handle, func_args, freq=freq) + c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s failed", mod) failures.append((name, e)) @@ -454,5 +484,5 @@ class Transforms(object): def run(self, name): raw_mods = self._read_transforms(name) - mostly_mods = self._transforms_modules(raw_mods) + mostly_mods = self._fixup_transforms(raw_mods) return self._run_transforms(mostly_mods) diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py index 8275b375..40affc4b 100644 --- a/cloudinit/transforms/__init__.py +++ b/cloudinit/transforms/__init__.py @@ -44,7 +44,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): else: freq = mod.frequency if freq and freq not in FREQUENCIES: - LOG.warn("Module %s has an unknown frequency %s", mod, freq) + LOG.warn("Transform %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'handle'): def empty_handle(_name, _cfg, _cloud, _log, _args): pass diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py index c4a543ed..a4e058c6 100644 --- a/cloudinit/transforms/cc_apt_update_upgrade.py +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -71,7 +71,7 @@ def handle(_name, cfg, cloud, log, _args): except: util.logexc(log, "Failed to run debconf-set-selections") - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + pkglist = util.get_cfg_option_list(cfg, 'packages', []) errors = [] if update or len(pkglist) or upgrade: @@ -96,7 +96,9 @@ def handle(_name, cfg, cloud, log, _args): errors.append(e) if len(errors): - raise errors[0] + log.warn("%s failed with exceptions, re-raising the last one", + len(errors)) + raise errors[-1] def mirror2lists_fileprefix(mirror): @@ -186,7 +188,8 @@ def add_sources(srclist, template_params=None): try: util.write_file(ent['filename'], source + "\n", omode="ab") except: - errorlist.append([source, "failed write to file %s" % ent['filename']]) + errorlist.append([source, + "failed write to file %s" % ent['filename']]) return errorlist @@ -219,9 +222,10 @@ def find_apt_mirror(cloud, cfg): doms.extend((".localdomain", "",)) mirror_list = [] + distro = cloud.distro.name mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) for post in doms: - mirror_list.append(mirrorfmt % post) + mirror_list.append(mirrorfmt % (post)) mirror = util.search_for_mirror(mirror_list) diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py index a2efad32..80afb5e7 100644 --- a/cloudinit/transforms/cc_bootcmd.py +++ b/cloudinit/transforms/cc_bootcmd.py @@ -30,7 +30,8 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if "bootcmd" not in cfg: - log.debug("Skipping module named %s, no 'bootcomd' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'bootcomd' key in configuration"), name) return with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf: @@ -39,7 +40,7 @@ def handle(name, cfg, cloud, log, _args): tmpf.write(content) tmpf.flush() except: - log.warn("Failed to shellify bootcmd") + util.logexc(log, "Failed to shellify bootcmd") raise try: @@ -48,5 +49,6 @@ def handle(name, cfg, cloud, log, _args): cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) except: - log.warn("Failed to run commands from bootcmd") + util.logexc(log, + ("Failed to run bootcmd transform %s"), name) raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py index 38586174..741aa934 100644 --- a/cloudinit/transforms/cc_byobu.py +++ b/cloudinit/transforms/cc_byobu.py @@ -30,7 +30,7 @@ def handle(name, cfg, _cloud, log, args): value = util.get_cfg_option_str(cfg, "byobu_by_default", "") if not value: - log.debug("Skipping module named %s, no 'byobu' values found", name) + log.debug("Skipping transform named %s, no 'byobu' values found", name) return if value == "user" or value == "system": diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py index 8ca9a200..e0802bfe 100644 --- a/cloudinit/transforms/cc_ca_certs.py +++ b/cloudinit/transforms/cc_ca_certs.py @@ -23,6 +23,8 @@ CA_CERT_FILENAME = "cloud-init-ca-certs.crt" CA_CERT_CONFIG = "/etc/ca-certificates.conf" CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" +distros = ['ubuntu'] + def update_ca_certs(): """ @@ -70,22 +72,25 @@ def handle(name, cfg, _cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: - log.debug("Skipping module named %s, no 'ca-certs' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'ca-certs' key in configuration"), name) return + ca_cert_cfg = cfg['ca-certs'] # If there is a remove-defaults option set to true, remove the system # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") + log.debug("Removing default certificates") remove_default_ca_certs() # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: - trusted_certs = util.get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) + log.debug("Adding %d certificates" % len(trusted_certs)) add_ca_certs(trusted_certs) # Update the system with the new cert configuration. + log.debug("Updating certificates") update_ca_certs() diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py index 12c2f539..473e5f8b 100644 --- a/cloudinit/transforms/cc_chef.py +++ b/cloudinit/transforms/cc_chef.py @@ -31,7 +31,8 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: - log.debug("Skipping module named %s, no 'chef' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'chef' key in configuration"), name) return chef_cfg = cfg['chef'] diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py index 4d2a7f55..3c0dd57b 100644 --- a/cloudinit/transforms/cc_disable_ec2_metadata.py +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -28,5 +28,6 @@ reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) + if disabled: util.subp(reject_cmd) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py index dc4ae34c..c257b6d0 100644 --- a/cloudinit/transforms/cc_final_message.py +++ b/cloudinit/transforms/cc_final_message.py @@ -32,7 +32,7 @@ final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." " Up {{uptime}} seconds.") -def handle(name, cfg, cloud, log, args): +def handle(_name, cfg, cloud, log, args): msg_in = None if len(args) != 0: @@ -60,7 +60,7 @@ def handle(name, cfg, cloud, log, args): # Use stdout, stderr or the logger?? content = templater.render_string(msg_in, subs) sys.stderr.write("%s\n" % (content)) - except Exception as e: + except Exception: util.logexc(log, "Failed to render final message template") boot_fin_fn = cloud.paths.boot_finished diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py index 8007f981..99135704 100644 --- a/cloudinit/transforms/cc_foo.py +++ b/cloudinit/transforms/cc_foo.py @@ -45,8 +45,8 @@ from cloudinit.settings import PER_INSTANCE # informational purposes. If non existent all distros are assumed and # no warning occurs. -frequency = settings.PER_INSTANCE +frequency = PER_INSTANCE -def handle(name, _cfg, _cloud, _log, _args): - print("Hi from %s" % (name)) +def handle(name, _cfg, _cloud, log, _args): + log.debug("Hi from transform %s", name) diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py index c048d5cc..02f05ce3 100644 --- a/cloudinit/transforms/cc_grub_dpkg.py +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -54,9 +54,9 @@ def handle(_name, cfg, _cloud, log, _args): # now idevs and idevs_empty are set to determined values # or, those set by user - dconf_sel = ("grub-pc grub-pc/install_devices string %s\n" + dconf_sel = (("grub-pc grub-pc/install_devices string %s\n" "grub-pc grub-pc/install_devices_empty boolean %s\n") % - (idevs, idevs_empty) + (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % (idevs, idevs_empty)) diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py index 2f2a5297..e974375f 100644 --- a/cloudinit/transforms/cc_keys_to_console.py +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -18,23 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE +# This is a tool that cloud init provides +helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints' + -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, +def handle(name, cfg, _cloud, log, _args): + if not os.path.exists(helper_tool): + log.warn(("Unable to activate transform %s," + " helper tool not found at %s"), name, helper_tool) + return + + fp_blacklist = util.get_cfg_option_list(cfg, "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, + key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", ["ssh-dss"]) + try: + cmd = [helper_tool] cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) - (stdout, stderr) = util.subp(cmd) + (stdout, _stderr) = util.subp(cmd) util.write_file('/dev/console', stdout) except: - log.warn("Writing keys to console failed!") + log.warn("Writing keys to /dev/console failed!") raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py index 48491992..19948d0e 100644 --- a/cloudinit/transforms/cc_landscape.py +++ b/cloudinit/transforms/cc_landscape.py @@ -55,7 +55,8 @@ def handle(name, cfg, _cloud, log, _args): /etc/landscape/client.conf """ if not ConfigObj: - log.warn("'ConfigObj' support not enabled, running %s disabled", name) + log.warn(("'ConfigObj' support not available," + " running transform %s disabled"), name) return ls_cloudcfg = cfg.get("landscape", {}) diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py index 3fb4c5d9..7f273123 100644 --- a/cloudinit/transforms/cc_locale.py +++ b/cloudinit/transforms/cc_locale.py @@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args): "/etc/default/locale") if not locale: - log.debug(("Skipping module named %s, " + log.debug(("Skipping transform named %s, " "no 'locale' configuration found"), name) return diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py index aeeda9d2..5464fe8c 100644 --- a/cloudinit/transforms/cc_mcollective.py +++ b/cloudinit/transforms/cc_mcollective.py @@ -19,13 +19,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from ConfigParser import ConfigParser from StringIO import StringIO -import os - +from cloudinit import cfg as config from cloudinit import util -from cloudinit import cfg pubcert_file = "/etc/mcollective/ssl/server-public.pem" pricert_file = "/etc/mcollective/ssl/server-private.pem" @@ -35,7 +32,7 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: - log.debug(("Skipping module named %s, " + log.debug(("Skipping transform named %s, " "no 'mcollective' key in configuration"), name) return @@ -47,7 +44,7 @@ def handle(name, cfg, cloud, log, _args): # ... and then update the mcollective configuration if 'conf' in mcollective_cfg: # Create object for reading server.cfg values - mcollective_config = cfg.DefaultingConfigParser() + mcollective_config = config.DefaultingConfigParser() # Read server.cfg values from original file in order to be able to mix # the rest up old_contents = util.load_file('/etc/mcollective/server.cfg') diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py index babcbda1..44182b87 100644 --- a/cloudinit/transforms/cc_mounts.py +++ b/cloudinit/transforms/cc_mounts.py @@ -20,7 +20,6 @@ from string import whitespace # pylint: disable=W0402 -import os import re from cloudinit import util @@ -28,7 +27,7 @@ from cloudinit import util # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" shortname = re.compile(shortname_filter) -ws = re.compile("[%s]+" % whitespace) +ws = re.compile("[%s]+" % (whitespace)) def is_mdname(name): @@ -65,13 +64,14 @@ def handle(_name, cfg, cloud, log, _args): continue startname = str(cfgmnt[i][0]) - LOG.debug("Attempting to determine the real name of %s", startname) + log.debug("Attempting to determine the real name of %s", startname) # workaround, allow user to specify 'ephemeral' # rather than more ec2 correct 'ephemeral0' if startname == "ephemeral": cfgmnt[i][0] = "ephemeral0" - log.debug("Adjusted mount option %s name from ephemeral to ephemeral0", (i + 1)) + log.debug(("Adjusted mount option %s " + "name from ephemeral to ephemeral0"), (i + 1)) if is_mdname(startname): newname = cloud.device_name_to_device(startname) @@ -136,7 +136,8 @@ def handle(_name, cfg, cloud, log, _args): break if cfgmnt_has: - log.debug("Not including %s, already previously included", startname) + log.debug(("Not including %s, already" + " previously included"), startname) continue cfgmnt.append(defmnt) @@ -159,7 +160,7 @@ def handle(_name, cfg, cloud, log, _args): dirs = [] for line in actlist: # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] + line[3] = "%s,%s" % (line[3], comment) if line[2] == "swap": needswap = True if line[1].startswith("/"): @@ -168,7 +169,7 @@ def handle(_name, cfg, cloud, log, _args): fstab_lines = [] fstab = util.load_file("/etc/fstab") - for line in fstab.read().splitlines(): + for line in fstab.splitlines(): try: toks = ws.split(line) if toks[3].find(comment) != -1: diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py index 36af6dfa..98ff2b85 100644 --- a/cloudinit/transforms/cc_phone_home.py +++ b/cloudinit/transforms/cc_phone_home.py @@ -24,9 +24,8 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE -from time import sleep - frequency = PER_INSTANCE + post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', 'hostname'] @@ -49,7 +48,7 @@ def handle(name, cfg, cloud, log, args): ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn(("Skipping module named %s, " + log.warn(("Skipping transform named %s, " "no 'url' found in 'phone_home' configuration"), name) return @@ -60,7 +59,8 @@ def handle(name, cfg, cloud, log, args): tries = int(tries) except: tries = 10 - util.logexc(log, "Configuration entry 'tries' is not an integer, using %s", tries) + util.logexc(log, ("Configuration entry 'tries'" + " is not an integer, using %s instead"), tries) if post_list == "all": post_list = post_list_all @@ -75,23 +75,37 @@ def handle(name, cfg, cloud, log, args): 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } - for n, path in pubkeys.iteritems(): + for (n, path) in pubkeys.iteritems(): try: all_keys[n] = util.load_file(path) except: - util.logexc(log, "%s: failed to open, can not phone home that data", path) + util.logexc(log, ("%s: failed to open, can not" + " phone home that data"), path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: - submit_keys[k] = "N/A" - log.warn("Requested key %s from 'post' configuration list not available", k) + submit_keys[k] = None + log.warn(("Requested key %s from 'post'" + " configuration list not available"), k) - url = templater.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + # Get them read to be posted + real_submit_keys = {} + for (k, v) in submit_keys.iteritems(): + if v is None: + real_submit_keys[k] = 'N/A' + else: + real_submit_keys[k] = str(v) + # Incase the url is parameterized + url_params = { + 'INSTANCE_ID': all_keys['instance_id'], + } + url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=submit_keys, retries=tries, sec_between=3) + uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) except: - util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) + util.logexc(log, ("Failed to post phone home data to" + " %s in %s tries"), url, tries) diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py index 0a21a929..76cc9732 100644 --- a/cloudinit/transforms/cc_puppet.py +++ b/cloudinit/transforms/cc_puppet.py @@ -24,31 +24,32 @@ import os import pwd import socket +from cloudinit import cfg as config from cloudinit import util -from cloudinit import cfg def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: - log.debug(("Skipping module named %s," + log.debug(("Skipping transform named %s," " no 'puppet' configuration found"), name) return puppet_cfg = cfg['puppet'] # Start by installing the puppet package ... - cloud.distro.install_packages(("puppet",)) + cloud.distro.install_packages(["puppet"]) # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf contents = util.load_file('/etc/puppet/puppet.conf') # Create object for reading puppet.conf values - puppet_config = cfg.DefaultingConfigParser() + puppet_config = config.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to # mix the rest up. First clean them up (TODO is this really needed??) - cleaned_contents = '\n'.join([i.lstrip() for i in contents.splitlines()]) + cleaned_lines = [i.lstrip() for i in contents.splitlines()] + cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), filename='/etc/puppet/puppet.conf') for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): @@ -81,7 +82,8 @@ def handle(name, cfg, cloud, log, _args): puppet_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - util.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + util.rename('/etc/puppet/puppet.conf', + '/etc/puppet/puppet.conf.old') contents = puppet_config.stringify() util.write_file('/etc/puppet/puppet.conf', contents) @@ -91,7 +93,8 @@ def handle(name, cfg, cloud, log, _args): '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) + util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + capture=False) elif os.path.exists('/sbin/chkconfig'): util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py index daaf4da9..fe012417 100644 --- a/cloudinit/transforms/cc_resizefs.py +++ b/cloudinit/transforms/cc_resizefs.py @@ -18,11 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import errno import os import stat -import sys -import tempfile import time from cloudinit import util @@ -46,15 +43,18 @@ def nodeify_path(devpth, where, log): if util.is_container(): log.debug("Inside container, ignoring mknod failure in resizefs") return - log.warn("Failed to make device node to resize %s at %s", where, devpth) + log.warn("Failed to make device node to resize %s at %s", + where, devpth) raise def get_fs_type(st_dev, path, log): try: - fs_type = util.find_devs_with(tag='TYPE', oformat='value', + dev_entries = util.find_devs_with(tag='TYPE', oformat='value', no_cache=True, path=path) - return fs_type + if not dev_entries: + return None + return dev_entries[0].strip() except util.ProcessExecutionError: util.logexc(log, ("Failed to get filesystem type" " of maj=%s, min=%s for path %s"), @@ -69,12 +69,16 @@ def handle(name, cfg, _cloud, log, args): resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) if not util.translate_bool(resize_root): - log.debug("Skipping module named %s, resizing disabled", name) + log.debug("Skipping transform named %s, resizing disabled", name) return # TODO is the directory ok to be used?? resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") util.ensure_dir(resize_root_d) + + # TODO: allow what is to be resized to + # be configurable?? + resize_what = "/" with util.SilentTemporaryFile(prefix="cloudinit.resizefs.", dir=resize_root_d, delete=True) as tfh: devpth = tfh.name @@ -86,23 +90,25 @@ def handle(name, cfg, _cloud, log, args): # auto deletion tfh.unlink_now() - # TODO: allow what is to be resized to - # be configurable?? - st_dev = nodeify_path(devpth, "/", log) - fs_type = get_fs_type(st_dev, devpath, log) + st_dev = nodeify_path(devpth, resize_what, log) + fs_type = get_fs_type(st_dev, devpth, log) + if not fs_type: + log.warn("Could not determine filesystem type of %s", resize_what) + return resizer = None - fstype_lc = fstype.lower() + fstype_lc = fs_type.lower() for (pfix, root_cmd) in resize_fs_prefixes_cmds: if fstype_lc.startswith(pfix): resizer = root_cmd break if not resizer: - log.warn("Not resizing unknown filesystem type %s", fs_type) + log.warn("Not resizing unknown filesystem type %s for %s", + fs_type, resize_what) return - log.debug("Resizing using %s", resizer) + log.debug("Resizing %s (%s) using %s", resize_what, fs_type, resizer) resize_cmd = [resizer, devpth] if resize_root == "noblock": @@ -125,8 +131,8 @@ def do_resize(resize_cmd, log): start = time.time() try: util.subp(resize_cmd) - except util.ProcessExecutionError as e: - util.logexc(log, "Failed to resize filesystem (using %s)", resize_cmd) + except util.ProcessExecutionError: + util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise tot_time = int(time.time() - start) log.debug("Resizing took %s seconds", tot_time) diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py index cde11b54..40d76c89 100644 --- a/cloudinit/transforms/cc_rightscale_userdata.py +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -53,16 +53,19 @@ def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: - log.warn("Failed to get raw userdata in module %s", name) + log.warn("Failed to get raw userdata in transform %s", name) return try: mdict = parse_qs(ud) if not mdict or not my_hookname in mdict: - log.debug("Skipping module %s, did not find %s in parsed raw userdata", name, my_hookname) + log.debug(("Skipping transform %s, " + "did not find %s in parsed" + " raw userdata"), name, my_hookname) return except: - log.warn("Failed to parse query string %s into a dictionary", ud) + util.logexc(log, ("Failed to parse query string %s" + " into a dictionary"), ud) raise wrote_fns = [] @@ -83,7 +86,8 @@ def handle(name, _cfg, cloud, log, _args): wrote_fns.append(fname) except Exception as e: captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", my_name, url, fname) + util.logexc(log, "%s failed to read %s and write %s", + my_name, url, fname) if wrote_fns: log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) @@ -93,6 +97,6 @@ def handle(name, _cfg, cloud, log, _args): log.debug("%s urls were skipped or failed", skipped) if captured_excps: - log.warn("%s failed with exceptions, re-raising the last one", len(captured_excps)) + log.warn("%s failed with exceptions, re-raising the last one", + len(captured_excps)) raise captured_excps[-1] - diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py index ccbe68ff..71b74711 100644 --- a/cloudinit/transforms/cc_rsyslog.py +++ b/cloudinit/transforms/cc_rsyslog.py @@ -36,7 +36,8 @@ def handle(name, cfg, cloud, log, _args): # process 'rsyslog' if not 'rsyslog' in cfg: - log.debug("Skipping module named %s, no 'rsyslog' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'rsyslog' key in configuration"), name) return def_dir = cfg.get('rsyslog_dir', DEF_DIR) @@ -62,15 +63,16 @@ def handle(name, cfg, cloud, log, _args): if not filename.startswith("/"): filename = os.path.join(def_dir, filename) + # Truncate filename first time you see it omode = "ab" - # truncate filename first time you see it if filename not in files: omode = "wb" files.append(filename) try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: + contents = "%s\n" % (content) + util.write_file(filename, contents, omode=omode) + except Exception: util.logexc(log, "Failed to write to %s", filename) # Attempt to restart syslogd @@ -87,8 +89,8 @@ def handle(name, cfg, cloud, log, _args): log.debug("Restarting rsyslog") util.subp(['service', 'rsyslog', 'restart']) restarted = True - except Exception as e: - util.logexc("Failed restarting rsyslog") + except Exception: + util.logexc(log, "Failed restarting rsyslog") if restarted: # This only needs to run if we *actually* restarted diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py index 19c0e721..31a254a5 100644 --- a/cloudinit/transforms/cc_runcmd.py +++ b/cloudinit/transforms/cc_runcmd.py @@ -25,13 +25,14 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: - log.debug("Skipping module named %s, no 'runcmd' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'runcmd' key in configuration"), name) return - outfile = os.path.join(cloud.get_ipath('scripts'), "runcmd") + out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd") cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(outfile, content, 0700) + util.write_file(out_fn, content, 0700) except: - util.logexc(log, "Failed to shellify %s into file %s", cmd, outfile) + util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py index 47cbc194..d05d2a1e 100644 --- a/cloudinit/transforms/cc_salt_minion.py +++ b/cloudinit/transforms/cc_salt_minion.py @@ -21,16 +21,17 @@ from cloudinit import util # Note: see http://saltstack.org/topics/installation/ -def handle(name, cfg, cloud, _log, _args): +def handle(name, cfg, cloud, log, _args): # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: - log.debug("Skipping module named %s, no 'salt_minion' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'salt_minion' key in configuration"), name) return salt_cfg = cfg['salt_minion'] # Start by installing the salt package ... - cloud.distro.install_packages(("salt",)) + cloud.distro.install_packages(["salt"]) # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py index bcdf4400..364e1d02 100644 --- a/cloudinit/transforms/cc_scripts_per_boot.py +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -29,12 +29,13 @@ frequency = PER_ALWAYS script_subdir = 'per-boot' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py index 8d6609a1..d75ab47d 100644 --- a/cloudinit/transforms/cc_scripts_per_instance.py +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -29,12 +29,13 @@ frequency = PER_INSTANCE script_subdir = 'per-instance' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py index dbcec05d..80f8c325 100644 --- a/cloudinit/transforms/cc_scripts_per_once.py +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -29,12 +29,13 @@ frequency = PER_ONCE script_subdir = 'per-once' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py index 1e438ee6..f4fe3a2a 100644 --- a/cloudinit/transforms/cc_scripts_user.py +++ b/cloudinit/transforms/cc_scripts_user.py @@ -26,14 +26,17 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE +script_subdir = 'scripts' -def handle(_name, _cfg, cloud, log, _args): + +def handle(name, _cfg, cloud, log, _args): # This is written to by the user data handlers # Ie, any custom shell scripts that come down # go here... - runparts_path = os.path.join(cloud.get_ipath_cur(), "scripts") + runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", "user-data", runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py index fa2b59c2..3ac8a8fa 100644 --- a/cloudinit/transforms/cc_set_hostname.py +++ b/cloudinit/transforms/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in %s"), name) + " not setting the hostname in transform %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py index 4f2cdb97..c0cc4e84 100644 --- a/cloudinit/transforms/cc_set_passwords.py +++ b/cloudinit/transforms/cc_set_passwords.py @@ -22,7 +22,7 @@ import sys from cloudinit import util -from string import letters, digits +from string import letters, digits # pylint: disable=W0402 # We are removing certain 'painful' letters/numbers pw_set = (letters.translate(None, 'loLOI') + @@ -71,11 +71,13 @@ def handle(_name, cfg, cloud, log, args): util.subp(['chpasswd'], ch_in) except Exception as e: errors.append(e) - util.logexc(log, "Failed to set passwords with chpasswd for %s", users) + util.logexc(log, + "Failed to set passwords with chpasswd for %s", users) if len(randlist): - sys.stderr.write("%s\n%s\n" % ("Set the following 'random' passwords\n", - '\n'.join(randlist))) + blurb = ("Set the following 'random' passwords\n", + '\n'.join(randlist)) + sys.stderr.write("%s\n%s\n" % blurb) if expire: expired_users = [] diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py index db6848d9..3c2b3622 100644 --- a/cloudinit/transforms/cc_ssh.py +++ b/cloudinit/transforms/cc_ssh.py @@ -65,8 +65,7 @@ def handle(_name, cfg, cloud, log, _args): tgt_fn = key2file[key][0] tgt_perms = key2file[key][1] util.write_file(tgt_fn, val, tgt_perms) - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue @@ -78,11 +77,15 @@ def handle(_name, cfg, cloud, log, _args): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except: - util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) + util.logexc(log, ("Failed generated a key" + " for %s from %s"), pair[0], pair[1]) else: # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', generate_keys): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + genkeys = util.get_cfg_option_list(cfg, + 'ssh_genkeytypes', + generate_keys) + for keytype in genkeys: + keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: @@ -90,26 +93,27 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: - util.logexc(log, "Failed generating key type %s to file %s", keytype, keyfile) + util.logexc(log, ("Failed generating key type" + " %s to file %s"), keytype, keyfile) try: user = util.get_cfg_option_str(cfg, 'user') disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) + DISABLE_ROOT_OPTS) keys = cloud.get_public_ssh_keys() or [] if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) - apply_credentials(keys, user, disable_root, disable_root_opts, log) + apply_credentials(keys, user, disable_root, disable_root_opts) except: util.logexc(log, "Applying ssh credentials failed!") def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): + disable_root_opts=DISABLE_ROOT_OPTS): keys = set(keys) if user: diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py index 019413d4..d57e4665 100644 --- a/cloudinit/transforms/cc_ssh_import_id.py +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -33,10 +33,14 @@ def handle(name, cfg, _cloud, log, args): ids = args[1:] else: user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) if len(ids) == 0: - log.debug("Skipping module named %s, no ids found to import", name) + log.debug("Skipping transform named %s, no ids found to import", name) + return + + if not user: + log.debug("Skipping transform named %s, no user found to import", name) return cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py index 02cbf2dc..747c436c 100644 --- a/cloudinit/transforms/cc_timezone.py +++ b/cloudinit/transforms/cc_timezone.py @@ -18,20 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import util + from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: timezone = args[0] else: timezone = util.get_cfg_option_str(cfg, "timezone", False) if not timezone: - log.debug("Skipping module named %s, no 'timezone' specified", name) + log.debug("Skipping transform named %s, no 'timezone' specified", name) return - + # Let the distro handle settings its timezone cloud.distro.set_timezone(timezone) diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py index 361097a6..d0e56183 100644 --- a/cloudinit/transforms/cc_update_etc_hosts.py +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -30,22 +30,30 @@ def handle(name, cfg, cloud, log, _args): manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - # Render from template file if not hostname: - log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + log.warn(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return - tpl_fn_name = cloud.get_template_filename("hosts.%s" % (cloud.distro.name())) + + # Render from a template file + distro_n = cloud.distro.name + tpl_fn_name = cloud.get_template_filename("hosts.%s" % (distro_n)) if not tpl_fn_name: - raise Exception("No hosts template could be found for distro %s" % (cloud.distro.name())) + raise Exception(("No hosts template could be" + " found for distro %s") % (distro_n)) + templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) + elif manage_hosts == "localhost": - log.debug("Managing localhost in /etc/hosts") (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + log.warn(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return + + log.debug("Managing localhost in /etc/hosts") cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in %s"), name) + " not managing /etc/hosts in transform %s"), name) diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py index 439bdcb3..58444fab 100644 --- a/cloudinit/transforms/cc_update_hostname.py +++ b/cloudinit/transforms/cc_update_hostname.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -27,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in %s"), name) + " not updating the hostname in transform %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/transforms/cc_welcome.py b/cloudinit/transforms/cc_welcome.py index 0db71125..04691d21 100644 --- a/cloudinit/transforms/cc_welcome.py +++ b/cloudinit/transforms/cc_welcome.py @@ -35,9 +35,9 @@ welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at " frequency = PER_ALWAYS -def handle(name, cfg, cloud, log, args): +def handle(_name, cfg, cloud, log, args): - welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg"): + welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg") if not welcome_msg: tpl_fn = cloud.get_template_filename("welcome_msg") if tpl_fn: @@ -54,7 +54,7 @@ def handle(name, cfg, cloud, log, args): 'stage': stage, 'version': version.version_string(), 'uptime': util.uptime(), - 'timestamp', util.time_rfc2822(), + 'timestamp': util.time_rfc2822(), } try: contents = templater.render_string(welcome_msg, tpl_params) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 64fc2734..9915b8b0 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -22,14 +22,15 @@ import os -import glob import email - +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from email.mime.base import MIMEBase from cloudinit import importer from cloudinit import log as logging +from cloudinit import url_helper from cloudinit import util from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) @@ -86,7 +87,7 @@ class UserDataProcessor(object): self.paths = paths def process(self, blob): - base_msg = ud.convert_string(blob) + base_msg = convert_string(blob) process_msg = MIMEMultipart() self._process_msg(base_msg, process_msg) return process_msg @@ -105,7 +106,7 @@ class UserDataProcessor(object): ctype_orig = UNDEF_TYPE if ctype_orig in TYPE_NEEDED: - ctype = ud.type_from_starts_with(payload) + ctype = type_from_starts_with(payload) if ctype is None: ctype = ctype_orig @@ -158,7 +159,7 @@ class UserDataProcessor(object): if not url_helper.ok_http_code(st): content = '' - new_msg = ud.convert_string(content) + new_msg = convert_string(content) self._process_msg(new_msg, append_msg) def _explode_archive(self, archive, append_msg): @@ -179,7 +180,7 @@ class UserDataProcessor(object): content = ent.get('content', '') mtype = ent.get('type') if not mtype: - mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) + mtype = type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) maintype, subtype = mtype.split('/', 1) if maintype == "text": diff --git a/cloudinit/util.py b/cloudinit/util.py index 7259d933..1f884df8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -22,8 +22,8 @@ from StringIO import StringIO +import copy as obj_copy import contextlib -import copy import errno import glob import grp @@ -35,12 +35,11 @@ import pwd import random import shutil import socket -import string +import string # pylint: disable=W0402 import subprocess import sys import tempfile import time -import traceback import types import urlparse @@ -171,7 +170,8 @@ def fork_cb(child_cb, *args): child_cb(*args) os._exit(0) # pylint: disable=W0212 except: - logexc(LOG, "Failed forking and calling callback %s", obj_name(child_cb)) + logexc(LOG, ("Failed forking and" + " calling callback %s"), obj_name(child_cb)) os._exit(1) # pylint: disable=W0212 else: LOG.debug("Forked child %s who will run callback %s", @@ -549,10 +549,11 @@ def load_yaml(blob, default=None, allowed=(dict,)): converted = yaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... - raise TypeError("Yaml load allows %s root types, but got %s instead" % + raise TypeError(("Yaml load allows %s root types," + " but got %s instead") % (allowed, obj_name(converted))) loaded = converted - except (yaml.YAMLError, TypeError, ValueError) as exc: + except (yaml.YAMLError, TypeError, ValueError): logexc(LOG, "Failed loading yaml blob") return loaded @@ -833,15 +834,12 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options (out, _err) = subp(cmd) - if path: - return out.strip() - else: - entries = [] - for line in out.splitlines(): - line = line.strip() - if line: - entries.append(line) - return entries + entries = [] + for line in out.splitlines(): + line = line.strip() + if line: + entries.append(line) + return entries def load_file(fname, read_cb=None, quiet=False): @@ -1109,7 +1107,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None): def get_builtin_cfg(): # Deep copy so that others can't modify - return copy.deepcopy(CFG_BUILTIN) + return obj_copy.deepcopy(CFG_BUILTIN) def sym_link(source, link): @@ -1140,16 +1138,14 @@ def time_rfc2822(): def uptime(): + uptime_str = '??' try: - uptimef = load_file("/proc/uptime").strip() - if not uptimef: - uptime = 'na' - else: - uptime = uptimef.split()[0] + contents = load_file("/proc/uptime").strip() + if contents: + uptime_str = contents.split()[0] except: logexc(LOG, "Unable to read uptime from /proc/uptime") - uptime = '??' - return uptime + return uptime_str def ensure_file(path): @@ -1261,7 +1257,8 @@ def shellify(cmdlist, add_header=True): content = "%s%s\n" % (content, args) else: raise RuntimeError(("Unable to shellify type %s" - " which is not a list or string") % (obj_name(args))) + " which is not a list or string") + % (obj_name(args))) LOG.debug("Shellified %s to %s", cmdlist, content) return content @@ -1275,8 +1272,7 @@ def is_container(): try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no - cmd = [helper] - subp(cmd, allowed_rc=[0]) + subp([helper]) return True except (IOError, OSError): pass -- cgit v1.2.3 From 5597e947248ee7fbda17d3876206f1d45f97bc00 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 10:56:08 -0700 Subject: Make the top level __init__ similar to the other top level __inits__ where just util functions and base classes are defined there. --- cloudinit/handlers/__init__.py | 196 +++++++++++++++++++++++++++++++++++++ cloudinit/handlers/boot_hook.py | 21 ++-- cloudinit/handlers/cloud_config.py | 28 +++--- cloudinit/handlers/shell_script.py | 10 +- cloudinit/handlers/upstart_job.py | 10 +- 5 files changed, 235 insertions(+), 30 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 09163abb..0ff80a9e 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -20,3 +20,199 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import abc +import os + +from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) + +from cloudinit import importer +from cloudinit import log as logging +from cloudinit import url_helper +from cloudinit import util + +LOG = logging.getLogger(__name__) + +# Used as the content type when a message is not multipart +# and it doesn't contain its own content-type +NOT_MULTIPART_TYPE = "text/x-not-multipart" + +# When none is assigned this gets used +OCTET_TYPE = 'application/octet-stream' + +# Special content types that signal the start and end of processing +CONTENT_END = "__end__" +CONTENT_START = "__begin__" +CONTENT_SIGNALS = [CONTENT_START, CONTENT_END] + +# Used when a part-handler type is encountered +# to allow for registration of new types. +PART_CONTENT_TYPES = ["text/part-handler"] +PART_HANDLER_FN_TMPL = 'part-handler-%03d' + +# For parts without filenames +PART_FN_TPL = 'part-%03d' + +# Different file beginnings to there content type +INCLUSION_TYPES_MAP = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook', + '#cloud-config-archive': 'text/cloud-config-archive', +} + +# Sorted longest first +INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), + key=(lambda e: 0 - len(e))) + + +class Handler(object): + + __metaclass__ = abc.ABCMeta + + def __init__(self, frequency, version=2): + self.handler_version = version + self.frequency = frequency + + def __repr__(self): + return "%s: [%s]" % (util.obj_name(self), self.list_types()) + + @abc.abstractmethod + def list_types(self): + raise NotImplementedError() + + def handle_part(self, data, ctype, filename, payload, frequency): + return self._handle_part(data, ctype, filename, payload, frequency) + + @abc.abstractmethod + def _handle_part(self, data, ctype, filename, payload, frequency): + raise NotImplementedError() + + +def run_part(mod, data, ctype, filename, payload, frequency): + mod_freq = mod.frequency + if not (mod_freq == PER_ALWAYS or + (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): + return + mod_ver = mod.handler_version + try: + if mod_ver == 1: + mod.handle_part(data, ctype, filename, payload) + else: + mod.handle_part(data, ctype, filename, payload, frequency) + except: + util.logexc(LOG, ("Failed calling mod %s (%s, %s, %s)" + " with frequency %s"), + mod, ctype, filename, + mod_ver, frequency) + + +def call_begin(mod, data, frequency): + run_part(mod, data, CONTENT_START, None, None, frequency) + + +def call_end(mod, data, frequency): + run_part(mod, data, CONTENT_END, None, None, frequency) + + +def walker_handle_handler(pdata, _ctype, _filename, payload): + curcount = pdata['handlercount'] + modname = PART_HANDLER_FN_TMPL % (curcount) + frequency = pdata['frequency'] + modfname = os.path.join(pdata['handlerdir'], "%s" % (modname)) + if not modfname.endswith(".py"): + modfname = "%s.py" % (modfname) + # TODO: Check if path exists?? + util.write_file(modfname, payload, 0600) + handlers = pdata['handlers'] + try: + mod = fixup_handler(importer.import_module(modname)) + handlers.register(mod) + call_begin(mod, pdata['data'], frequency) + pdata['handlercount'] = curcount + 1 + except: + util.logexc(LOG, "Failed at registered python file: %s", modfname) + + +def _extract_first_or_bytes(blob, size): + # Extract the first line upto X bytes or X bytes from more than the + # first line if the first line does not contain enough bytes + first_line = blob.split("\n", 1)[0] + if len(first_line) >= size: + start = first_line[:size] + else: + start = blob[0:size] + return start + + +def walker_callback(pdata, ctype, filename, payload): + if ctype in PART_CONTENT_TYPES: + walker_handle_handler(pdata, ctype, filename, payload) + return + handlers = pdata['handlers'] + if ctype not in handlers: + # Extract the first line or 24 bytes for displaying in the log + start = _extract_first_or_bytes(payload, 24) + details = "'%s...'" % (start.encode("string-escape")) + if ctype == NOT_MULTIPART_TYPE: + LOG.warning("Unhandled non-multipart (%s) userdata: %s", + ctype, details) + else: + LOG.warning("Unhandled unknown content-type (%s) userdata: %s", + ctype, details) + else: + run_part(handlers[ctype], pdata['data'], ctype, filename, + payload, pdata['frequency']) + + +# Callback is a function that will be called with +# (data, content_type, filename, payload) +def walk(msg, callback, data): + partnum = 0 + for part in msg.walk(): + # multipart/* are just containers + if part.get_content_maintype() == 'multipart': + continue + + ctype = part.get_content_type() + if ctype is None: + ctype = OCTET_TYPE + + filename = part.get_filename() + if not filename: + filename = PART_FN_TPL % (partnum) + + callback(data, ctype, filename, part.get_payload(decode=True)) + partnum = partnum + 1 + + +def fixup_handler(mod, def_freq=PER_INSTANCE): + if not hasattr(mod, "handler_version"): + setattr(mod, "handler_version", 1) + if not hasattr(mod, 'list_types'): + def empty_types(): + return [] + setattr(mod, 'list_types', empty_types) + if not hasattr(mod, 'frequency'): + setattr(mod, 'frequency', def_freq) + else: + freq = mod.frequency + if freq and freq not in FREQUENCIES: + LOG.warn("Handler %s has an unknown frequency %s", mod, freq) + if not hasattr(mod, 'handle_part'): + def empty_handler(_data, _ctype, _filename, _payload): + pass + setattr(mod, 'handle_part', empty_handler) + return mod + + +def type_from_starts_with(payload, default=None): + for text in INCLUSION_SRCH: + if payload.startswith(text): + return INCLUSION_TYPES_MAP[text] + return default + + diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index b3aab366..10f60b8d 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -22,8 +22,8 @@ import os +from cloudinit import handlers from cloudinit import log as logging -from cloudinit import user_data as ud from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -31,31 +31,34 @@ from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) -class BootHookPartHandler(ud.PartHandler): +class BootHookPartHandler(handlers.Handler): def __init__(self, paths, instance_id, **_kwargs): - ud.PartHandler.__init__(self, PER_ALWAYS) + handlers.Handler.__init__(self, PER_ALWAYS) self.boothook_dir = paths.get_ipath("boothooks") self.instance_id = instance_id def list_types(self): return [ - ud.type_from_starts_with("#cloud-boothook"), + handlers.type_from_starts_with("#cloud-boothook"), ] - def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in ud.CONTENT_SIGNALS: - return - + def _write_part(self, payload, filename): filename = util.clean_filename(filename) payload = util.dos2unix(payload) prefix = "#cloud-boothook" start = 0 if payload.startswith(prefix): start = len(prefix) + 1 - filepath = os.path.join(self.boothook_dir, filename) contents = payload[start:] util.write_file(filepath, contents, 0700) + return filepath + + def _handle_part(self, _data, ctype, filename, payload, _frequency): + if ctype in handlers.CONTENT_SIGNALS: + return + + filepath = self._write_part(payload, filename) try: env = os.environ.copy() if self.instance_id: diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 12d1bd96..f6d95244 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -20,8 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import handlers from cloudinit import log as logging -from cloudinit import user_data as ud from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -29,28 +29,34 @@ from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) -class CloudConfigPartHandler(ud.PartHandler): +class CloudConfigPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): - ud.PartHandler.__init__(self, PER_ALWAYS) + handlers.Handler.__init__(self, PER_ALWAYS) self.cloud_buf = [] self.cloud_fn = paths.get_ipath("cloud_config") def list_types(self): return [ - ud.type_from_starts_with("#cloud-config"), + handlers.type_from_starts_with("#cloud-config"), ] + def _write_cloud_config(self, buf): + if not self.cloud_fn: + return + lines = [str(b) for b in buf] + payload = "\n".join(lines) + util.write_file(self.cloud_fn, payload, 0600) + def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype == ud.CONTENT_START: + if ctype == handlers.CONTENT_START: self.cloud_buf = [] return - - if ctype == ud.CONTENT_END: - payload = "\n".join(self.cloud_buf) - util.write_file(self.cloud_fn, payload, 0600) + if ctype == handlers.CONTENT_END: + self._write_cloud_config(self.cloud_buf) self.cloud_buf = [] return filename = util.clean_filename(filename) - entry = "\n".join(["#%s" % (filename), str(payload)]) - self.cloud_buf.append(entry) + if not filename: + filename = '??' + self.cloud_buf.extend(["#%s" % (filename), str(payload)]) diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index f6e2ef16..a9d8e544 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -22,8 +22,8 @@ import os +from cloudinit import handlers from cloudinit import log as logging -from cloudinit import user_data as ud from cloudinit import util from cloudinit.settings import (PER_ALWAYS) @@ -31,18 +31,18 @@ from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) -class ShellScriptPartHandler(ud.PartHandler): +class ShellScriptPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): - ud.PartHandler.__init__(self, PER_ALWAYS) + handlers.Handler.__init__(self, PER_ALWAYS) self.script_dir = paths.get_ipath_cur('scripts') def list_types(self): return [ - ud.type_from_starts_with("#!"), + handlers.type_from_starts_with("#!"), ] def _handle_part(self, _data, ctype, filename, payload, _frequency): - if ctype in ud.CONTENT_SIGNALS: + if ctype in handlers.CONTENT_SIGNALS: # TODO: maybe delete existing things here return diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 059a4851..411a5d68 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -23,8 +23,8 @@ import os +from cloudinit import handlers from cloudinit import log as logging -from cloudinit import user_data as ud from cloudinit import util from cloudinit.settings import (PER_INSTANCE) @@ -32,18 +32,18 @@ from cloudinit.settings import (PER_INSTANCE) LOG = logging.getLogger(__name__) -class UpstartJobPartHandler(ud.PartHandler): +class UpstartJobPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): - ud.PartHandler.__init__(self, PER_INSTANCE) + handlers.Handler.__init__(self, PER_INSTANCE) self.upstart_dir = paths.upstart_conf_d def list_types(self): return [ - ud.type_from_starts_with("#upstart-job"), + handlers.type_from_starts_with("#upstart-job"), ] def _handle_part(self, _data, ctype, filename, payload, frequency): - if ctype in ud.CONTENT_SIGNALS: + if ctype in handlers.CONTENT_SIGNALS: return if not self.upstart_dir: -- cgit v1.2.3 From 37da8d7e9dc74e65537bd64ea406cdebe0a6b539 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 12:58:30 -0700 Subject: They are handlers not mods ;) --- cloudinit/handlers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 0ff80a9e..afa9ec3b 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -104,7 +104,7 @@ def run_part(mod, data, ctype, filename, payload, frequency): else: mod.handle_part(data, ctype, filename, payload, frequency) except: - util.logexc(LOG, ("Failed calling mod %s (%s, %s, %s)" + util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" " with frequency %s"), mod, ctype, filename, mod_ver, frequency) -- cgit v1.2.3 From c9c3de2fad73af119cf9002799a875c71fda7b7a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:06:02 -0700 Subject: When a handler version is set but to an unknown non-int convertable value, treat it as 1 --- cloudinit/handlers/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index afa9ec3b..156e228d 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -98,11 +98,18 @@ def run_part(mod, data, ctype, filename, payload, frequency): (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): return mod_ver = mod.handler_version + # Sanity checks on version (should be an int convertable) try: - if mod_ver == 1: - mod.handle_part(data, ctype, filename, payload) - else: + mod_ver = int(mod_ver) + except: + mod_ver = None + try: + if mod_ver and mod_ver >= 2: + # Treat as v. 2 which does get a frequency mod.handle_part(data, ctype, filename, payload, frequency) + else: + # Treat as v. 1 which gets no frequency + mod.handle_part(data, ctype, filename, payload) except: util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" " with frequency %s"), -- cgit v1.2.3 From 457b9998d760150efb17658f9a0fd4816417577e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:11:22 -0700 Subject: Pass in the datasource as a option, instead of the lower level instance id. This allows for others to use datasource functions if they desire to instead of being restricted. +1 for future use ;) --- cloudinit/handlers/boot_hook.py | 6 ++++-- cloudinit/stages.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 10f60b8d..fa675f09 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -32,10 +32,12 @@ LOG = logging.getLogger(__name__) class BootHookPartHandler(handlers.Handler): - def __init__(self, paths, instance_id, **_kwargs): + def __init__(self, paths, datasource, **_kwargs): handlers.Handler.__init__(self, PER_ALWAYS) self.boothook_dir = paths.get_ipath("boothooks") - self.instance_id = instance_id + self.instance_id = None + if datasource: + self.instance_id = datasource.get_instance_id() def list_types(self): return [ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8fa9d6d3..2931830c 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -287,7 +287,7 @@ class Init(object): def _default_userdata_handlers(self): opts = { 'paths': self.paths, - 'instance_id': self.datasource.get_instance_id(), + 'datasource': self.datasource, } # TODO Hmmm, should we dynamically import these?? def_handlers = [ -- cgit v1.2.3 From a1f20dff76b00c7502f005c2b597f49c89cc2a09 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:23:32 -0700 Subject: Check instance id against none, and not just empty/false/0/none since 0 or empty might be valid --- cloudinit/handlers/boot_hook.py | 2 +- cloudinit/stages.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index fa675f09..456b8020 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -63,7 +63,7 @@ class BootHookPartHandler(handlers.Handler): filepath = self._write_part(payload, filename) try: env = os.environ.copy() - if self.instance_id: + if self.instance_id is not None: env['INSTANCE_ID'] = str(self.instance_id) util.subp([filepath], env=env) except util.ProcessExecutionError: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2931830c..c2d78a78 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -233,13 +233,13 @@ class Init(object): # Write what the datasource was and is.. ds = "%s: %s" % (util.obj_name(self.datasource), self.datasource) - previous_ds = '' + previous_ds = None ds_fn = os.path.join(idir, 'datasource') try: previous_ds = util.load_file(ds_fn).strip() except Exception: pass - if not previous_ds: + if previous_ds is None: # TODO: ?? is this right previous_ds = ds util.write_file(ds_fn, "%s\n" % ds) @@ -248,14 +248,14 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = '' + previous_iid = None p_iid_fn = os.path.join(dp, 'previous-instance-id') c_iid_fn = os.path.join(dp, 'instance-id') try: previous_iid = util.load_file(p_iid_fn).strip() except Exception: pass - if not previous_iid: + if previous_iid is None: # TODO: ?? is this right previous_iid = iid util.write_file(c_iid_fn, "%s\n" % iid) -- cgit v1.2.3 From 005ff658c6f301303269e14dfef4c27ada5c3982 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 20:16:26 -0700 Subject: 1. Ensure a that when a bad version is found, that it gets set to 1. 2. Increment part handler count even if it doesn't get registered (this shouldn't cause any problems) --- cloudinit/handlers/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 156e228d..0ef704f7 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -102,7 +102,7 @@ def run_part(mod, data, ctype, filename, payload, frequency): try: mod_ver = int(mod_ver) except: - mod_ver = None + mod_ver = 1 try: if mod_ver and mod_ver >= 2: # Treat as v. 2 which does get a frequency @@ -134,12 +134,12 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): modfname = "%s.py" % (modfname) # TODO: Check if path exists?? util.write_file(modfname, payload, 0600) + pdata['handlercount'] = curcount + 1 handlers = pdata['handlers'] try: mod = fixup_handler(importer.import_module(modname)) handlers.register(mod) call_begin(mod, pdata['data'], frequency) - pdata['handlercount'] = curcount + 1 except: util.logexc(LOG, "Failed at registered python file: %s", modfname) -- cgit v1.2.3 From 71a661de85dc910c82c6769430232b555a86198a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 20:22:16 -0700 Subject: Update error messaging for when importing/registring a part-handler fails --- cloudinit/handlers/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 0ef704f7..3944c661 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -141,7 +141,8 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): handlers.register(mod) call_begin(mod, pdata['data'], frequency) except: - util.logexc(LOG, "Failed at registered python file: %s", modfname) + util.logexc(LOG, ("Failed at registering python file: %s" + " (part handler %s)"), modfname, curcount) def _extract_first_or_bytes(blob, size): -- cgit v1.2.3 From f90d81a118cc119cba7130b30f71ac1f0d0f409e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 23:15:05 -0700 Subject: 1. Allow for checking against lower case include types. 2. Perform lstrip on the payload, just incase people put spaces/whitespace before the type. --- cloudinit/handlers/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 3944c661..d3374666 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -218,8 +218,10 @@ def fixup_handler(mod, def_freq=PER_INSTANCE): def type_from_starts_with(payload, default=None): + payload_lc = payload.lower() + payload_lc = payload_lc.lstrip() for text in INCLUSION_SRCH: - if payload.startswith(text): + if payload_lc.startswith(text): return INCLUSION_TYPES_MAP[text] return default -- cgit v1.2.3 From bcaa16409d7db7bc13456f8df47688e9e6f4f5ed Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 17 Jun 2012 00:15:04 -0700 Subject: Log handler + info that is about to be called --- cloudinit/handlers/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d3374666..c6f2119c 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -104,7 +104,9 @@ def run_part(mod, data, ctype, filename, payload, frequency): except: mod_ver = 1 try: - if mod_ver and mod_ver >= 2: + LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", + mod, ctype, filename, mod_ver, frequency) + if mod_ver >= 2: # Treat as v. 2 which does get a frequency mod.handle_part(data, ctype, filename, payload, frequency) else: -- cgit v1.2.3 From ec4bdc4fb8d8d3a8f8b4f498eb47eac740485ede Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 17:13:55 -0700 Subject: Massive pylint + pep8 fixups! --- bin/cloud-init | 20 +++++++-------- cloudinit/cloud.py | 3 ++- cloudinit/config/__init__.py | 2 +- cloudinit/config/cc_chef.py | 14 +++++------ cloudinit/config/cc_disable_ec2_metadata.py | 4 +-- cloudinit/config/cc_final_message.py | 8 +++--- cloudinit/config/cc_foo.py | 8 +++--- cloudinit/config/cc_keys_to_console.py | 8 +++--- cloudinit/config/cc_landscape.py | 2 +- cloudinit/config/cc_mcollective.py | 4 +-- cloudinit/config/cc_mounts.py | 13 +++++----- cloudinit/config/cc_phone_home.py | 11 +++++--- cloudinit/config/cc_puppet.py | 2 +- cloudinit/config/cc_resizefs.py | 10 ++++---- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/config/cc_scripts_per_boot.py | 6 ++--- cloudinit/config/cc_scripts_per_instance.py | 6 ++--- cloudinit/config/cc_scripts_per_once.py | 6 ++--- cloudinit/config/cc_scripts_user.py | 6 ++--- cloudinit/config/cc_set_passwords.py | 4 +-- cloudinit/config/cc_ssh.py | 39 +++++++++++++++-------------- cloudinit/distros/__init__.py | 1 - cloudinit/distros/rhel.py | 14 +++++------ cloudinit/distros/ubuntu.py | 6 ++--- cloudinit/handlers/__init__.py | 8 +++--- cloudinit/helpers.py | 6 ++--- cloudinit/log.py | 2 -- cloudinit/settings.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceEc2.py | 6 ++--- cloudinit/sources/DataSourceMAAS.py | 1 + cloudinit/sources/DataSourceNoCloud.py | 2 +- cloudinit/ssh_util.py | 5 ++-- cloudinit/stages.py | 6 ++--- cloudinit/url_helper.py | 14 +++++------ cloudinit/user_data.py | 28 +++++++++------------ cloudinit/util.py | 37 ++++++++++++++------------- 38 files changed, 159 insertions(+), 161 deletions(-) (limited to 'cloudinit/handlers') diff --git a/bin/cloud-init b/bin/cloud-init index 032d5f39..c1788ef4 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -75,6 +75,7 @@ def welcome(action): sys.stderr.flush() LOG.info(welcome_msg) + def extract_fns(args): # Files are already opened so lets just pass that along # since it would of broke if it couldn't have @@ -329,11 +330,11 @@ def main_single(name, args): def main(): parser = argparse.ArgumentParser() - + # Top level args - parser.add_argument('--version', '-v', action='version', + parser.add_argument('--version', '-v', action='version', version='%(prog)s ' + (version.version_string())) - parser.add_argument('--file', '-f', action='append', + parser.add_argument('--file', '-f', action='append', dest='files', help=('additional yaml configuration' ' files to use'), @@ -345,18 +346,18 @@ def main(): subparsers = parser.add_subparsers() # Each action and its sub-options (if any) - parser_init = subparsers.add_parser('init', + parser_init = subparsers.add_parser('init', help=('initializes cloud-init and' ' performs initial modules')) parser_init.add_argument("--local", '-l', action='store_true', help="start in local mode (default: %(default)s)", default=False) - # This is used so that we can know which action is selected + + # This is used so that we can know which action is selected + # the functor to use to run this subcommand parser_init.set_defaults(action=('init', main_init)) # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser('modules', + parser_mod = subparsers.add_parser('modules', help=('activates modules ' 'using a given configuration key')) parser_mod.add_argument("--mode", '-m', action='store', @@ -368,7 +369,7 @@ def main(): # These settings are used when you want to query information # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', + parser_query = subparsers.add_parser('query', help=('query information stored ' 'in cloud-init')) parser_query.add_argument("--name", '-n', action="store", @@ -378,7 +379,7 @@ def main(): parser_query.set_defaults(action=('query', main_query)) # This subcommand allows you to run a single module - parser_single = subparsers.add_parser('single', + parser_single = subparsers.add_parser('single', help=('run a single module ')) parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", @@ -394,10 +395,10 @@ def main(): ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) - args = parser.parse_args() # Setup basic logging to start (until reinitialized) + # iff in debug mode... if args.debug: logging.setupBasicLogging() @@ -407,4 +408,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 90679202..6cdcb76a 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -38,6 +38,7 @@ LOG = logging.getLogger(__name__) # as providing a backwards compatible object that can be maintained # while the stages/other objects can be worked on independently... + class Cloud(object): def __init__(self, datasource, paths, cfg, distro, runners): self.datasource = datasource @@ -71,7 +72,7 @@ class Cloud(object): # The rest of thes are just useful proxies def get_userdata(self): return self.datasource.get_userdata() - + def get_instance_id(self): return self.datasource.get_instance_id() diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 74e2f275..02e32462 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -25,7 +25,7 @@ from cloudinit import log as logging LOG = logging.getLogger(__name__) -# This prefix is used to make it less +# This prefix is used to make it less # of a change that when importing # we will not find something else with the same # name in the lookup path... diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 4e8ef346..74af2a7e 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -24,7 +24,7 @@ import os from cloudinit import templater from cloudinit import util -ruby_version_default = "1.8" +RUBY_VERSION_DEFAULT = "1.8" def handle(name, cfg, cloud, log, _args): @@ -38,11 +38,11 @@ def handle(name, cfg, cloud, log, _args): # Ensure the chef directories we use exist c_dirs = [ - '/etc/chef', - '/var/log/chef', - '/var/lib/chef', - '/var/cache/chef', - '/var/backups/chef', + '/etc/chef', + '/var/log/chef', + '/var/lib/chef', + '/var/cache/chef', + '/var/backups/chef', '/var/run/chef', ] for d in c_dirs: @@ -92,7 +92,7 @@ def handle(name, cfg, cloud, log, _args): # this will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - ruby_version_default) + RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) # and finally, run chef-client log.debug('Running chef-client') diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index c7d26029..62cca7cc 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -24,13 +24,13 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] +REJECT_CMD = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: - util.subp(reject_cmd) + util.subp(REJECT_CMD) else: log.debug(("Skipping transform named %s," " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index c257b6d0..fd59aa1e 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -28,7 +28,7 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." +FINAL_MESSAGE_DEF = ("Cloud-init v. {{version}} finished at {{timestamp}}." " Up {{uptime}} seconds.") @@ -39,21 +39,21 @@ def handle(_name, cfg, cloud, log, args): msg_in = args[0] else: msg_in = util.get_cfg_option_str(cfg, "final_message") - + if not msg_in: template_fn = cloud.get_template_filename('final_message') if template_fn: msg_in = util.load_file(template_fn) if not msg_in: - msg_in = final_message_def + msg_in = FINAL_MESSAGE_DEF uptime = util.uptime() ts = util.time_rfc2822() cver = version.version_string() try: subs = { - 'uptime': uptime, + 'uptime': uptime, 'timestamp': ts, 'version': cver, } diff --git a/cloudinit/config/cc_foo.py b/cloudinit/config/cc_foo.py index 99135704..e81e7faa 100644 --- a/cloudinit/config/cc_foo.py +++ b/cloudinit/config/cc_foo.py @@ -30,19 +30,19 @@ from cloudinit.settings import PER_INSTANCE # as well as any datasource provided configuration # c) A cloud object that can be used to access various # datasource and paths for the given distro and data provided -# by the various datasource instance types. +# by the various datasource instance types. # d) A argument list that may or may not be empty to this module. # Typically those are from module configuration where the module # is defined with some extra configuration that will eventually # be translated from yaml into arguments to this module. # 2. A optional 'frequency' that defines how often this module should be ran. -# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not -# provided PER_INSTANCE will be assumed. +# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not +# provided PER_INSTANCE will be assumed. # See settings.py for these constants. # 3. A optional 'distros' array/set/tuple that defines the known distros # this module will work with (if not all of them). This is used to write # a warning out if a module is being ran on a untested distribution for -# informational purposes. If non existent all distros are assumed and +# informational purposes. If non existent all distros are assumed and # no warning occurs. frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index 40758198..a8fb3ba7 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -26,13 +26,13 @@ from cloudinit import util frequency = PER_INSTANCE # This is a tool that cloud init provides -helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints' +HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints' def handle(name, cfg, cloud, log, _args): - if not os.path.exists(helper_tool): + if not os.path.exists(HELPER_TOOL): log.warn(("Unable to activate transform %s," - " helper tool not found at %s"), name, helper_tool) + " helper tool not found at %s"), name, HELPER_TOOL) return fp_blacklist = util.get_cfg_option_list(cfg, @@ -42,7 +42,7 @@ def handle(name, cfg, cloud, log, _args): ["ssh-dss"]) try: - cmd = [helper_tool] + cmd = [HELPER_TOOL] cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) (stdout, _stderr) = util.subp(cmd) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 29ce41b9..599276a7 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -62,7 +62,7 @@ def handle(name, cfg, cloud, log, _args): ls_cloudcfg = cfg.get("landscape", {}) if not isinstance(ls_cloudcfg, dict): - raise Exception(("'landscape' key existed in config," + raise Exception(("'landscape' key existed in config," " but not a dictionary type," " is a %s instead"), util.obj_name(ls_cloudcfg)) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 4cec6494..ba5e13ca 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -52,7 +52,7 @@ def handle(name, cfg, cloud, log, _args): # It doesn't contain any sections so just add one temporarily # Use a hash id based off the contents, # just incase of conflicts... (try to not have any...) - # This is so that an error won't occur when reading (and no + # This is so that an error won't occur when reading (and no # sections exist in the file) section_tpl = "[nullsection_%s]" attempts = 0 @@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args): # the previous server.cfg and create our new one old_fn = "%s.old" % (server_cfg_fn) util.rename(server_cfg_fn, old_fn) - # Now we got the whole file, write to disk except the section + # Now we got the whole file, write to disk except the section # we added so that config parser won't error out when trying to read. # Note below, that we've just used ConfigParser because it generally # works. Below, we remove the initial 'nullsection' header. diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 700fbc44..ab097c2a 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -24,10 +24,10 @@ import re from cloudinit import util -# shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 -shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" -shortname = re.compile(shortname_filter) -ws = re.compile("[%s]+" % (whitespace)) +# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 +SHORTNAME_FILTER = r"^[x]{0,1}[shv]d[a-z][0-9]*$" +SHORTNAME = re.compile(SHORTNAME_FILTER) +WS = re.compile("[%s]+" % (whitespace)) def is_mdname(name): @@ -55,7 +55,6 @@ def handle(_name, cfg, cloud, log, _args): if "mounts" in cfg: cfgmnt = cfg["mounts"] - for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): @@ -85,7 +84,7 @@ def handle(_name, cfg, cloud, log, _args): cfgmnt[i][0] = renamed log.debug("Mapped metadata name %s to %s", startname, renamed) else: - if shortname.match(startname): + if SHORTNAME.match(startname): renamed = "/dev/%s" % startname log.debug("Mapped shortname name %s to %s", startname, renamed) cfgmnt[i][0] = renamed @@ -171,7 +170,7 @@ def handle(_name, cfg, cloud, log, _args): fstab = util.load_file(cloud.paths.join(True, "/etc/fstab")) for line in fstab.splitlines(): try: - toks = ws.split(line) + toks = WS.split(line) if toks[3].find(comment) != -1: continue except: diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index a8752527..dcb07b66 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -26,8 +26,13 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', - 'instance_id', 'hostname'] +POST_LIST_ALL = [ + 'pub_key_dsa', + 'pub_key_rsa', + 'pub_key_ecdsa', + 'instance_id', + 'hostname' +] # phone_home: @@ -63,7 +68,7 @@ def handle(name, cfg, cloud, log, args): " is not an integer, using %s instead"), tries) if post_list == "all": - post_list = post_list_all + post_list = POST_LIST_ALL all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 5fb88bf2..5154efba 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -63,7 +63,7 @@ def handle(name, cfg, cloud, log, _args): util.ensure_dir(pp_ssl_dir, 0771) util.chownbyid(pp_ssl_dir, pwd.getpwnam('puppet').pw_uid, 0) - pp_ssl_certs = cloud.paths.join(False, + pp_ssl_certs = cloud.paths.join(False, '/var/lib/puppet/ssl/certs/') util.ensure_dir(pp_ssl_certs) util.chownbyid(pp_ssl_certs, diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 1690094a..c019989e 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -27,7 +27,7 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -resize_fs_prefixes_cmds = [ +RESIZE_FS_PREFIXES_CMDS = [ ('ext', 'resize2fs'), ('xfs', 'xfs_growfs'), ] @@ -89,16 +89,16 @@ def handle(name, cfg, cloud, log, args): # occurs this temporary file will still benefit from # auto deletion tfh.unlink_now() - + st_dev = nodeify_path(devpth, resize_what, log) fs_type = get_fs_type(st_dev, devpth, log) if not fs_type: log.warn("Could not determine filesystem type of %s", resize_what) return - + resizer = None fstype_lc = fs_type.lower() - for (pfix, root_cmd) in resize_fs_prefixes_cmds: + for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS: if fstype_lc.startswith(pfix): resizer = root_cmd break @@ -112,7 +112,7 @@ def handle(name, cfg, cloud, log, args): resize_cmd = [resizer, devpth] if resize_root == "noblock": - # Fork to a child that will run + # Fork to a child that will run # the resize command util.fork_cb(do_resize, resize_cmd, log) # Don't delete the file now in the parent diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 16f5286d..986e6db6 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -32,7 +32,7 @@ def handle(name, cfg, cloud, log, _args): # Start by installing the salt package ... cloud.distro.install_packages(["salt"]) - + # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') config_dir = cloud.paths.join(False, config_dir) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 364e1d02..d3c47442 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -26,16 +26,16 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -script_subdir = 'per-boot' +SCRIPT_SUBDIR = 'per-boot' def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index d75ab47d..8e428ac2 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -26,16 +26,16 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -script_subdir = 'per-instance' +SCRIPT_SUBDIR = 'per-instance' def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index 80f8c325..e7a29a44 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -26,16 +26,16 @@ from cloudinit.settings import PER_ONCE frequency = PER_ONCE -script_subdir = 'per-once' +SCRIPT_SUBDIR = 'per-once' def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index f4fe3a2a..1ff05aae 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -26,17 +26,17 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -script_subdir = 'scripts' +SCRIPT_SUBDIR = 'scripts' def handle(name, _cfg, cloud, log, _args): # This is written to by the user data handlers # Ie, any custom shell scripts that come down # go here... - runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir) + runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index e7049f22..ce17f357 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -25,7 +25,7 @@ from cloudinit import util from string import letters, digits # pylint: disable=W0402 # We are removing certain 'painful' letters/numbers -pw_set = (letters.translate(None, 'loLOI') + +PW_SET = (letters.translate(None, 'loLOI') + digits.translate(None, '01')) @@ -148,4 +148,4 @@ def handle(_name, cfg, cloud, log, args): def rand_user_password(pwlen=9): - return util.rand_str(pwlen, select_from=pw_set) + return util.rand_str(pwlen, select_from=PW_SET) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index e5e99560..4019ae90 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -24,11 +24,11 @@ import glob from cloudinit import util from cloudinit import ssh_util -DISABLE_ROOT_OPTS = ( "no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " +DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " "rather than the user \\\"root\\\".\';echo;sleep 10\"") -key2file = { +KEY_2_FILE = { "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), @@ -37,15 +37,17 @@ key2file = { "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), } -priv2pub = { - 'rsa_private': 'rsa_public', +PRIV_2_PUB = { + 'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', 'ecdsa_private': 'ecdsa_public', } -key_gen_tpl = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' +KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' -generate_keys = ['rsa', 'dsa', 'ecdsa'] +GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa'] + +KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' def handle(_name, cfg, cloud, log, _args): @@ -58,21 +60,21 @@ def handle(_name, cfg, cloud, log, _args): util.del_file(f) except: util.logexc(log, "Failed deleting key file %s", f) - + if "ssh_keys" in cfg: # if there are keys in cloud-config, use them for (key, val) in cfg["ssh_keys"].iteritems(): - if key in key2file: - tgt_fn = key2file[key][0] - tgt_perms = key2file[key][1] + if key in KEY_2_FILE: + tgt_fn = KEY_2_FILE[key][0] + tgt_perms = KEY_2_FILE[key][1] util.write_file(cloud.paths.join(False, tgt_fn), val, tgt_perms) - for (priv, pub) in priv2pub.iteritems(): + for (priv, pub) in PRIV_2_PUB.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue - pair = (key2file[priv][0], key2file[pub][0]) - cmd = ['sh', '-xc', key_gen_tpl % pair] + pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) + cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: # TODO: Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): @@ -84,12 +86,11 @@ def handle(_name, cfg, cloud, log, _args): else: # if not, generate them genkeys = util.get_cfg_option_list(cfg, - 'ssh_genkeytypes', - generate_keys) + 'ssh_genkeytypes', + GENERATE_KEY_NAMES) for keytype in genkeys: - keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype) - keyfile = cloud.paths.join(False, keyfile) - util.ensure_dir(os.path.dirname(keyfile)) + keyfile = cloud.paths.join(False, KEY_FILE_TPL % (keytype)) + util.ensure_dir(os.path.dirname(keyfile)) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 45dd85ec..25a60c52 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -157,4 +157,3 @@ def fetch(distro_name, mods=(__name__, )): % (distro_name)) distro_cls = getattr(mod, 'Distro') return distro_cls - diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index b67ae5b8..5cbefa6e 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -35,7 +35,7 @@ class Distro(distros.Distro): def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) - + def install_packages(self, pkglist): self.package_command('install', pkglist) @@ -210,12 +210,12 @@ class Distro(distros.Distro): def package_command(self, command, args=None): cmd = ['yum'] # If enabled, then yum will be tolerant of errors on the command line - # with regard to packages. - # For example: if you request to install foo, bar and baz and baz is + # with regard to packages. + # For example: if you request to install foo, bar and baz and baz is # installed; yum won't error out complaining that baz is already - # installed. + # installed. cmd.append("-t") - # Determines whether or not yum prompts for confirmation + # Determines whether or not yum prompts for confirmation # of critical actions. We don't want to prompt... cmd.append("-y") cmd.append(command) @@ -223,8 +223,8 @@ class Distro(distros.Distro): cmd.extend(args) # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, capture=False) - - + + # This is a util function to translate a ubuntu /etc/network/interfaces 'blob' # to a rhel equiv. that can then be written to /etc/sysconfig/network-scripts/ # TODO remove when we have python-netcf active... diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 5a1b572e..fd7b7b8d 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -36,11 +36,11 @@ class Distro(distros.Distro): def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain + # This will be used to restrict certain # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) - + def install_packages(self, pkglist): self._update_package_sources() self.package_command('install', pkglist) @@ -131,4 +131,4 @@ class Distro(distros.Distro): def _update_package_sources(self): self._runner.run("update-sources", self.package_command, - ["update"], freq=PER_INSTANCE) \ No newline at end of file + ["update"], freq=PER_INSTANCE) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index c6f2119c..d52b1cba 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -104,7 +104,7 @@ def run_part(mod, data, ctype, filename, payload, frequency): except: mod_ver = 1 try: - LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", + LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) if mod_ver >= 2: # Treat as v. 2 which does get a frequency @@ -114,7 +114,7 @@ def run_part(mod, data, ctype, filename, payload, frequency): mod.handle_part(data, ctype, filename, payload) except: util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" - " with frequency %s"), + " with frequency %s"), mod, ctype, filename, mod_ver, frequency) @@ -178,7 +178,7 @@ def walker_callback(pdata, ctype, filename, payload): payload, pdata['frequency']) -# Callback is a function that will be called with +# Callback is a function that will be called with # (data, content_type, filename, payload) def walk(msg, callback, data): partnum = 0 @@ -226,5 +226,3 @@ def type_from_starts_with(payload, default=None): if payload_lc.startswith(text): return INCLUSION_TYPES_MAP[text] return default - - diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 45633e0f..4447d1ee 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -84,7 +84,7 @@ class FileSemaphores(object): try: util.del_dir(self.sem_path) except (IOError, OSError): - util.logexc(LOG, "Failed deleting semaphore directory %s", + util.logexc(LOG, "Failed deleting semaphore directory %s", self.sem_path) def _acquire(self, name, freq): @@ -212,7 +212,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = self.join(False, - path_cfgs.get('cloud_dir', + path_cfgs.get('cloud_dir', '/var/lib/cloud')) self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") @@ -237,7 +237,7 @@ class Paths(object): # Set when a datasource becomes active self.datasource = ds - # joins the paths but also appends a read + # joins the paths but also appends a read # or write root if available def join(self, read_only, *paths): if read_only: diff --git a/cloudinit/log.py b/cloudinit/log.py index 478946f8..fc1428a2 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -20,7 +20,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import logging import logging.handlers import logging.config @@ -53,7 +52,6 @@ def setupBasicLogging(): root.setLevel(DEBUG) - def setupLogging(cfg=None): # See if the config provides any logging conf... if not cfg: diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 8a1eaeb3..fac9b862 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -47,7 +47,7 @@ CFG_BUILTIN = { 'paths': { 'cloud_dir': '/var/lib/cloud', 'templates_dir': '/etc/cloud/templates/', - }, + }, 'distro': 'ubuntu', }, } diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index b1817654..83c577e6 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -121,7 +121,7 @@ class DataSourceCloudStack(sources.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - LOG.debug("Crawl of metadata service took %s seconds", + LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True except Exception: diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 5da1ffea..9905dad4 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -36,7 +36,7 @@ CFG_DRIVE_FILES = [ "meta.js", ] DEFAULT_METADATA = { - "instance-id": DEFAULT_IID, + "instance-id": DEFAULT_IID, "dsmode": DEFAULT_MODE, } CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 55447102..0598dfa2 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -38,7 +38,7 @@ DEF_MD_URL = "http://169.254.169.254" DEF_MD_VERSION = '2009-04-04' # Default metadata urls that will be used if none are provided -# They will be checked for 'resolveability' and some of the +# They will be checked for 'resolveability' and some of the # following may be discarded if they do not resolve DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"] @@ -69,7 +69,7 @@ class DataSourceEc2(sources.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - LOG.debug("Crawl of metadata service took %s seconds", + LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True except Exception: @@ -201,7 +201,7 @@ class DataSourceEc2(sources.DataSource): return None # Example: - # 'block-device-mapping': + # 'block-device-mapping': # {'ami': '/dev/sda1', # 'ephemeral0': '/dev/sdb', # 'root': '/dev/sda1'} diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index bb8fbac1..104e7a54 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -251,6 +251,7 @@ datasources = [ (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] + # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 2b016d1c..8499a97c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -154,7 +154,7 @@ class DataSourceNoCloud(sources.DataSource): (self.dsmode in ("local", seeded_interfaces))): LOG.info("Updating network interfaces from %s", self) self.distro.apply_network(md['network-interfaces']) - + if md['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = md diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index fef3d40f..45dd5535 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -103,10 +103,10 @@ class AuthKeyLineParser(object): elif curc == '"': quoted = not quoted i = i + 1 - + options = ent[0:i] options_lst = [] - + # Now use a csv parser to pull the options # out of the above string that we just found an endpoint for. # @@ -211,7 +211,6 @@ def update_authorized_keys(fname, keys): def setup_user_keys(keys, user, key_prefix, paths): - # Make sure the users .ssh dir is setup accordingly pwent = pwd.getpwnam(user) ssh_dir = os.path.join(pwent.pw_dir, '.ssh') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ae6e2de5..84a965c2 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -287,7 +287,7 @@ class Init(object): def cloudify(self): # Form the needed options to cloudify our members - return cloud.Cloud(self.datasource, + return cloud.Cloud(self.datasource, self.paths, self.cfg, self.distro, helpers.Runners(self.paths)) @@ -318,7 +318,7 @@ class Init(object): def consume(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") idir = self.paths.get_ipath("handlers") - + # Add the path to the plugins dir to the top of our list for import # instance dir should be read before cloud-dir if cdir and cdir not in sys.path: @@ -417,7 +417,7 @@ class Modules(object): except: util.logexc(LOG, ("Failed loading of datasource" " config object from %s"), self.datasource) - + if self.base_cfg: t_cfgs.append(self.base_cfg) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 1c583eba..223278ce 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -47,11 +47,11 @@ class UrlResponse(object): @property def contents(self): return self._contents - + @property def headers(self): return self._headers - + def __str__(self): if not self.contents: return '' @@ -66,7 +66,7 @@ class UrlResponse(object): return True else: return False - + def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None): @@ -89,8 +89,8 @@ def readurl(url, data=None, timeout=None, excepts = [] LOG.info(("Attempting to open '%s' with %s attempts" - " (%s retries, timeout=%s) to be performed"), - url, attempts, retries, timeout) + " (%s retries, timeout=%s) to be performed"), + url, attempts, retries, timeout) open_args = {} if timeout is not None: open_args['timeout'] = int(timeout) @@ -112,7 +112,7 @@ def readurl(url, data=None, timeout=None, excepts.append(e) except urllib2.URLError as e: # This can be a message string or - # another exception instance + # another exception instance # (socket.error for remote URLs, OSError for local URLs). if (isinstance(e.reason, (OSError)) and e.reason.errno == errno.ENOENT): @@ -128,7 +128,7 @@ def readurl(url, data=None, timeout=None, # Didn't work out LOG.warn("Failed reading from %s after %s attempts", url, attempts) - + # It must of errored at least once for code # to get here so re-raise the last error LOG.debug("%s errors occured, re-raising the last one", len(excepts)) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index b7902d44..4babb8e5 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -65,33 +65,33 @@ class UserDataProcessor(object): # multipart/* are just containers if part.get_content_maintype() == 'multipart': continue - + ctype = None ctype_orig = part.get_content_type() payload = part.get_payload(decode=True) - + if not ctype_orig: ctype_orig = UNDEF_TYPE - + if ctype_orig in TYPE_NEEDED: ctype = handlers.type_from_starts_with(payload) - + if ctype is None: ctype = ctype_orig - + if ctype in INCLUDE_TYPES: self._do_include(payload, append_msg) continue - + if ctype in ARCHIVE_TYPES: self._explode_archive(payload, append_msg) continue - + if 'Content-Type' in base_msg: base_msg.replace_header('Content-Type', ctype) else: base_msg['Content-Type'] = ctype - + self._attach_part(append_msg, part) def _get_include_once_filename(self, entry): @@ -108,8 +108,8 @@ class UserDataProcessor(object): lc_line = line.lower() if lc_line.startswith("#include-once"): line = line[len("#include-once"):].lstrip() - # Every following include will now - # not be refetched.... but will be + # Every following include will now + # not be refetched.... but will be # re-read from a local urlcache (if it worked) include_once_on = True elif lc_line.startswith("#include"): @@ -190,10 +190,10 @@ class UserDataProcessor(object): """ if ATTACHMENT_FIELD not in outer_msg: outer_msg[ATTACHMENT_FIELD] = '0' - + if new_count is not None: outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) - + fetched_count = 0 try: fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) @@ -234,7 +234,3 @@ def convert_string(raw_data, headers=None): msg = MIMEBase(maintype, subtype, *headers) msg.set_payload(data) return msg - - - - diff --git a/cloudinit/util.py b/cloudinit/util.py index 91d20a76..56c01fab 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -35,7 +35,7 @@ import pwd import random import shutil import socket -import string # pylint: disable=W0402 +import string # pylint: disable=W0402 import subprocess import sys import tempfile @@ -153,13 +153,15 @@ def SilentTemporaryFile(**kwargs): # file to unlink has been unlinked elsewhere.. LOG.debug("Created temporary file %s", fh.name) fh.unlink = del_file - # Add a new method that will unlink + + # Add a new method that will unlink # right 'now' but still lets the exit # method attempt to remove it (which will # not throw due to our del file being quiet # about files that are not there) def unlink_now(): fh.unlink(fh.name) + setattr(fh, 'unlink_now', unlink_now) return fh @@ -199,7 +201,7 @@ def is_false_str(val, addons=None): def translate_bool(val, addons=None): if not val: - # This handles empty lists and false and + # This handles empty lists and false and # other things that python believes are false return False # If its already a boolean skip @@ -214,7 +216,6 @@ def rand_str(strlen=32, select_from=None): return "".join([random.choice(select_from) for _x in range(0, strlen)]) - def read_conf(fname): try: return load_yaml(load_file(fname), default={}) @@ -275,7 +276,7 @@ def is_ipv4(instr): def merge_base_cfg(cfgfile, cfg_builtin=None): syscfg = read_conf_with_confd(cfgfile) - + kern_contents = read_cc_from_cmdline() kerncfg = {} if kern_contents: @@ -575,7 +576,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): try: blob = str(blob) LOG.debug(("Attempting to load yaml from string " - "of length %s with allowed root types %s"), + "of length %s with allowed root types %s"), len(blob), allowed) converted = yaml.load(blob) if not isinstance(converted, allowed): @@ -625,7 +626,7 @@ def read_conf_d(confd): # remove anything not ending in '.cfg' confs = [f for f in confs if f.endswith(".cfg")] - + # remove anything not a file confs = [f for f in confs if os.path.isfile(os.path.join(confd, f))] @@ -726,9 +727,9 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): """ For each host a single line should be present with the following information: - - IP_address canonical_hostname [aliases...] - + + IP_address canonical_hostname [aliases...] + Fields of the entry are separated by any number of blanks and/or tab characters. Text from a "#" character until the end of the line is a comment, and is ignored. Host names may contain only alphanumeric @@ -747,7 +748,7 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): if not line: continue - # If there there is less than 3 entries + # If there there is less than 3 entries # (IP_address, canonical_hostname, alias) # then ignore this line toks = line.split() @@ -829,7 +830,7 @@ def close_stdin(): os.dup2(fp.fileno(), sys.stdin.fileno()) -def find_devs_with(criteria=None, oformat='device', +def find_devs_with(criteria=None, oformat='device', tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) @@ -841,23 +842,23 @@ def find_devs_with(criteria=None, oformat='device', blk_id_cmd = ['blkid'] options = [] if criteria: - # Search for block devices with tokens named NAME that + # Search for block devices with tokens named NAME that # have the value 'value' and display any devices which are found. # Common values for NAME include TYPE, LABEL, and UUID. # If there are no devices specified on the command line, - # all block devices will be searched; otherwise, + # all block devices will be searched; otherwise, # only search the devices specified by the user. options.append("-t%s" % (criteria)) if tag: # For each (specified) device, show only the tags that match tag. options.append("-s%s" % (tag)) if no_cache: - # If you want to start with a clean cache - # (i.e. don't report devices previously scanned + # If you want to start with a clean cache + # (i.e. don't report devices previously scanned # but not necessarily available at this time), specify /dev/null. options.extend(["-c", "/dev/null"]) if oformat: - # Display blkid's output using the specified format. + # Display blkid's output using the specified format. # The format parameter may be: # full, value, list, device, udev, export options.append('-o%s' % (oformat)) @@ -1104,7 +1105,7 @@ def mounts(): (dev, mp, fstype, opts, _freq, _passno) = mpline.split() except: continue - # If the name of the mount point contains spaces these + # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. mp = mp.replace("\\040", " ") mounted[dev] = { -- cgit v1.2.3 From 95e0fa29af3656c1011c41ab0f35dc4e9317269c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 23:40:00 -0700 Subject: 1. Add an importer function that will search for a given module in a set of search module 'prefixes' that also has a potential set of required attributes. 2. Use this new importer to find the distro class, the userdata handler modules, the config modules and the datasource modules, if none can be found error out accordingly. --- cloudinit/config/__init__.py | 4 ---- cloudinit/distros/__init__.py | 23 ++++++++++------------ cloudinit/handlers/__init__.py | 8 -------- cloudinit/importer.py | 44 +++++++++++++++++++++++++++++++++--------- cloudinit/sources/__init__.py | 34 +++++++++++--------------------- cloudinit/stages.py | 22 +++++++++++++++++---- 6 files changed, 74 insertions(+), 61 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 02e32462..ab13045f 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -51,10 +51,6 @@ def fixup_module(mod, def_freq=PER_INSTANCE): freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'handle'): - def empty_handle(_name, _cfg, _cloud, _log, _args): - pass - setattr(mod, 'handle', empty_handle) if not hasattr(mod, 'distros'): setattr(mod, 'distros', None) return mod diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 25a60c52..e0ef6ee0 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -144,16 +144,13 @@ class Distro(object): return False -def fetch(distro_name, mods=(__name__, )): - mod = None - for m in mods: - mod_name = "%s.%s" % (m, distro_name) - try: - mod = importer.import_module(mod_name) - except ImportError: - pass - if not mod: - raise RuntimeError("No distribution found for distro %s" - % (distro_name)) - distro_cls = getattr(mod, 'Distro') - return distro_cls +def fetch(name): + locs = importer.find_module(name, + ['', __name__], + ['Distro']) + if not locs: + raise ImportError("No distribution found for distro %s" + % (name)) + mod = importer.import_module(locs[0]) + cls = getattr(mod, 'Distro') + return cls diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d52b1cba..0f7432e5 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -202,20 +202,12 @@ def walk(msg, callback, data): def fixup_handler(mod, def_freq=PER_INSTANCE): if not hasattr(mod, "handler_version"): setattr(mod, "handler_version", 1) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) if not hasattr(mod, 'frequency'): setattr(mod, 'frequency', def_freq) else: freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warn("Handler %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'handle_part'): - def empty_handler(_data, _ctype, _filename, _payload): - pass - setattr(mod, 'handle_part', empty_handler) return mod diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a36b87bc..71cf2726 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -23,17 +23,43 @@ import sys from cloudinit import log as logging -from cloudinit import util LOG = logging.getLogger(__name__) -# Simple wrapper that allows us to add more logging in... def import_module(module_name): - try: - LOG.debug("Attempting to import module %s", module_name) - __import__(module_name) - return sys.modules[module_name] - except: - util.logexc(LOG, 'Failed at importing %s', module_name) - raise + __import__(module_name) + return sys.modules[module_name] + + +def find_module(base_name, search_paths, required_attrs=None): + found_places = [] + if not required_attrs: + required_attrs = [] + real_paths = [] + for path in search_paths: + real_path = [] + if path: + real_path.extend(path.split(".")) + real_path.append(base_name) + full_path = '.'.join(real_path) + real_paths.append(full_path) + LOG.debug("Looking for modules %s that have attributes %s", + real_paths, required_attrs) + for full_path in real_paths: + mod = None + try: + mod = import_module(full_path) + except ImportError: + pass + if not mod: + continue + found_attrs = 0 + for attr in required_attrs: + if hasattr(mod, attr): + found_attrs += 1 + if found_attrs == len(required_attrs): + found_places.append(full_path) + LOG.debug("Found %s with attributes %s in %s", base_name, + required_attrs, found_places) + return found_places diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 8ab7cf54..42e924b0 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -191,31 +191,19 @@ def list_sources(cfg_list, depends, pkg_list): LOG.info(("Looking for for data source in: %s," " via packages %s that matches dependencies %s"), cfg_list, pkg_list, depends) - for ds_coll in cfg_list: - ds_name = str(ds_coll) + for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) - for pkg in pkg_list: - pkg_name = [] - if pkg: - # Any package name given, this affects - # the lookup path - pkg_name.append(str(pkg)) - pkg_name.append(ds_name) - try: - mod = importer.import_module(".".join(pkg_name)) - except ImportError: - continue - lister = getattr(mod, "get_datasource_list", None) - if not lister: - continue - cls_matches = lister(depends) - if not cls_matches: - continue - src_list.extend(cls_matches) - LOG.debug(("Found a match" - " in %s with matches %s"), mod, cls_matches) - break + m_locs = importer.find_module(ds_name, + pkg_list, + ['get_datasource_list']) + for m_loc in m_locs: + mod = importer.import_module(m_loc) + lister = getattr(mod, "get_datasource_list") + matches = lister(depends) + if matches: + src_list.extend(matches) + break return src_list diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 84a965c2..1997301a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -205,7 +205,7 @@ class Init(object): # Any config provided??? pkg_list = self.cfg.get('datasource_pkg_list') or [] # Add the defaults at the end - for n in [util.obj_name(sources), '']: + for n in ['', util.obj_name(sources)]: if n not in pkg_list: pkg_list.append(n) cfg_list = self.cfg.get('datasource_list') or [] @@ -334,9 +334,17 @@ class Init(object): # Add handlers in cdir potential_handlers = util.find_modules(cdir) - for (fname, modname) in potential_handlers.iteritems(): + for (fname, mod_name) in potential_handlers.iteritems(): try: - mod = handlers.fixup_handler(importer.import_module(modname)) + mod_locs = importer.find_module(mod_name, [''], + ['list_types', + 'handle_part']) + if not mod_locs: + LOG.warn(("Could not find a valid user-data handler" + " named %s in file %s"), mod_name, fname) + continue + mod = importer.import_module(mod_locs[0]) + mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) LOG.debug("Added handler for %s from %s", types, fname) except: @@ -482,7 +490,13 @@ class Modules(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = config.fixup_module(importer.import_module(mod_name)) + mod_locs = importer.find_module(mod_name, + ['', util.obj_name(config)], + ['handle']) + if not mod_locs: + LOG.warn("Could not find module named %s", mod_name) + continue + mod = config.fixup_module(importer.import_module(mod_locs[0])) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods -- cgit v1.2.3 From 549bdae7cb150a0d2465cf9931c3a0aec304764a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 21 Jun 2012 12:33:53 -0700 Subject: Remove a unused import. --- cloudinit/handlers/__init__.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 0f7432e5..94cb699e 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -27,7 +27,6 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) from cloudinit import importer from cloudinit import log as logging -from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) -- cgit v1.2.3 From 155fa982b895b70572857ab25a324809470d74b3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 21 Jun 2012 17:15:45 -0700 Subject: 1. Only register and increment the handler count after we have ensured it imports and call_begin passes. 2. Fixup the 'test__init__.py' file to now be working again. --- cloudinit/handlers/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 94cb699e..20a822bd 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -135,12 +135,15 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): modfname = "%s.py" % (modfname) # TODO: Check if path exists?? util.write_file(modfname, payload, 0600) - pdata['handlercount'] = curcount + 1 handlers = pdata['handlers'] try: mod = fixup_handler(importer.import_module(modname)) - handlers.register(mod) call_begin(mod, pdata['data'], frequency) + # Only register and increment + # after the above have worked (so we don't if it + # fails) + handlers.register(mod) + pdata['handlercount'] = curcount + 1 except: util.logexc(LOG, ("Failed at registering python file: %s" " (part handler %s)"), modfname, curcount) -- cgit v1.2.3 From bb75af01d9bf802985e21d0b8d42f0855e57953f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 29 Jun 2012 11:41:43 -0700 Subject: Ensure that only when incoming frequency is per-instance that this runs. --- cloudinit/handlers/upstart_job.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 411a5d68..99e0afde 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -46,6 +46,10 @@ class UpstartJobPartHandler(handlers.Handler): if ctype in handlers.CONTENT_SIGNALS: return + # See: https://bugs.launchpad.net/bugs/819507 + if frequency != PER_INSTANCE: + return + if not self.upstart_dir: return -- cgit v1.2.3