From f7bfe4aaae9850ab179a39436d4b6a9c9da707a5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 16:39:09 -0700 Subject: Renamed back to 'cc_*' with the reasoning being that 'cc_' provides some protection against module name collisions when importing. --- cloudinit/config/cc_apt_update_upgrade.py | 243 ++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 cloudinit/config/cc_apt_update_upgrade.py (limited to 'cloudinit/config/cc_apt_update_upgrade.py') diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py new file mode 100644 index 00000000..f5b4b58f --- /dev/null +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -0,0 +1,243 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2009-2010 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import glob +import os + +from cloudinit import templater +from cloudinit import util + +distros = ['ubuntu', 'debian'] + +PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" + + +def handle(_name, cfg, cloud, log, _args): + update = util.get_cfg_option_bool(cfg, 'apt_update', False) + upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) + + release = get_release() + + mirror = find_apt_mirror(cloud, cfg) + + log.debug("Selected mirror at: %s" % mirror) + + if not util.get_cfg_option_bool(cfg, + 'apt_preserve_sources_list', False): + generate_sources_list(release, mirror, cloud, log) + old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', + "archive.ubuntu.com/ubuntu") + rename_apt_lists(old_mir, mirror) + + # Set up any apt proxy + proxy = cfg.get("apt_proxy", None) + proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + if proxy: + try: + # See man 'apt.conf' + contents = PROXY_TPL % (proxy) + util.write_file(cloud.paths.join(False, proxy_filename), + contents) + except Exception as e: + util.logexc(log, "Failed to write proxy to %s", proxy_filename) + elif os.path.isfile(proxy_filename): + util.del_file(proxy_filename) + + # Process 'apt_sources' + if 'apt_sources' in cfg: + errors = add_sources(cloud, cfg['apt_sources'], + {'MIRROR': mirror, 'RELEASE': release}) + for e in errors: + log.warn("Source Error: %s", ':'.join(e)) + + dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) + if dconf_sel: + log.debug("setting debconf selections per cloud config") + try: + util.subp(('debconf-set-selections', '-'), dconf_sel) + except: + util.logexc(log, "Failed to run debconf-set-selections") + + pkglist = util.get_cfg_option_list(cfg, 'packages', []) + + errors = [] + if update or len(pkglist) or upgrade: + try: + cloud.distro.update_package_sources() + except Exception as e: + util.logexc(log, "Package update failed") + errors.append(e) + + if upgrade: + try: + cloud.distro.package_command("upgrade") + except Exception as e: + util.logexc(log, "Package upgrade failed") + errors.append(e) + + if len(pkglist): + try: + cloud.distro.install_packages(pkglist) + except Exception as e: + util.logexc(log, "Failed to install packages: %s ", pkglist) + errors.append(e) + + if len(errors): + log.warn("%s failed with exceptions, re-raising the last one", + len(errors)) + raise errors[-1] + + +def mirror2lists_fileprefix(mirror): + string = mirror + # take of http:// or ftp:// + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): + oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) + nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + if oprefix == nprefix: + return + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + # TODO use the cloud.paths.join... + util.rename(filename, "%s%s" % (nprefix, filename[olen:])) + + +def get_release(): + (stdout, _stderr) = util.subp(['lsb_release', '-cs']) + return stdout.strip() + + +def generate_sources_list(codename, mirror, cloud, log): + template_fn = cloud.get_template_filename('sources.list') + if template_fn: + params = {'mirror': mirror, 'codename': codename} + out_fn = cloud.paths.join(False, '/etc/apt/sources.list') + templater.render_to_file(template_fn, out_fn, params) + else: + log.warn("No template found, not rendering /etc/apt/sources.list") + + +def add_sources(cloud, srclist, template_params=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + errorlist = [] + for ent in srclist: + if 'source' not in ent: + errorlist.append(["", "missing source"]) + continue + + source = ent['source'] + if source.startswith("ppa:"): + try: + util.subp(["add-apt-repository", source]) + except: + errorlist.append([source, "add-apt-repository failed"]) + continue + + source = templater.render_string(source, template_params) + + if 'filename' not in ent: + ent['filename'] = 'cloud_config_sources.list' + + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + + if ('keyid' in ent and 'key' not in ent): + ks = "keyserver.ubuntu.com" + if 'keyserver' in ent: + ks = ent['keyserver'] + try: + ent['key'] = util.getkeybyid(ent['keyid'], ks) + except: + errorlist.append([source, "failed to get key from %s" % ks]) + continue + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + errorlist.append([source, "failed add key"]) + + try: + contents = "%s\n" % (source) + util.write_file(cloud.paths.join(False, ent['filename']), + contents, omode="ab") + except: + errorlist.append([source, + "failed write to file %s" % ent['filename']]) + + return errorlist + + +def find_apt_mirror(cloud, cfg): + """ find an apt_mirror given the cloud and cfg provided """ + + mirror = None + + cfg_mirror = cfg.get("apt_mirror", None) + if cfg_mirror: + mirror = cfg["apt_mirror"] + elif "apt_mirror_search" in cfg: + mirror = util.search_for_mirror(cfg['apt_mirror_search']) + else: + mirror = cloud.get_local_mirror() + + mydom = "" + + doms = [] + + if not mirror: + # if we have a fqdn, then search its domain portion first + (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + mydom = ".".join(fqdn.split(".")[1:]) + if mydom: + doms.append(".%s" % mydom) + + if not mirror: + doms.extend((".localdomain", "",)) + + mirror_list = [] + distro = cloud.distro.name + mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + for post in doms: + mirror_list.append(mirrorfmt % (post)) + + mirror = util.search_for_mirror(mirror_list) + + if not mirror: + mirror = cloud.distro.get_package_mirror() + + return mirror -- cgit v1.2.3 From aff1783ff4c6b5afdbf42d474f3ab270b64b3d14 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 22 Jun 2012 20:53:35 -0700 Subject: Use a module level constant for the filename --- cloudinit/config/cc_apt_update_upgrade.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config/cc_apt_update_upgrade.py') diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py index f5b4b58f..6719da52 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -27,6 +27,7 @@ from cloudinit import util distros = ['ubuntu', 'debian'] PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" +PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" def handle(_name, cfg, cloud, log, _args): @@ -48,7 +49,7 @@ def handle(_name, cfg, cloud, log, _args): # Set up any apt proxy proxy = cfg.get("apt_proxy", None) - proxy_filename = "/etc/apt/apt.conf.d/95cloud-init-proxy" + proxy_filename = PROXY_FN if proxy: try: # See man 'apt.conf' -- cgit v1.2.3 From f8413af9168adc0ad7c730b9adea9eba67949ba5 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 22 Jun 2012 23:26:50 -0700 Subject: 1. Move the getkeybyid function back here but add some slight adjustments a. Instead of executing a bash string, write out a temporary file and then just execute '/bin/sh' on that file with the right arguments instead. 2. Rename util.SilentTemporaryFile to util.ExtendedTemporaryFile and update the usages of the previous name accordingly, this better reflects what this temp file is. 3. More teenie pep8 line length fixings --- cloudinit/config/cc_apt_update_upgrade.py | 27 ++++++++++++++++++++++++++- cloudinit/config/cc_bootcmd.py | 3 +-- cloudinit/config/cc_chef.py | 2 +- cloudinit/config/cc_resizefs.py | 4 ++-- cloudinit/util.py | 22 +--------------------- 5 files changed, 31 insertions(+), 27 deletions(-) (limited to 'cloudinit/config/cc_apt_update_upgrade.py') diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py index 6719da52..8ecd9c94 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -29,6 +29,21 @@ distros = ['ubuntu', 'debian'] PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" +# A temporary shell program to get a given gpg key +# from a given keyserver +EXPORT_GPG_KEYID = """ + k=${1} ks=${2}; + exec 2>/dev/null + [ -n "$k" ] || exit 1; + armour=$(gpg --list-keys --armour "${k}") + if [ -z "${armour}" ]; then + gpg --keyserver ${ks} --recv $k >/dev/null && + armour=$(gpg --export --armour "${k}") && + gpg --batch --yes --delete-keys "${k}" + fi + [ -n "${armour}" ] && echo "${armour}" +""" + def handle(_name, cfg, cloud, log, _args): update = util.get_cfg_option_bool(cfg, 'apt_update', False) @@ -106,6 +121,16 @@ def handle(_name, cfg, cloud, log, _args): raise errors[-1] +# get gpg keyid from keyserver +def getkeybyid(keyid, keyserver): + with util.ExtendedTemporaryFile(suffix='.sh') as fh: + fh.write(EXPORT_GPG_KEYID) + fh.flush() + cmd = ['/bin/sh', fh.name, keyid, keyserver] + (stdout, _stderr) = util.subp(cmd) + return stdout.strip() + + def mirror2lists_fileprefix(mirror): string = mirror # take of http:// or ftp:// @@ -181,7 +206,7 @@ def add_sources(cloud, srclist, template_params=None): if 'keyserver' in ent: ks = ent['keyserver'] try: - ent['key'] = util.getkeybyid(ent['keyid'], ks) + ent['key'] = getkeybyid(ent['keyid'], ks) except: errorlist.append([source, "failed to get key from %s" % ks]) continue diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 89ccf3f1..bae1ea54 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -19,7 +19,6 @@ # along with this program. If not, see . import os -import tempfile from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -34,7 +33,7 @@ def handle(name, cfg, cloud, log, _args): " no 'bootcmd' key in configuration"), name) return - with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf: + with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) tmpf.write(content) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index d682398a..6f568261 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -96,7 +96,7 @@ def handle(name, cfg, cloud, log, _args): install_chef_from_gems(cloud.distro, ruby_version, chef_version) # and finally, run chef-client log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', + util.subp(['/usr/bin/chef-client', '-d', '-i', '1800', '-s', '20'], capture=False) elif install_type == 'packages': # this will install and run the chef-client from packages diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 7e1428e9..69cd8872 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -79,8 +79,8 @@ def handle(name, cfg, cloud, log, args): # TODO: allow what is to be resized to be configurable?? resize_what = cloud.paths.join(False, "/") - with util.SilentTemporaryFile(prefix="cloudinit.resizefs.", - dir=resize_root_d, delete=True) as tfh: + with util.ExtendedTemporaryFile(prefix="cloudinit.resizefs.", + dir=resize_root_d, delete=True) as tfh: devpth = tfh.name # Delete the file so that mknod will work diff --git a/cloudinit/util.py b/cloudinit/util.py index baa3def1..6cdf9ff3 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -157,7 +157,7 @@ class MountFailedError(Exception): pass -def SilentTemporaryFile(**kwargs): +def ExtendedTemporaryFile(**kwargs): fh = tempfile.NamedTemporaryFile(**kwargs) # Replace its unlink with a quiet version # that does not raise errors when the @@ -517,26 +517,6 @@ def del_dir(path): shutil.rmtree(path) -# get gpg keyid from keyserver -def getkeybyid(keyid, keyserver): - # TODO fix this... - shcmd = """ - k=${1} ks=${2}; - exec 2>/dev/null - [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") - if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && - armour=$(gpg --export --armour "${k}") && - gpg --batch --yes --delete-keys "${k}" - fi - [ -n "${armour}" ] && echo "${armour}" - """ - args = ['sh', '-c', shcmd, "export-gpg-keyid", keyid, keyserver] - (stdout, _stderr) = subp(args) - return stdout - - def runparts(dirp, skip_no_exist=True): if skip_no_exist and not os.path.isdir(dirp): return -- cgit v1.2.3 From fdbd341aae7a41d3985e544fe40375087056b1f9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 22 Jun 2012 23:36:08 -0700 Subject: Use os.path.join instead of custom string path formation --- cloudinit/config/cc_apt_update_upgrade.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/config/cc_apt_update_upgrade.py') diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py index 8ecd9c94..42b6f3e9 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -133,7 +133,7 @@ def getkeybyid(keyid, keyserver): def mirror2lists_fileprefix(mirror): string = mirror - # take of http:// or ftp:// + # take off http:// or ftp:// if string.endswith("/"): string = string[0:-1] pos = string.find("://") @@ -144,8 +144,8 @@ def mirror2lists_fileprefix(mirror): def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): - oprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(omirror)) - nprefix = "%s/%s" % (lists_d, mirror2lists_fileprefix(new_mirror)) + oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) + nprefix = os.path.join(lists_d, mirror2lists_fileprefix(new_mirror)) if oprefix == nprefix: return olen = len(oprefix) -- cgit v1.2.3 From b3975ef46bbee1c713b963f3897fce9c5d3cbe94 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 6 Jul 2012 10:47:19 -0700 Subject: Updated so that if no mirror is found, the module stops running. --- cloudinit/config/cc_apt_update_upgrade.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'cloudinit/config/cc_apt_update_upgrade.py') diff --git a/cloudinit/config/cc_apt_update_upgrade.py b/cloudinit/config/cc_apt_update_upgrade.py index 42b6f3e9..5c5e510c 100644 --- a/cloudinit/config/cc_apt_update_upgrade.py +++ b/cloudinit/config/cc_apt_update_upgrade.py @@ -45,13 +45,16 @@ EXPORT_GPG_KEYID = """ """ -def handle(_name, cfg, cloud, log, _args): +def handle(name, cfg, cloud, log, _args): update = util.get_cfg_option_bool(cfg, 'apt_update', False) upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) release = get_release() - mirror = find_apt_mirror(cloud, cfg) + if not mirror: + log.debug(("Skipping module named %s," + " no package 'mirror' located"), name) + return log.debug("Selected mirror at: %s" % mirror) -- cgit v1.2.3