diff options
61 files changed, 795 insertions, 352 deletions
@@ -6,6 +6,19 @@ - SmartOS test: do not require existance of /dev/ttyS1. [LP: #1316597] - doc: fix user-groups doc to reference plural ssh-authorized-keys (LP: #1327065) [Joern Heissler] + - fix 'make test' in python 2.6 + - support jinja2 as a templating engine. Drop the hard requirement on + cheetah. This helps in python3 effort. (LP: #1219223) + - change install path for systemd files to /lib/systemd/system + [Dimitri John Ledkov] + - change trunk debian packaging to use pybuild and drop cdbs. + [Dimitri John Ledkov] + - SeLinuxGuard: remove invalid check that looked for stat.st_mode in os.lstat. + - do not write comments in /etc/timezone (LP: #1341710) + - ubuntu: provide 'ubuntu-init-switch' module to aid in systemd testing. + - status/result json: remove 'end' entry which was always null + - systemd: make cloud-init block ssh service startup to guarantee keys + are generated. [Jordan Evans] (LP: #1333920) 0.7.5: - open 0.7.5 - Add a debug log message around import failures @@ -1,46 +0,0 @@ -- Consider a 'failsafe' DataSource - If all others fail, setting a default that - - sets the user password, writing it to console - - logs to console that this happened -- Consider a 'previous' DataSource - If no other data source is found, fall back to the 'previous' one - keep a indication of what instance id that is in /var/lib/cloud -- Rewrite "cloud-init-query" (currently not implemented) - Possibly have DataSource and cloudinit expose explicit fields - - instance-id - - hostname - - mirror - - release - - ssh public keys -- Remove the conversion of the ubuntu network interface format conversion - to a RH/fedora format and replace it with a top level format that uses - the netcf libraries format instead (which itself knows how to translate - into the specific formats) -- Replace the 'apt*' modules with variants that now use the distro classes - to perform distro independent packaging commands (where possible) -- Canonicalize the semaphore/lock name for modules and user data handlers - a. It is most likely a bug that currently exists that if a module in config - alters its name and it has already ran, then it will get ran again since - the lock name hasn't be canonicalized -- Replace some the LOG.debug calls with a LOG.info where appropriate instead - of how right now there is really only 2 levels (WARN and DEBUG) -- Remove the 'cc_' for config modules, either have them fully specified (ie - 'cloudinit.config.resizefs') or by default only look in the 'cloudinit.config' - for these modules (or have a combination of the above), this avoids having - to understand where your modules are coming from (which can be altered by - the current python inclusion path) -- Depending on if people think the wrapper around 'os.path.join' provided - by the 'paths' object is useful (allowing us to modify based off a 'read' - and 'write' configuration based 'root') or is just to confusing, it might be - something to remove later, and just recommend using 'chroot' instead (or the X - different other options which are similar to 'chroot'), which is might be more - natural and less confusing... -- Instead of just warning when a module is being ran on a 'unknown' distribution - perhaps we should not run that module in that case? Or we might want to start - reworking those modules so they will run on all distributions? Or if that is - not the case, then maybe we want to allow fully specified python paths for - modules and start encouraging packages of 'ubuntu' modules, packages of 'rhel' - specific modules that people can add instead of having them all under the - cloud-init 'root' tree? This might encourage more development of other modules - instead of having to go edit the cloud-init code to accomplish this. - diff --git a/TODO.rst b/TODO.rst new file mode 100644 index 00000000..7d126864 --- /dev/null +++ b/TODO.rst @@ -0,0 +1,43 @@ +============================================== +Things that cloud-init may do (better) someday +============================================== + +- Consider making ``failsafe`` ``DataSource`` + - sets the user password, writing it to console + +- Consider a ``previous`` ``DataSource``, if no other data source is + found, fall back to the ``previous`` one that worked. +- Rewrite ``cloud-init-query`` (currently not implemented) +- Possibly have a ``DataSource`` expose explicit fields: + + - instance-id + - hostname + - mirror + - release + - ssh public keys + +- Remove the conversion of the ubuntu network interface format conversion + to a RH/fedora format and replace it with a top level format that uses + the netcf libraries format instead (which itself knows how to translate + into the specific formats). See for example `netcf`_ which seems to be + an active project that has this capability. +- Replace the ``apt*`` modules with variants that now use the distro classes + to perform distro independent packaging commands (wherever possible). +- Replace some the LOG.debug calls with a LOG.info where appropriate instead + of how right now there is really only 2 levels (``WARN`` and ``DEBUG``) +- Remove the ``cc_`` prefix for config modules, either have them fully + specified (ie ``cloudinit.config.resizefs``) or by default only look in + the ``cloudinit.config`` namespace for these modules (or have a combination + of the above), this avoids having to understand where your modules are + coming from (which can be altered by the current python inclusion path) +- Instead of just warning when a module is being ran on a ``unknown`` + distribution perhaps we should not run that module in that case? Or we might + want to start reworking those modules so they will run on all + distributions? Or if that is not the case, then maybe we want to allow + fully specified python paths for modules and start encouraging + packages of ``ubuntu`` modules, packages of ``rhel`` specific modules that + people can add instead of having them all under the cloud-init ``root`` + tree? This might encourage more development of other modules instead of + having to go edit the cloud-init code to accomplish this. + +.. _netcf: https://fedorahosted.org/netcf/ diff --git a/bin/cloud-init b/bin/cloud-init index 6ede60af..866f8ca4 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -224,6 +224,9 @@ def main_init(name, args): LOG.debug("Exiting early due to the existence of %s files", existing_files) return (None, []) + else: + LOG.debug("Execution continuing, no previous run detected that" + " would allow us to stop early.") else: # The cache is not instance specific, so it has to be purged # but we want 'start' to benefit from a cache if @@ -475,7 +478,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): nullstatus = { 'errors': [], 'start': None, - 'end': None, + 'finished': None, } status = {'v1': {}} for m in modes: diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 8f99e887..638daef8 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -89,8 +89,9 @@ def load_power_state(cfg): mode = pstate.get("mode") if mode not in opt_map: - raise TypeError("power_state[mode] required, must be one of: %s." % - ','.join(opt_map.keys())) + raise TypeError( + "power_state[mode] required, must be one of: %s. found: '%s'." % + (','.join(opt_map.keys()), mode)) delay = pstate.get("delay", "now") # convert integer 30 or string '30' to '+30' @@ -100,7 +101,9 @@ def load_power_state(cfg): pass if delay != "now" and not re.match(r"\+[0-9]+", delay): - raise TypeError("power_state[delay] must be 'now' or '+m' (minutes).") + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % delay) args = ["shutdown", opt_map[mode], delay] if pstate.get("message"): diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py new file mode 100644 index 00000000..6f994bff --- /dev/null +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -0,0 +1,164 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Canonical Ltd. +# +# Author: Scott Moser <scott.moser@canonical.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +ubuntu_init_switch: reboot system into another init + +This provides a way for the user to boot with systemd even if the +image is set to boot with upstart. It should be run as one of the first +cloud_init_modules, and will switch the init system and then issue a reboot. +The next boot will come up in the target init system and no action will +be taken. + +This should be inert on non-ubuntu systems, and also exit quickly. + +config is comes under the top level 'init_switch' dictionary. + +#cloud-config +init_switch: + target: systemd + reboot: true + +'target' can be 'systemd' or 'upstart'. Best effort is made, but its possible +this system will break, and probably won't interact well with any other +mechanism you've used to switch the init system. + +'reboot': [default=true]. + true: reboot if a change was made. + false: do not reboot. +""" + +from cloudinit.settings import PER_INSTANCE +from cloudinit import log as logging +from cloudinit import util +from cloudinit.distros import ubuntu + +import os +import time + +frequency = PER_INSTANCE +REBOOT_CMD = ["/sbin/reboot", "--force"] + +DEFAULT_CONFIG = { + 'init_switch': {'target': None, 'reboot': True} +} + +SWITCH_INIT = """ +#!/bin/sh +# switch_init: [upstart | systemd] + +is_systemd() { + [ "$(dpkg-divert --listpackage /sbin/init)" = "systemd-sysv" ] +} +debug() { echo "$@" 1>&2; } +fail() { echo "$@" 1>&2; exit 1; } + +if [ "$1" = "systemd" ]; then + if is_systemd; then + debug "already systemd, nothing to do" + else + [ -f /lib/systemd/systemd ] || fail "no systemd available"; + dpkg-divert --package systemd-sysv --divert /sbin/init.diverted \\ + --rename /sbin/init + fi + [ -f /sbin/init ] || ln /lib/systemd/systemd /sbin/init +elif [ "$1" = "upstart" ]; then + if is_systemd; then + rm -f /sbin/init + dpkg-divert --package systemd-sysv --rename --remove /sbin/init + else + debug "already upstart, nothing to do." + fi +else + fail "Error. expect 'upstart' or 'systemd'" +fi +""" + + +def handle(name, cfg, cloud, log, args): + + if not isinstance(cloud.distro, ubuntu.Distro): + log.debug("%s: distro is '%s', not ubuntu. returning", + name, cloud.distro.__class__) + return + + cfg = util.mergemanydict([cfg, DEFAULT_CONFIG]) + target = cfg['init_switch']['target'] + reboot = cfg['init_switch']['reboot'] + + if len(args) != 0: + target = args[0] + if len(args) > 1: + reboot = util.is_true(args[1]) + + if not target: + log.debug("%s: target=%s. nothing to do", name, target) + return + + if not util.which('dpkg'): + log.warn("%s: 'dpkg' not available. Assuming not ubuntu", name) + return + + supported = ('upstart', 'systemd') + if target not in supported: + log.warn("%s: target set to %s, expected one of: %s", + name, target, str(supported)) + + if os.path.exists("/run/systemd/system"): + current = "systemd" + else: + current = "upstart" + + if current == target: + log.debug("%s: current = target = %s. nothing to do", name, target) + return + + try: + util.subp(['sh', '-s', target], data=SWITCH_INIT) + except util.ProcessExecutionError as e: + log.warn("%s: Failed to switch to init '%s'. %s", name, target, e) + return + + if util.is_false(reboot): + log.info("%s: switched '%s' to '%s'. reboot=false, not rebooting.", + name, current, target) + return + + try: + log.warn("%s: switched '%s' to '%s'. rebooting.", + name, current, target) + logging.flushLoggers(log) + _fire_reboot(log, wait_attempts=4, initial_sleep=4) + except Exception as e: + util.logexc(log, "Requested reboot did not happen!") + raise + + +def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): + util.subp(REBOOT_CMD) + start = time.time() + wait_time = initial_sleep + for _i in range(0, wait_attempts): + time.sleep(wait_time) + wait_time *= backoff + elapsed = time.time() - start + log.debug("Rebooted, but still running after %s seconds", int(elapsed)) + # If we got here, not good + elapsed = time.time() - start + raise RuntimeError(("Reboot did not happen" + " after %s seconds!") % (int(elapsed))) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 55d6bcbc..1a56dfb3 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -856,3 +856,12 @@ def fetch(name): mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls + + +def set_etc_timezone(tz, tz_file=None, tz_conf="/etc/timezone", + tz_local="/etc/localtime"): + util.write_file(tz_conf, str(tz).rstrip() + "\n") + # This ensures that the correct tz will be used for the system + if tz_local and tz_file: + util.copy(tz_file, self.tz_local_fn) + return diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 310c3dff..9f11b89c 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -32,8 +32,6 @@ LOG = logging.getLogger(__name__) class Distro(distros.Distro): locale_conf_fn = "/etc/locale.gen" network_conf_dir = "/etc/netctl" - tz_conf_fn = "/etc/timezone" - tz_local_fn = "/etc/localtime" resolve_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts @@ -161,16 +159,7 @@ class Distro(distros.Distro): return hostname def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Note: "" provides trailing newline during join - tz_lines = [ - util.make_header(), - str(tz), - "", - ] - util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1ae232fd..7cf4a9ef 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -46,8 +46,6 @@ class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" locale_conf_fn = "/etc/default/locale" network_conf_fn = "/etc/network/interfaces" - tz_conf_fn = "/etc/timezone" - tz_local_fn = "/etc/localtime" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -133,16 +131,7 @@ class Distro(distros.Distro): return "127.0.1.1" def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Note: "" provides trailing newline during join - tz_lines = [ - util.make_header(), - str(tz), - "", - ] - util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 09f8d8ea..c4b02de1 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -31,8 +31,6 @@ LOG = logging.getLogger(__name__) class Distro(distros.Distro): locale_conf_fn = "/etc/locale.gen" network_conf_fn = "/etc/conf.d/net" - tz_conf_fn = "/etc/timezone" - tz_local_fn = "/etc/localtime" init_cmd = [''] # init scripts def __init__(self, name, cfg, paths): @@ -140,16 +138,7 @@ class Distro(distros.Distro): return hostname def set_timezone(self, tz): - tz_file = self._find_tz_file(tz) - # Note: "" provides trailing newline during join - tz_lines = [ - util.make_header(), - str(tz), - "", - ] - util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + set_etc_timezone(tz=tz, tz_file=self._find_tz_file(tz)) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a094141a..a1929137 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -45,8 +45,6 @@ def find_module(base_name, search_paths, required_attrs=None): real_path.append(base_name) full_path = '.'.join(real_path) real_paths.append(full_path) - LOG.debug("Looking for modules %s that have attributes %s", - real_paths, required_attrs) for full_path in real_paths: mod = None try: @@ -62,6 +60,4 @@ def find_module(base_name, search_paths, required_attrs=None): found_attrs += 1 if found_attrs == len(required_attrs): found_places.append(full_path) - LOG.debug("Found %s with attributes %s in %s", base_name, - required_attrs, found_places) return found_places diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 0978b2c6..650b42a9 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -55,9 +55,6 @@ class UnknownMerger(object): if not meth: meth = self._handle_unknown args.insert(0, method_name) - LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'", - type_name, type_utils.obj_name(merge_with), - meth.__name__, self) return meth(*args) @@ -84,8 +81,6 @@ class LookupMerger(UnknownMerger): # First one that has that method/attr gets to be # the one that will be called meth = getattr(merger, meth_wanted) - LOG.debug(("Merging using located merger '%s'" - " since it had method '%s'"), merger, meth_wanted) break if not meth: return UnknownMerger._handle_unknown(self, meth_wanted, diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index 30b6f3b3..1bdca9f7 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -56,8 +56,8 @@ def netdev_info(empty=""): # newer (freebsd and fedora) show 'inet xx.yy' # just skip this 'inet' entry. (LP: #1285185) try: - if (toks[i] in ("inet", "inet6") and - toks[i + 1].startswith("addr:")): + if ((toks[i] in ("inet", "inet6") and + toks[i + 1].startswith("addr:"))): continue except IndexError: pass diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 08f661e4..1bbeca59 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -78,7 +78,8 @@ class DataSourceCloudStack(sources.DataSource): (max_wait, timeout) = self._get_url_settings() - urls = [self.metadata_address + "/latest/meta-data/instance-id"] + urls = [uhelp.combine_url(self.metadata_address, + 'latest/meta-data/instance-id')] start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, timeout=timeout, status_cb=LOG.warn) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 58349ffc..9e071fc4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -397,8 +397,8 @@ class Init(object): mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) if types: - LOG.debug("Added custom handler for %s from %s", - types, fname) + LOG.debug("Added custom handler for %s [%s] from %s", + types, mod, fname) except Exception: util.logexc(LOG, "Failed to register handler from %s", fname) @@ -644,6 +644,8 @@ class Modules(object): freq = mod.frequency if not freq in FREQUENCIES: freq = PER_INSTANCE + LOG.debug("Running module %s (%s) with frequency %s", + name, mod, freq) # Use the configs logger and not our own # TODO(harlowja): possibly check the module @@ -657,7 +659,7 @@ class Modules(object): run_name = "config-%s" % (name) cc.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: - util.logexc(LOG, "Running %s (%s) failed", name, mod) + util.logexc(LOG, "Running module %s (%s) failed", name, mod) failures.append((name, e)) return (which_ran, failures) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 77af1270..02f6261d 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -20,13 +20,119 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from Cheetah.Template import Template +import collections +import re +try: + from Cheetah.Template import Template as CTemplate + CHEETAH_AVAILABLE = True +except (ImportError, AttributeError): + CHEETAH_AVAILABLE = False + +try: + import jinja2 + from jinja2 import Template as JTemplate + JINJA_AVAILABLE = True +except (ImportError, AttributeError): + JINJA_AVAILABLE = False + +from cloudinit import log as logging +from cloudinit import type_utils as tu from cloudinit import util +LOG = logging.getLogger(__name__) +TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I) +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') + + +def basic_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. + """ + + def replacer(match): + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) + if name is None: + name = match.group(2) + if name is None: + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not traverse into" + " non-dictionary '%s' of type %s while" + " looking for subkey '%s'" + % (selected_params, + tu.obj_name(selected_params), + key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, + tu.obj_name(selected_params))) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) + + +def detect_template(text): + + def cheetah_render(content, params): + return CTemplate(content, searchList=[params]).respond() + + def jinja_render(content, params): + return JTemplate(content, + undefined=jinja2.StrictUndefined, + trim_blocks=True).render(**params) + + if text.find("\n") != -1: + ident, rest = text.split("\n", 1) + else: + ident = text + rest = '' + type_match = TYPE_MATCHER.match(ident) + if not type_match: + if not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the default renderer for" + " unknown template, reverting to the basic renderer.") + return ('basic', basic_render, text) + else: + return ('cheetah', cheetah_render, text) + else: + template_type = type_match.group(1).lower().strip() + if template_type not in ('jinja', 'cheetah', 'basic'): + raise ValueError("Unknown template rendering type '%s' requested" + % template_type) + if template_type == 'jinja' and not JINJA_AVAILABLE: + LOG.warn("Jinja not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'jinja' and JINJA_AVAILABLE: + return ('jinja', jinja_render, rest) + if template_type == 'cheetah' and not CHEETAH_AVAILABLE: + LOG.warn("Cheetah not available as the selected renderer for" + " desired template, reverting to the basic renderer.") + return ('basic', basic_render, rest) + elif template_type == 'cheetah' and CHEETAH_AVAILABLE: + return ('cheetah', cheetah_render, rest) + # Only thing left over is the basic renderer (it is always available). + return ('basic', basic_render, rest) + def render_from_file(fn, params): - return render_string(util.load_file(fn), params) + if not params: + params = {} + template_type, renderer, content = detect_template(util.load_file(fn)) + LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type) + return renderer(content, params) def render_to_file(fn, outfn, params, mode=0644): @@ -37,4 +143,5 @@ def render_to_file(fn, outfn, params, mode=0644): def render_string(content, params): if not params: params = {} - return Template(content, searchList=[params]).respond() + template_type, renderer, content = detect_template(content) + return renderer(content, params) diff --git a/cloudinit/util.py b/cloudinit/util.py index 06039ee2..bc681f4a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -146,23 +146,23 @@ class SeLinuxGuard(object): return False def __exit__(self, excp_type, excp_value, excp_traceback): - if self.selinux and self.selinux.is_selinux_enabled(): - path = os.path.realpath(os.path.expanduser(self.path)) - # path should be a string, not unicode - path = str(path) - do_restore = False - try: - # See if even worth restoring?? - stats = os.lstat(path) - if stat.ST_MODE in stats: - self.selinux.matchpathcon(path, stats[stat.ST_MODE]) - do_restore = True - except OSError: - pass - if do_restore: - LOG.debug("Restoring selinux mode for %s (recursive=%s)", - path, self.recursive) - self.selinux.restorecon(path, recursive=self.recursive) + if not self.selinux or not self.selinux.is_selinux_enabled(): + return + if not os.path.lexists(self.path): + return + + path = os.path.realpath(self.path) + # path should be a string, not unicode + path = str(path) + try: + stats = os.lstat(path) + self.selinux.matchpathcon(path, stats[stat.ST_MODE]) + except OSError: + return + + LOG.debug("Restoring selinux mode for %s (recursive=%s)", + path, self.recursive) + self.selinux.restorecon(path, recursive=self.recursive) class MountFailedError(Exception): diff --git a/config/cloud.cfg b/config/cloud.cfg index b746e3db..200050d3 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -24,6 +24,7 @@ preserve_hostname: false # The modules that run in the 'init' stage cloud_init_modules: - migrator + - ubuntu-init-switch - seed_random - bootcmd - write-files diff --git a/packages/bddeb b/packages/bddeb index 9552aa40..9d264f92 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -31,6 +31,7 @@ PKG_MP = { 'argparse': 'python-argparse', 'cheetah': 'python-cheetah', 'configobj': 'python-configobj', + 'jinja2': 'python-jinja2', 'jsonpatch': 'python-jsonpatch | python-json-patch', 'oauth': 'python-oauth', 'prettytable': 'python-prettytable', @@ -38,7 +39,7 @@ PKG_MP = { 'pyyaml': 'python-yaml', 'requests': 'python-requests', } -DEBUILD_ARGS = ["-us", "-S", "-uc", "-d"] +DEBUILD_ARGS = ["-S", "-d"] def write_debian_folder(root, version, revno, append_requires=[]): @@ -75,7 +76,7 @@ def write_debian_folder(root, version, revno, append_requires=[]): params={'requires': requires}) # Just copy the following directly - for base_fn in ['dirs', 'copyright', 'compat', 'pycompat', 'rules']: + for base_fn in ['dirs', 'copyright', 'compat', 'rules']: shutil.copy(util.abs_join(find_root(), 'packages', 'debian', base_fn), util.abs_join(deb_dir, base_fn)) @@ -98,15 +99,23 @@ def main(): parser.add_argument("--init-system", dest="init_system", help=("build deb with INIT_SYSTEM=xxx" " (default: %(default)s"), - default=os.environ.get("INIT_SYSTEM", "upstart")) + default=os.environ.get("INIT_SYSTEM", + "upstart,systemd")) for ent in DEBUILD_ARGS: parser.add_argument(ent, dest="debuild_args", action='append_const', - const=ent, help=("pass through '%s' to debuild" % ent)) + const=ent, help=("pass through '%s' to debuild" % ent), + default=[]) + + parser.add_argument("--sign", default=False, action='store_true', + help="sign result. do not pass -us -uc to debuild") args = parser.parse_args() + if not args.sign: + args.debuild_args.extend(['-us', '-uc']) + os.environ['INIT_SYSTEM'] = args.init_system capture = True @@ -166,7 +175,8 @@ def main(): print("Copied that archive to %r for local usage (if desired)." % (util.abs_join(os.getcwd(), tar_fn))) - print("Running 'debuild' in %r" % (xdir)) + print("Running 'debuild %s' in %r" % (' '.join(args.debuild_args), + xdir)) with util.chdir(xdir): cmd = ['debuild', '--preserve-envvar', 'INIT_SYSTEM'] if args.debuild_args: diff --git a/packages/brpm b/packages/brpm index f8ba1db1..b8bbff9d 100755 --- a/packages/brpm +++ b/packages/brpm @@ -37,6 +37,7 @@ PKG_MP = { 'redhat': { 'argparse': 'python-argparse', 'cheetah': 'python-cheetah', + 'jinja2': 'python-jinja2', 'configobj': 'python-configobj', 'jsonpatch': 'python-jsonpatch', 'oauth': 'python-oauth', diff --git a/packages/debian/compat b/packages/debian/compat index 7ed6ff82..ec635144 100644 --- a/packages/debian/compat +++ b/packages/debian/compat @@ -1 +1 @@ -5 +9 diff --git a/packages/debian/control.in b/packages/debian/control.in index 7e42b94b..c892747c 100644 --- a/packages/debian/control.in +++ b/packages/debian/control.in @@ -1,18 +1,23 @@ ## This is a cheetah template Source: cloud-init Section: admin -Priority: extra +Priority: optional Maintainer: Scott Moser <smoser@ubuntu.com> -Build-Depends: cdbs, - debhelper (>= 5.0.38), +Build-Depends: debhelper (>= 9), + dh-python, + dh-systemd, python (>= 2.6.6-3~), python-nose, pyflakes, pylint, python-setuptools, + python-selinux, python-cheetah, python-mocker, - python-setuptools + python-httpretty, +#for $r in $requires + ${r}, +#end for XS-Python-Version: all Standards-Version: 3.9.3 diff --git a/packages/debian/pycompat b/packages/debian/pycompat deleted file mode 100644 index 0cfbf088..00000000 --- a/packages/debian/pycompat +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/packages/debian/rules b/packages/debian/rules index 7623ac9d..9e0c5ddb 100755 --- a/packages/debian/rules +++ b/packages/debian/rules @@ -1,18 +1,17 @@ #!/usr/bin/make -f -DEB_PYTHON2_MODULE_PACKAGES = cloud-init -INIT_SYSTEM ?= upstart +INIT_SYSTEM ?= upstart,systemd +export PYBUILD_INSTALL_ARGS=--init-system=$(INIT_SYSTEM) -binary-install/cloud-init::cloud-init-fixups +%: + dh $@ --with python2,systemd --buildsystem pybuild -include /usr/share/cdbs/1/rules/debhelper.mk -include /usr/share/cdbs/1/class/python-distutils.mk +override_dh_install: + dh_install + install -d debian/cloud-init/etc/rsyslog.d + cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf -DEB_PYTHON_INSTALL_ARGS_ALL += --init-system=$(INIT_SYSTEM) - -DEB_DH_INSTALL_SOURCEDIR := debian/tmp - -cloud-init-fixups: - install -d $(DEB_DESTDIR)/etc/rsyslog.d - cp tools/21-cloudinit.conf $(DEB_DESTDIR)/etc/rsyslog.d/21-cloudinit.conf - +override_dh_auto_test: + # Becuase setup tools didn't copy data... + cp -r tests/data .pybuild/pythonX.Y_2.7/build/tests + http_proxy= dh_auto_test -- --test-nose diff --git a/requirements.txt b/requirements.txt index fdcbd143..943dbef7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ # Used for untemplating any files or strings with parameters. cheetah +jinja2 # This is used for any pretty printing of tabular data. PrettyTable @@ -36,21 +36,6 @@ def is_f(p): return os.path.isfile(p) -INITSYS_FILES = { - 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], - 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], - 'systemd': [f for f in glob('systemd/*') if is_f(f)], - 'upstart': [f for f in glob('upstart/*') if is_f(f)], -} -INITSYS_ROOTS = { - 'sysvinit': '/etc/rc.d/init.d', - 'sysvinit_deb': '/etc/init.d', - 'systemd': '/etc/systemd/system/', - 'upstart': '/etc/init/', -} -INITSYS_TYPES = sorted(list(INITSYS_ROOTS.keys())) - - def tiny_p(cmd, capture=True): # Darn python 2.6 doesn't have check_output (argggg) stdout = subprocess.PIPE @@ -68,6 +53,29 @@ def tiny_p(cmd, capture=True): return (out, err) +def systemd_unitdir(): + cmd = ['pkg-config', '--variable=systemdsystemunitdir', 'systemd'] + try: + (path, err) = tiny_p(cmd) + except: + return '/lib/systemd/system' + return str(path).strip() + +INITSYS_FILES = { + 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], + 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], + 'systemd': [f for f in glob('systemd/*') if is_f(f)], + 'upstart': [f for f in glob('upstart/*') if is_f(f)], +} +INITSYS_ROOTS = { + 'sysvinit': '/etc/rc.d/init.d', + 'sysvinit_deb': '/etc/init.d', + 'systemd': systemd_unitdir(), + 'upstart': '/etc/init/', +} +INITSYS_TYPES = sorted(list(INITSYS_ROOTS.keys())) + + def get_version(): cmd = ['tools/read-version'] (ver, _e) = tiny_p(cmd) @@ -86,26 +94,35 @@ class InitsysInstallData(install): user_options = install.user_options + [ # This will magically show up in member variable 'init_sys' ('init-system=', None, - ('init system to configure (%s) [default: None]') % + ('init system(s) to configure (%s) [default: None]') % (", ".join(INITSYS_TYPES)) ), ] def initialize_options(self): install.initialize_options(self) - self.init_system = None + self.init_system = "" def finalize_options(self): install.finalize_options(self) - if self.init_system and self.init_system not in INITSYS_TYPES: + + if self.init_system and isinstance(self.init_system, str): + self.init_system = self.init_system.split(",") + + if len(self.init_system) == 0: raise DistutilsArgError(("You must specify one of (%s) when" - " specifying a init system!") % (", ".join(INITSYS_TYPES))) - elif self.init_system: + " specifying init system(s)!") % (", ".join(INITSYS_TYPES))) + + bad = [f for f in self.init_system if f not in INITSYS_TYPES] + if len(bad) != 0: + raise DistutilsArgError( + "Invalid --init-system: %s" % (','.join(bad))) + + for sys in self.init_system: self.distribution.data_files.append( - (INITSYS_ROOTS[self.init_system], - INITSYS_FILES[self.init_system])) - # Force that command to reinitalize (with new file list) - self.distribution.reinitialize_command('install_data', True) + (INITSYS_ROOTS[sys], INITSYS_FILES[sys])) + # Force that command to reinitalize (with new file list) + self.distribution.reinitialize_command('install_data', True) setuptools.setup(name='cloud-init', diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 7981cba7..538850ca 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -1,25 +1,25 @@ -#* - This file is only utilized if the module 'cc_chef' is enabled in - cloud-config. Specifically, in order to enable it - you need to add the following to config: - chef: - validation_key: XYZ - validation_cert: XYZ - validation_name: XYZ - server_url: XYZ -*# +## template:jinja +{# +This file is only utilized if the module 'cc_chef' is enabled in +cloud-config. Specifically, in order to enable it +you need to add the following to config: + chef: + validation_key: XYZ + validation_cert: XYZ + validation_name: XYZ + server_url: XYZ +-#} log_level :info log_location "/var/log/chef/client.log" ssl_verify_mode :verify_none -validation_client_name "$validation_name" +validation_client_name "{{validation_name}}" validation_key "/etc/chef/validation.pem" client_key "/etc/chef/client.pem" -chef_server_url "$server_url" -environment "$environment" -node_name "$node_name" +chef_server_url "{{server_url}}" +environment "{{environment}}" +node_name "{{node_name}}" json_attribs "/etc/chef/firstboot.json" file_cache_path "/var/cache/chef" file_backup_path "/var/backups/chef" pid_file "/var/run/chef/client.pid" Chef::Log::Formatter.show_time = true - diff --git a/templates/hosts.debian.tmpl b/templates/hosts.debian.tmpl index ae120b02..a1d97212 100644 --- a/templates/hosts.debian.tmpl +++ b/templates/hosts.debian.tmpl @@ -1,19 +1,19 @@ -## This file (/etc/cloud/templates/hosts.tmpl) is only utilized -## if enabled in cloud-config. Specifically, in order to enable it -## you need to add the following to config: -## manage_etc_hosts: True -## -## Note, double-hash commented lines will not appear in /etc/hosts -# +## template:jinja +{# +This file (/etc/cloud/templates/hosts.tmpl) is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} # Your system has configured 'manage_etc_hosts' as True. # As a result, if you wish for changes to this file to persist # then you will need to either # a.) make changes to the master file in /etc/cloud/templates/hosts.tmpl # b.) change or remove the value of 'manage_etc_hosts' in # /etc/cloud/cloud.cfg or cloud-config from user-data -# -## The value '$hostname' will be replaced with the local-hostname -127.0.1.1 $fqdn $hostname +# +{# The value '{{hostname}}' will be replaced with the local-hostname -#} +127.0.1.1 {{fqdn}} {{hostname}} 127.0.0.1 localhost # The following lines are desirable for IPv6 capable hosts @@ -23,3 +23,4 @@ ff00::0 ip6-mcastprefix ff02::1 ip6-allnodes ff02::2 ip6-allrouters ff02::3 ip6-allhosts + diff --git a/templates/hosts.redhat.tmpl b/templates/hosts.redhat.tmpl index 80459d95..bc5da32c 100644 --- a/templates/hosts.redhat.tmpl +++ b/templates/hosts.redhat.tmpl @@ -1,9 +1,10 @@ -#* - This file /etc/cloud/templates/hosts.redhat.tmpl is only utilized - if enabled in cloud-config. Specifically, in order to enable it - you need to add the following to config: - manage_etc_hosts: True -*# +## template:jinja +{# +This file /etc/cloud/templates/hosts.redhat.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} # Your system has configured 'manage_etc_hosts' as True. # As a result, if you wish for changes to this file to persist # then you will need to either @@ -12,12 +13,12 @@ # /etc/cloud/cloud.cfg or cloud-config from user-data # # The following lines are desirable for IPv4 capable hosts -127.0.0.1 ${fqdn} ${hostname} +127.0.0.1 {{fqdn}} {{hostname}} 127.0.0.1 localhost.localdomain localhost 127.0.0.1 localhost4.localdomain4 localhost4 # The following lines are desirable for IPv6 capable hosts -::1 ${fqdn} ${hostname} +::1 {{fqdn}} {{hostname}} ::1 localhost.localdomain localhost ::1 localhost6.localdomain6 localhost6 diff --git a/templates/hosts.suse.tmpl b/templates/hosts.suse.tmpl index 5d3d57e4..b6082692 100644 --- a/templates/hosts.suse.tmpl +++ b/templates/hosts.suse.tmpl @@ -1,9 +1,10 @@ -#* - This file /etc/cloud/templates/hosts.suse.tmpl is only utilized - if enabled in cloud-config. Specifically, in order to enable it - you need to add the following to config: - manage_etc_hosts: True -*# +## template:jinja +{# +This file /etc/cloud/templates/hosts.suse.tmpl is only utilized +if enabled in cloud-config. Specifically, in order to enable it +you need to add the following to config: + manage_etc_hosts: True +-#} # Your system has configured 'manage_etc_hosts' as True. # As a result, if you wish for changes to this file to persist # then you will need to either @@ -22,3 +23,4 @@ ff00::0 ipv6-mcastprefix ff02::1 ipv6-allnodes ff02::2 ipv6-allrouters ff02::3 ipv6-allhosts + diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index b7e97b13..6f908f30 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -1,39 +1,30 @@ -# +## template:jinja # Your system has been configured with 'manage-resolv-conf' set to true. # As a result, cloud-init has written this file with configuration data # that it has been provided. Cloud-init, by default, will write this file # a single time (PER_ONCE). # +{% if nameservers is defined %} +{% for server in nameservers %} +nameserver {{server}} +{% endfor %} -#if $varExists('nameservers') -#for $server in $nameservers -nameserver $server -#end for -#end if -#if $varExists('searchdomains') -search #slurp -#for $search in $searchdomains -$search #slurp -#end for +{% endif -%} +{% if searchdomains is defined %} +search {% for search in searchdomains %}{{search}} {% endfor %} -#end if -#if $varExists('domain') -domain $domain -#end if -#if $varExists('sortlist') -sortlist #slurp -#for $sort in $sortlist -$sort #slurp -#end for +{% endif %} +{% if domain is defined %} +domain {{domain}} +{% endif %} +{% if sortlist is defined %} -#end if -#if $varExists('options') or $varExists('flags') -options #slurp -#for $flag in $flags -$flag #slurp -#end for -#for $key, $value in $options.items() -$key:$value #slurp -#end for +sortlist {% for sort in sortlist %}{{sort}} {% endfor %} +{% endif %} +{% if options is defined or flags is defined %} -#end if +options {% for flag in flags %}{{flag}} {% endfor %} +{% for key, value in options.iteritems() -%} + {{key}}:{{value}} +{% endfor %} +{% endif %} diff --git a/templates/sources.list.debian.tmpl b/templates/sources.list.debian.tmpl index 609bc6bd..c8043f76 100644 --- a/templates/sources.list.debian.tmpl +++ b/templates/sources.list.debian.tmpl @@ -1,28 +1,32 @@ -\## Note, this file is written by cloud-init on first boot of an instance -\## modifications made here will not survive a re-bundle. -\## if you wish to make changes you can: -\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -\## or do the same in user-data -\## b.) add sources in /etc/apt/sources.list.d -\## c.) make changes to template file /etc/cloud/templates/sources.list.debian.tmpl -\### +## template:jinja +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.debian.tmpl +### # See http://www.debian.org/releases/stable/i386/release-notes/ch-upgrading.html # for how to upgrade to newer versions of the distribution. -deb $mirror $codename main contrib non-free -deb-src $mirror $codename main contrib non-free +deb {{mirror}} {{codename}} main contrib non-free +deb-src {{mirror}} {{codename}} main contrib non-free -\## Major bug fix updates produced after the final release of the -\## distribution. -deb $security $codename/updates main contrib non-free -deb-src $security $codename/updates main contrib non-free -deb $mirror $codename-updates main contrib non-free -deb-src $mirror $codename-updates main contrib non-free +## Major bug fix updates produced after the final release of the +## distribution. +deb {{security}} {{codename}}/updates main contrib non-free +deb-src {{security}} {{codename}}/updates main contrib non-free +deb {{mirror}} {{codename}}-updates main contrib non-free +deb-src {{mirror}} {{codename}}-updates main contrib non-free -\## Uncomment the following two lines to add software from the 'backports' -\## repository. -\## N.B. software from this repository may not have been tested as -\## extensively as that contained in the main release, although it includes -\## newer versions of some applications which may provide useful features. -# deb http://backports.debian.org/debian-backports $codename-backports main contrib non-free -# deb-src http://backports.debian.org/debian-backports $codename-backports main contrib non-free +## Uncomment the following two lines to add software from the 'backports' +## repository. +## +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +{# +deb http://backports.debian.org/debian-backports {{codename}}-backports main contrib non-free +deb-src http://backports.debian.org/debian-backports {{codename}}-backports main contrib non-free +-#} diff --git a/templates/sources.list.ubuntu.tmpl b/templates/sources.list.ubuntu.tmpl index ce395b3d..4b1b019a 100644 --- a/templates/sources.list.ubuntu.tmpl +++ b/templates/sources.list.ubuntu.tmpl @@ -1,60 +1,60 @@ -\## Note, this file is written by cloud-init on first boot of an instance -\## modifications made here will not survive a re-bundle. -\## if you wish to make changes you can: -\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -\## or do the same in user-data -\## b.) add sources in /etc/apt/sources.list.d -\## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl -\### +## template:jinja +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. -deb $mirror $codename main -deb-src $mirror $codename main +deb {{mirror}} {{codename}} main +deb-src {{mirror}} {{codename}} main -\## Major bug fix updates produced after the final release of the -\## distribution. -deb $mirror $codename-updates main -deb-src $mirror $codename-updates main +## Major bug fix updates produced after the final release of the +## distribution. +deb {{mirror}} {{codename}}-updates main +deb-src {{mirror}} {{codename}}-updates main -\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -\## team. Also, please note that software in universe WILL NOT receive any -\## review or updates from the Ubuntu security team. -deb $mirror $codename universe -deb-src $mirror $codename universe -deb $mirror $codename-updates universe -deb-src $mirror $codename-updates universe +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team. Also, please note that software in universe WILL NOT receive any +## review or updates from the Ubuntu security team. +deb {{mirror}} {{codename}} universe +deb-src {{mirror}} {{codename}} universe +deb {{mirror}} {{codename}}-updates universe +deb-src {{mirror}} {{codename}}-updates universe -\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu -\## team, and may not be under a free licence. Please satisfy yourself as to -\## your rights to use the software. Also, please note that software in -\## multiverse WILL NOT receive any review or updates from the Ubuntu -\## security team. -# deb $mirror $codename multiverse -# deb-src $mirror $codename multiverse -# deb $mirror $codename-updates multiverse -# deb-src $mirror $codename-updates multiverse +## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu +## team, and may not be under a free licence. Please satisfy yourself as to +## your rights to use the software. Also, please note that software in +## multiverse WILL NOT receive any review or updates from the Ubuntu +## security team. +# deb {{mirror}} {{codename}} multiverse +# deb-src {{mirror}} {{codename}} multiverse +# deb {{mirror}} {{codename}}-updates multiverse +# deb-src {{mirror}} {{codename}}-updates multiverse -\## Uncomment the following two lines to add software from the 'backports' -\## repository. -\## N.B. software from this repository may not have been tested as -\## extensively as that contained in the main release, although it includes -\## newer versions of some applications which may provide useful features. -\## Also, please note that software in backports WILL NOT receive any review -\## or updates from the Ubuntu security team. -# deb $mirror $codename-backports main restricted universe multiverse -# deb-src $mirror $codename-backports main restricted universe multiverse +## Uncomment the following two lines to add software from the 'backports' +## repository. +## N.B. software from this repository may not have been tested as +## extensively as that contained in the main release, although it includes +## newer versions of some applications which may provide useful features. +## Also, please note that software in backports WILL NOT receive any review +## or updates from the Ubuntu security team. +# deb {{mirror}} {{codename}}-backports main restricted universe multiverse +# deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse -\## Uncomment the following two lines to add software from Canonical's -\## 'partner' repository. -\## This software is not part of Ubuntu, but is offered by Canonical and the -\## respective vendors as a service to Ubuntu users. -# deb http://archive.canonical.com/ubuntu $codename partner -# deb-src http://archive.canonical.com/ubuntu $codename partner +## Uncomment the following two lines to add software from Canonical's +## 'partner' repository. +## This software is not part of Ubuntu, but is offered by Canonical and the +## respective vendors as a service to Ubuntu users. +# deb http://archive.canonical.com/ubuntu {{codename}} partner +# deb-src http://archive.canonical.com/ubuntu {{codename}} partner -deb $security $codename-security main -deb-src $security $codename-security main -deb $security $codename-security universe -deb-src $security $codename-security universe -# deb $security $codename-security multiverse -# deb-src $security $codename-security multiverse +deb {{security}} {{codename}}-security main +deb-src {{security}} {{codename}}-security main +deb {{security}} {{codename}}-security universe +deb-src {{security}} {{codename}}-security universe +# deb {{security}} {{codename}}-security multiverse +# deb-src {{security}} {{codename}}-security multiverse diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 5bed13cc..9700a4ca 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -52,6 +52,30 @@ if PY26: standardMsg = standardMsg % (value) self.fail(self._formatMessage(msg, standardMsg)) + def assertDictContainsSubset(self, expected, actual, msg=None): + missing = [] + mismatched = [] + for k, v in expected.iteritems(): + if k not in actual: + missing.append(k) + elif actual[k] != v: + mismatched.append('%r, expected: %r, actual: %r' + % (k, v, actual[k])) + + if len(missing) == 0 and len(mismatched) == 0: + return + + standardMsg = '' + if missing: + standardMsg = 'Missing: %r' % ','.join(m for m in missing) + if mismatched: + if standardMsg: + standardMsg += '; ' + standardMsg += 'Mismatched values: %s' % ','.join(mismatched) + + self.fail(self._formatMessage(msg, standardMsg)) + + else: class TestCase(unittest.TestCase): pass @@ -209,6 +233,21 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): self.patched_funcs.append((mod, f, func)) +class HttprettyTestCase(TestCase): + # necessary as http_proxy gets in the way of httpretty + # https://github.com/gabrielfalcao/HTTPretty/issues/122 + def setUp(self): + self.restore_proxy = os.environ.get('http_proxy') + if self.restore_proxy is not None: + del os.environ['http_proxy'] + super(HttprettyTestCase, self).setUp() + + def tearDown(self): + if self.restore_proxy: + os.environ['http_proxy'] = self.restore_proxy + super(HttprettyTestCase, self).tearDown() + + def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index b387f13b..af7f442e 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -2,7 +2,7 @@ import os -from tests.unittests import helpers as test_helpers +from . import helpers as test_helpers from cloudinit import handlers from cloudinit import helpers diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 68729c57..41d0dc29 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -20,7 +20,7 @@ from cloudinit import util INSTANCE_ID = "i-testing" -from tests.unittests import helpers +from . import helpers class FakeDataSource(sources.DataSource): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index ccfd672a..88c82d5e 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.util import load_file from cloudinit.sources import DataSourceAzure -from tests.unittests.helpers import populate_dir +from ..helpers import populate_dir import base64 import crypt diff --git a/tests/unittests/test_datasource/test_cloudsigma.py b/tests/unittests/test_datasource/test_cloudsigma.py index f92e07b7..306ac7d8 100644 --- a/tests/unittests/test_datasource/test_cloudsigma.py +++ b/tests/unittests/test_datasource/test_cloudsigma.py @@ -1,10 +1,11 @@ # coding: utf-8 import copy -from unittest import TestCase from cloudinit.cs_utils import Cepko from cloudinit.sources import DataSourceCloudSigma +from .. import helpers as test_helpers + SERVER_CONTEXT = { "cpu": 1000, @@ -36,7 +37,7 @@ class CepkoMock(Cepko): return self -class DataSourceCloudSigmaTest(TestCase): +class DataSourceCloudSigmaTest(test_helpers.TestCase): def setUp(self): self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "") self.datasource.is_running_in_cloudsigma = lambda: True diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 4404668e..d88066e5 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -12,7 +12,7 @@ from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack from cloudinit import util -from tests.unittests import helpers as unit_helpers +from .. import helpers as unit_helpers PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index d91bd531..60a0ce48 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -15,7 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import unittest import httpretty import re @@ -25,6 +24,8 @@ from cloudinit import settings from cloudinit import helpers from cloudinit.sources import DataSourceGCE +from .. import helpers as test_helpers + GCE_META = { 'instance/id': '123', 'instance/zone': 'foo/bar', @@ -54,12 +55,13 @@ def _request_callback(method, uri, headers): return (404, headers, '') -class TestDataSourceGCE(unittest.TestCase): +class TestDataSourceGCE(test_helpers.HttprettyTestCase): def setUp(self): self.ds = DataSourceGCE.DataSourceGCE( settings.CFG_BUILTIN, None, helpers.Paths({})) + super(TestDataSourceGCE, self).setUp() @httpretty.activate def test_connection(self): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 73cfadcb..c157beb8 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -3,7 +3,7 @@ import os from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper -from tests.unittests.helpers import populate_dir +from ..helpers import populate_dir import mocker diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index a65833eb..14274562 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceNoCloud from cloudinit import util -from tests.unittests.helpers import populate_dir +from ..helpers import populate_dir from mocker import MockerTestCase import os diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ec6b752b..b4fd1f4d 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -2,7 +2,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util from mocker import MockerTestCase -from tests.unittests.helpers import populate_dir +from ..helpers import populate_dir from base64 import b64encode import os diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 3a64430a..c088bb55 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -24,7 +24,7 @@ from StringIO import StringIO from urlparse import urlparse -from tests.unittests import helpers as test_helpers +from .. import helpers as test_helpers from cloudinit import helpers from cloudinit import settings @@ -121,7 +121,7 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): body=get_request_callback) -class TestOpenStackDataSource(test_helpers.TestCase): +class TestOpenStackDataSource(test_helpers.HttprettyTestCase): VERSION = 'latest' @hp.activate diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index f64aea07..b197b600 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -25,7 +25,7 @@ import base64 from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS -from tests.unittests import helpers +from .. import helpers import os import os.path import re diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 7befb8c8..c24c790e 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -1,7 +1,7 @@ from cloudinit import distros from cloudinit import util -from tests.unittests import helpers +from .. import helpers import os diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index dd87665d..700254a3 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -1,4 +1,4 @@ -from tests.unittests import helpers +from . import helpers from cloudinit import ec2_utils as eu from cloudinit import url_helper as uh @@ -6,7 +6,7 @@ from cloudinit import url_helper as uh import httpretty as hp -class TestEc2Util(helpers.TestCase): +class TestEc2Util(helpers.HttprettyTestCase): VERSION = 'latest' @hp.activate diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 773bb312..2f4c2fda 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -1,6 +1,6 @@ import copy -from tests.unittests import helpers +from .. import helpers import itertools diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index f2ed4597..f6dc4521 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -188,8 +188,8 @@ class TestResize(MockerTestCase): self.assertEqual(cc_growpart.RESIZE.SKIPPED, find(enoent[0], resized)[1]) #self.assertEqual(resize_calls, - #[("/dev/XXda", "1", "/dev/XXda1"), - #("/dev/YYda", "2", "/dev/YYda2")]) + # [("/dev/XXda", "1", "/dev/XXda1"), + # ("/dev/YYda", "2", "/dev/YYda2")]) finally: cc_growpart.device_part_info = opinfo os.stat = real_stat diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index 72ad00fd..eb251636 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -25,7 +25,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from tests.unittests import helpers as t_help +from .. import helpers as t_help from configobj import ConfigObj diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index f6e37fa5..4b7b2112 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -1,6 +1,6 @@ from cloudinit.config import cc_power_state_change as psc -from tests.unittests import helpers as t_help +from .. import helpers as t_help class TestLoadPowerState(t_help.TestCase): diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index be2fa4a4..40481f16 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -1,4 +1,4 @@ - # Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. # # Author: Juerg Haefliger <juerg.haefliger@hp.com> # @@ -31,7 +31,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNone -from tests.unittests import helpers as t_help +from .. import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index 6344ec0c..03004ab9 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -5,7 +5,7 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import util -from tests.unittests import helpers as t_help +from .. import helpers as t_help import logging diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index 40b69773..874db340 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -25,7 +25,7 @@ from cloudinit import util from cloudinit.sources import DataSourceNoCloud -from tests.unittests import helpers as t_help +from .. import helpers as t_help from configobj import ConfigObj diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 7c6f7c40..156441c7 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -2,7 +2,7 @@ from cloudinit import util from cloudinit.config import cc_yum_add_repo -from tests.unittests import helpers +from .. import helpers import logging diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 486b9158..17704f8e 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,4 +1,4 @@ -from tests.unittests import helpers +from . import helpers from cloudinit.handlers import cloud_config from cloudinit.handlers import (CONTENT_START, CONTENT_END) diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py index c68c263c..590c4b82 100644 --- a/tests/unittests/test_pathprefix2dict.py +++ b/tests/unittests/test_pathprefix2dict.py @@ -1,7 +1,7 @@ from cloudinit import util from mocker import MockerTestCase -from tests.unittests.helpers import populate_dir +from .helpers import populate_dir class TestPathPrefix2Dict(MockerTestCase): diff --git a/tests/unittests/test_runs/test_merge_run.py b/tests/unittests/test_runs/test_merge_run.py index 5ffe95a2..32b41925 100644 --- a/tests/unittests/test_runs/test_merge_run.py +++ b/tests/unittests/test_runs/test_merge_run.py @@ -1,6 +1,6 @@ import os -from tests.unittests import helpers +from .. import helpers from cloudinit.settings import (PER_INSTANCE) from cloudinit import stages diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 9a7178d1..c9ba949e 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -1,6 +1,6 @@ import os -from tests.unittests import helpers +from .. import helpers from cloudinit.settings import (PER_INSTANCE) from cloudinit import stages diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py new file mode 100644 index 00000000..87681f0f --- /dev/null +++ b/tests/unittests/test_templating.py @@ -0,0 +1,107 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow <harlowja@yahoo-inc.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from . import helpers as test_helpers +import textwrap + +from cloudinit import templater + + +class TestTemplates(test_helpers.TestCase): + def test_render_basic(self): + in_data = textwrap.dedent(""" + ${b} + + c = d + """) + in_data = in_data.strip() + expected_data = textwrap.dedent(""" + 2 + + c = d + """) + out_data = templater.basic_render(in_data, {'b': 2}) + self.assertEqual(expected_data.strip(), out_data) + + def test_detection(self): + blob = "## template:cheetah" + + (template_type, renderer, contents) = templater.detect_template(blob) + self.assertIn("cheetah", template_type) + self.assertEqual("", contents.strip()) + + blob = "blahblah $blah" + (template_type, renderer, contents) = templater.detect_template(blob) + self.assertIn("cheetah", template_type) + self.assertEquals(blob, contents) + + blob = '##template:something-new' + self.assertRaises(ValueError, templater.detect_template, blob) + + def test_render_cheetah(self): + blob = '''## template:cheetah +$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_jinja(self): + blob = '''## template:jinja +{{a}},{{b}}''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_default(self): + blob = '''$a,$b''' + c = templater.render_string(blob, {"a": 1, "b": 2}) + self.assertEquals("1,2", c) + + def test_render_basic_deeper(self): + hn = 'myfoohost.yahoo.com' + expected_data = "h=%s\nc=d\n" % hn + in_data = "h=$hostname.canonical_name\nc=d\n" + params = { + "hostname": { + "canonical_name": hn, + }, + } + out_data = templater.render_string(in_data, params) + self.assertEqual(expected_data, out_data) + + def test_render_basic_no_parens(self): + hn = "myfoohost" + in_data = "h=$hostname\nc=d\n" + expected_data = "h=%s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic_parens(self): + hn = "myfoohost" + in_data = "h = ${hostname}\nc=d\n" + expected_data = "h = %s\nc=d\n" % hn + out_data = templater.basic_render(in_data, {'hostname': hn}) + self.assertEqual(expected_data, out_data) + + def test_render_basic2(self): + mirror = "mymirror" + codename = "zany" + in_data = "deb $mirror $codename-updates main contrib non-free" + ex_data = "deb %s %s-updates main contrib non-free" % (mirror, codename) + + out_data = templater.basic_render(in_data, + {'mirror': mirror, 'codename': codename}) + self.assertEqual(ex_data, out_data) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 38ab0c96..0cb41520 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -5,8 +5,8 @@ import stat import yaml from mocker import MockerTestCase -from tests.unittests import helpers -from unittest import TestCase +from . import helpers +import unittest from cloudinit import importer from cloudinit import util @@ -31,7 +31,7 @@ class FakeSelinux(object): self.restored.append(path) -class TestGetCfgOptionListOrStr(TestCase): +class TestGetCfgOptionListOrStr(unittest.TestCase): def test_not_found_no_default(self): """None is returned if key is not found and no default given.""" config = {} @@ -124,16 +124,21 @@ class TestWriteFile(MockerTestCase): def test_restorecon_if_possible_is_called(self): """Make sure the selinux guard is called correctly.""" + my_file = os.path.join(self.tmp, "my_file") + with open(my_file, "w") as fp: + fp.write("My Content") + import_mock = self.mocker.replace(importer.import_module, passthrough=False) import_mock('selinux') - fake_se = FakeSelinux('/etc/hosts') + + fake_se = FakeSelinux(my_file) self.mocker.result(fake_se) self.mocker.replay() - with util.SeLinuxGuard("/etc/hosts") as is_on: + with util.SeLinuxGuard(my_file) as is_on: self.assertTrue(is_on) self.assertEqual(1, len(fake_se.restored)) - self.assertEqual('/etc/hosts', fake_se.restored[0]) + self.assertEqual(my_file, fake_se.restored[0]) class TestDeleteDirContents(MockerTestCase): @@ -201,20 +206,20 @@ class TestDeleteDirContents(MockerTestCase): self.assertDirEmpty(self.tmp) -class TestKeyValStrings(TestCase): +class TestKeyValStrings(unittest.TestCase): def test_keyval_str_to_dict(self): expected = {'1': 'one', '2': 'one+one', 'ro': True} cmdline = "1=one ro 2=one+one" self.assertEqual(expected, util.keyval_str_to_dict(cmdline)) -class TestGetCmdline(TestCase): +class TestGetCmdline(unittest.TestCase): def test_cmdline_reads_debug_env(self): os.environ['DEBUG_PROC_CMDLINE'] = 'abcd 123' self.assertEqual(os.environ['DEBUG_PROC_CMDLINE'], util.get_cmdline()) -class TestLoadYaml(TestCase): +class TestLoadYaml(unittest.TestCase): mydefault = "7b03a8ebace993d806255121073fed52" def test_simple(self): |