From 7e2d084eabbabd035e56a65c52b277a0732b09d5 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 23 Jan 2014 16:51:49 -0600 Subject: Initial add Gentoo distro --- cloudinit/distros/__init__.py | 3 +- cloudinit/distros/gentoo.py | 169 +++++++++++++++++++++++++++++++++++++++ setup.py | 6 +- sysvinit/gentoo/cloud-config | 14 ++++ sysvinit/gentoo/cloud-final | 11 +++ sysvinit/gentoo/cloud-init | 12 +++ sysvinit/gentoo/cloud-init-local | 13 +++ 7 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 cloudinit/distros/gentoo.py create mode 100644 sysvinit/gentoo/cloud-config create mode 100644 sysvinit/gentoo/cloud-final create mode 100644 sysvinit/gentoo/cloud-init create mode 100644 sysvinit/gentoo/cloud-init-local diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74e95797..2b821926 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -39,7 +39,8 @@ from cloudinit.distros.parsers import hosts OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['fedora', 'rhel'], - 'suse': ['sles'] + 'suse': ['sles'], + 'gentoo': ['gentoo'], } LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py new file mode 100644 index 00000000..b6291dc6 --- /dev/null +++ b/cloudinit/distros/gentoo.py @@ -0,0 +1,169 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import distros +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + locale_conf_fn = "/etc/locale.gen" + network_conf_fn = "/etc/conf.d/net" + tz_conf_fn = "/etc/timezone" + tz_local_fn = "/etc/localtime" + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + self.osfamily = 'gentoo' + + def apply_locale(self, locale, out_fn=None): + if not out_fn: + out_fn = self.locale_conf_fn + util.subp(['locale-gen', '-G', locale], capture=False) + # "" provides trailing newline during join + lines = [ + util.make_header(), + 'LANG="%s"' % (locale), + "", + ] + util.write_file(out_fn, "\n".join(lines)) + + def install_packages(self, pkglist): + self.update_package_sources() + self.package_command('install', pkgs=pkglist) + + def _write_network(self, settings): + util.write_file(self.network_conf_fn, settings) + return ['all'] + + # TODO(NateH): Update to use init scripts + def _bring_up_interface(self, device_name): + cmd = ['ifup', device_name] + LOG.debug("Attempting to run bring up interface %s using command %s", + device_name, cmd) + try: + (_out, err) = util.subp(cmd) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", cmd, err) + return True + except util.ProcessExecutionError: + util.logexc(LOG, "Running interface command %s failed", cmd) + return False + + # TODO(NateH): Refactor for gentoo net init scripts + def _bring_up_interfaces(self, device_names): + use_all = False + for d in device_names: + if d == 'all': + use_all = True + if use_all: + return distros.Distro._bring_up_interface(self, '--all') + else: + return distros.Distro._bring_up_interfaces(self, device_names) + + def _select_hostname(self, hostname, fqdn): + # Prefer the short hostname over the long + # fully qualified domain name + if not hostname: + return fqdn + return hostname + + def _write_hostname(self, your_hostname, out_fn): + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(your_hostname) + util.write_file(out_fn, str(conf), 0644) + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) + return (self.hostname_conf_fn, sys_hostname) + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_hostname(self, filename, default=None): + hostname = None + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def set_timezone(self, tz): + tz_file = self._find_tz_file(tz) + # Note: "" provides trailing newline during join + tz_lines = [ + util.make_header(), + str(tz), + "", + ] + util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['emerge'] + # Redirect output + cmd.append("--quiet") + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, capture=False) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ["-u", "world", "--quiet"], freq=PER_INSTANCE) diff --git a/setup.py b/setup.py index 8d18b97e..08a8d771 100755 --- a/setup.py +++ b/setup.py @@ -39,12 +39,14 @@ def is_f(p): INITSYS_FILES = { 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], + 'sysvinit_gentoo': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], 'systemd': [f for f in glob('systemd/*') if is_f(f)], 'upstart': [f for f in glob('upstart/*') if is_f(f)], } INITSYS_ROOTS = { 'sysvinit': '/etc/rc.d/init.d', 'sysvinit_deb': '/etc/init.d', + 'sysvinit_gentoo': '/etc/init.d', 'systemd': '/etc/systemd/system/', 'upstart': '/etc/init/', } @@ -63,7 +65,7 @@ def tiny_p(cmd, capture=True): (out, err) = sp.communicate() ret = sp.returncode # pylint: disable=E1101 if ret not in [0]: - raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" + raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" % (cmd, ret, out, err)) return (out, err) @@ -102,7 +104,7 @@ class InitsysInstallData(install): " specifying a init system!") % (", ".join(INITSYS_TYPES))) elif self.init_system: self.distribution.data_files.append( - (INITSYS_ROOTS[self.init_system], + (INITSYS_ROOTS[self.init_system], INITSYS_FILES[self.init_system])) # Force that command to reinitalize (with new file list) self.distribution.reinitialize_command('install_data', True) diff --git a/sysvinit/gentoo/cloud-config b/sysvinit/gentoo/cloud-config new file mode 100644 index 00000000..affd0cb3 --- /dev/null +++ b/sysvinit/gentoo/cloud-config @@ -0,0 +1,14 @@ +#!/sbin/runscript + +depend() { + after net # remove after nova-agent fix + before cloud-init-local + before cloud-init + before cloud-final + provide cloud-config +} + +start() { + cloud-init modules --mode config + eend 0 +} diff --git a/sysvinit/gentoo/cloud-final b/sysvinit/gentoo/cloud-final new file mode 100644 index 00000000..140c0f8e --- /dev/null +++ b/sysvinit/gentoo/cloud-final @@ -0,0 +1,11 @@ +#!/sbin/runscript +# add depends for network, dns, fs etc +depend() { + after cloud-init + provide cloud-final +} + +start() { + cloud-init modules --mode final + eend 0 +} diff --git a/sysvinit/gentoo/cloud-init b/sysvinit/gentoo/cloud-init new file mode 100644 index 00000000..6f3ff8ed --- /dev/null +++ b/sysvinit/gentoo/cloud-init @@ -0,0 +1,12 @@ +#!/sbin/runscript +# add depends for network, dns, fs etc +depend() { + after cloud-init-local + before cloud-final + provide cloud-init +} + +start() { + cloud-init init + eend 0 +} diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local new file mode 100644 index 00000000..7c39ff78 --- /dev/null +++ b/sysvinit/gentoo/cloud-init-local @@ -0,0 +1,13 @@ +#!/sbin/runscript + +depend() { + after cloud-config + before cloud-init + before cloud-final + provide cloud-init-local +} + +start() { + cloud-init init --local + eend 0 +} -- cgit v1.2.3 From f64cd3d53fb09fb41104f600ead60eefe9dbd4ab Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 23 Jan 2014 21:31:53 -0600 Subject: Package manager install / update fixes and service restart foo --- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/distros/debian.py | 1 + cloudinit/distros/gentoo.py | 2 +- cloudinit/distros/rhel.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 56a36906..06974248 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -136,7 +136,7 @@ def handle(_name, cfg, cloud, log, args): util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) try: - cmd = ['service'] + cmd = cloud.distro.init_cmd or ['service'] cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) cmd.append('restart') util.subp(cmd) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1ae232fd..3b912728 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -48,6 +48,7 @@ class Distro(distros.Distro): network_conf_fn = "/etc/network/interfaces" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" + init_cmd = ['service'] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index b6291dc6..82d49fc3 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -60,7 +60,7 @@ class Distro(distros.Distro): def install_packages(self, pkglist): self.update_package_sources() - self.package_command('install', pkgs=pkglist) + self.package_command('', pkgs=pkglist) def _write_network(self, settings): util.write_file(self.network_conf_fn, settings) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 30195384..7cc7a813 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -49,6 +49,7 @@ class Distro(distros.Distro): network_script_tpl = '/etc/sysconfig/network-scripts/ifcfg-%s' resolve_conf_fn = "/etc/resolv.conf" tz_local_fn = "/etc/localtime" + init_cmd = ['service'] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) -- cgit v1.2.3 From 29780092f57931d7c22bfbf3af37dea00edc76c9 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Fri, 24 Jan 2014 13:14:12 -0600 Subject: init_cmd distro unique supports gentoo init scripts --- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/distros/__init__.py | 1 + cloudinit/distros/gentoo.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 06974248..579735ed 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -136,7 +136,7 @@ def handle(_name, cfg, cloud, log, args): util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) try: - cmd = cloud.distro.init_cmd or ['service'] + cmd = cloud.distro.init_cmd cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) cmd.append('restart') util.subp(cmd) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2b821926..61904b6f 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -53,6 +53,7 @@ class Distro(object): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" + init_cmd = [] # Not implemented def __init__(self, name, cfg, paths): self._paths = paths diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 82d49fc3..ede40bff 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -156,7 +156,8 @@ class Distro(distros.Distro): elif args and isinstance(args, list): cmd.extend(args) - cmd.append(command) + if command: + cmd.append(command) pkglist = util.expand_package_list('%s-%s', pkgs) cmd.extend(pkglist) -- cgit v1.2.3 From 2b416f666e26f37ad694d9cdefd0a80b77009848 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Fri, 24 Jan 2014 15:57:35 -0600 Subject: Fixes gentoo sysvinit ordering --- sysvinit/gentoo/cloud-config | 5 ++--- sysvinit/gentoo/cloud-final | 4 ++-- sysvinit/gentoo/cloud-init | 2 +- sysvinit/gentoo/cloud-init-local | 3 +-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/sysvinit/gentoo/cloud-config b/sysvinit/gentoo/cloud-config index affd0cb3..b0fa786d 100644 --- a/sysvinit/gentoo/cloud-config +++ b/sysvinit/gentoo/cloud-config @@ -1,9 +1,8 @@ #!/sbin/runscript depend() { - after net # remove after nova-agent fix - before cloud-init-local - before cloud-init + after cloud-init-local + after cloud-init before cloud-final provide cloud-config } diff --git a/sysvinit/gentoo/cloud-final b/sysvinit/gentoo/cloud-final index 140c0f8e..b457a354 100644 --- a/sysvinit/gentoo/cloud-final +++ b/sysvinit/gentoo/cloud-final @@ -1,7 +1,7 @@ #!/sbin/runscript -# add depends for network, dns, fs etc + depend() { - after cloud-init + after cloud-config provide cloud-final } diff --git a/sysvinit/gentoo/cloud-init b/sysvinit/gentoo/cloud-init index 6f3ff8ed..9ab64ad8 100644 --- a/sysvinit/gentoo/cloud-init +++ b/sysvinit/gentoo/cloud-init @@ -2,7 +2,7 @@ # add depends for network, dns, fs etc depend() { after cloud-init-local - before cloud-final + before cloud-config provide cloud-init } diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local index 7c39ff78..1d22a79d 100644 --- a/sysvinit/gentoo/cloud-init-local +++ b/sysvinit/gentoo/cloud-init-local @@ -1,9 +1,8 @@ #!/sbin/runscript depend() { - after cloud-config + after net # remove after nova-agent fix before cloud-init - before cloud-final provide cloud-init-local } -- cgit v1.2.3 From 8f6c9ff5ed7e301db2efa9ffa9a4bf454eb819e9 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 27 Jan 2014 11:43:13 -0600 Subject: init_cmd inheritance fixes --- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/debian.py | 1 - cloudinit/distros/gentoo.py | 1 + cloudinit/distros/rhel.py | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 61904b6f..9c9ac384 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -53,7 +53,7 @@ class Distro(object): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" - init_cmd = [] # Not implemented + init_cmd = ['service'] # systemctl, service etc def __init__(self, name, cfg, paths): self._paths = paths diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 3b912728..1ae232fd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -48,7 +48,6 @@ class Distro(distros.Distro): network_conf_fn = "/etc/network/interfaces" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" - init_cmd = ['service'] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index ede40bff..565c68ed 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -37,6 +37,7 @@ class Distro(distros.Distro): network_conf_fn = "/etc/conf.d/net" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" + init_cmd = [''] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 7cc7a813..30195384 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -49,7 +49,6 @@ class Distro(distros.Distro): network_script_tpl = '/etc/sysconfig/network-scripts/ifcfg-%s' resolve_conf_fn = "/etc/resolv.conf" tz_local_fn = "/etc/localtime" - init_cmd = ['service'] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) -- cgit v1.2.3 From 309457a96a84132d2e570bf902c464b2a1aa9d0b Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 27 Jan 2014 12:22:21 -0600 Subject: Adds distro module exclude support --- cloudinit/config/cc_apt_configure.py | 2 ++ cloudinit/config/cc_apt_pipelining.py | 3 ++- cloudinit/config/cc_byobu.py | 2 ++ cloudinit/config/cc_ca_certs.py | 2 ++ cloudinit/config/cc_chef.py | 3 ++- cloudinit/config/cc_disk_setup.py | 2 ++ cloudinit/config/cc_emit_upstart.py | 2 ++ cloudinit/config/cc_growpart.py | 2 ++ cloudinit/config/cc_grub_dpkg.py | 3 +++ cloudinit/config/cc_keys_to_console.py | 2 ++ cloudinit/config/cc_landscape.py | 2 ++ cloudinit/config/cc_locale.py | 2 ++ cloudinit/config/cc_mcollective.py | 3 ++- cloudinit/config/cc_migrator.py | 2 ++ cloudinit/config/cc_mounts.py | 2 ++ cloudinit/config/cc_package_update_upgrade_install.py | 2 ++ cloudinit/config/cc_phone_home.py | 2 ++ cloudinit/config/cc_power_state_change.py | 3 ++- cloudinit/config/cc_puppet.py | 2 ++ cloudinit/config/cc_resizefs.py | 2 ++ cloudinit/config/cc_resolv_conf.py | 2 ++ cloudinit/config/cc_rightscale_userdata.py | 2 ++ cloudinit/config/cc_rsyslog.py | 2 ++ cloudinit/config/cc_salt_minion.py | 2 ++ cloudinit/config/cc_seed_random.py | 2 ++ cloudinit/config/cc_set_hostname.py | 2 ++ cloudinit/config/cc_set_passwords.py | 2 ++ cloudinit/config/cc_ssh.py | 3 ++- cloudinit/config/cc_ssh_authkey_fingerprints.py | 2 ++ cloudinit/config/cc_ssh_import_id.py | 3 ++- cloudinit/config/cc_timezone.py | 2 ++ cloudinit/config/cc_update_etc_hosts.py | 2 ++ cloudinit/config/cc_update_hostname.py | 2 ++ cloudinit/config/cc_users_groups.py | 2 ++ cloudinit/config/cc_yum_add_repo.py | 2 ++ cloudinit/distros/__init__.py | 8 ++++++++ cloudinit/distros/gentoo.py | 1 + 37 files changed, 80 insertions(+), 6 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 29c13a3d..a1be7514 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,6 +51,8 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index e5629175..15e509ac 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -35,7 +35,8 @@ APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" def handle(_name, cfg, _cloud, log, _args): - + if _cloud.is_excluded(_name): + return apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value_s = str(apt_pipe_value).lower().strip() diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 92d428b7..f708ca5f 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -29,6 +29,8 @@ distros = ['ubuntu', 'debian'] def handle(name, cfg, cloud, log, args): + if cloud.is_excluded(name): + return if len(args) != 0: value = args[0] else: diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 4f2a46a1..97af3705 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -79,6 +79,8 @@ def handle(name, cfg, _cloud, log, _args): @param args: Any module arguments from cloud.cfg """ # If there isn't a ca-certs section in the configuration don't do anything + if _cloud.is_excluded(name): + return if "ca-certs" not in cfg: log.debug(("Skipping module named %s," " no 'ca-certs' key in configuration"), name) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 727769cd..a74c4d27 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -40,7 +40,8 @@ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" def handle(name, cfg, cloud, log, _args): - + if cloud.is_excluded(name): + return # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 0b970e4e..7b5e2f3d 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -40,6 +40,8 @@ def handle(_name, cfg, cloud, log, _args): See doc/examples/cloud-config_disk-setup.txt for documentation on the format. """ + if cloud.is_excluded(_name): + return disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): update_disk_setup_devices(disk_setup, cloud.device_name_to_device) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 6d376184..4392c1f5 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -29,6 +29,8 @@ distros = ['ubuntu', 'debian'] def handle(name, _cfg, cloud, log, args): + if cloud.is_excluded(name): + return event_names = args if not event_names: # Default to the 'cloud-config' diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 6bddf847..91f472af 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -213,6 +213,8 @@ def resize_devices(resizer, devices): def handle(_name, cfg, _cloud, log, _args): + if _cloud.is_excluded(_name): + return if 'growpart' not in cfg: log.debug("No 'growpart' entry in cfg. Using default: %s" % DEFAULT_CONFIG) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index b3ce6fb6..ada0f472 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -26,9 +26,12 @@ distros = ['ubuntu', 'debian'] def handle(_name, cfg, _cloud, log, _args): + idevs = None idevs_empty = None + if _cloud.is_excluded(_name): + return if "grub-dpkg" in cfg: idevs = util.get_cfg_option_str(cfg["grub-dpkg"], "grub-pc/install_devices", None) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index ed7af690..ae47438a 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -30,6 +30,8 @@ HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints' def handle(name, cfg, _cloud, log, _args): + if _cloud.is_excluded(name): + return if not os.path.exists(HELPER_TOOL): log.warn(("Unable to activate module %s," " helper tool not found at %s"), name, HELPER_TOOL) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 8a709677..645b059d 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -56,6 +56,8 @@ def handle(_name, cfg, cloud, log, _args): ls_cloudcfg = cfg.get("landscape", {}) + if cloud.is_excluded(_name): + return if not isinstance(ls_cloudcfg, (dict)): raise RuntimeError(("'landscape' key existed in config," " but not a dictionary type," diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 6feaae9d..a3f0933a 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -22,6 +22,8 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): + if cloud.is_excluded(name): + return if len(args) != 0: locale = args[0] else: diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index b670390d..76e149f5 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -33,7 +33,8 @@ SERVER_CFG = '/etc/mcollective/server.cfg' def handle(name, cfg, cloud, log, _args): - + if cloud.is_excluded(name): + return # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: log.debug(("Skipping module named %s, " diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index facaa538..b9a012f5 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -75,6 +75,8 @@ def _migrate_legacy_sems(cloud, log): def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return do_migrate = util.get_cfg_option_str(cfg, "migrate", True) if not util.translate_bool(do_migrate): log.debug("Skipping module named %s, migration disabled", name) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 80590118..1476d19f 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -76,6 +76,8 @@ def sanitize_devname(startname, transformer, log): def handle(_name, cfg, cloud, log, _args): + if cloud.is_excluded(_name): + return # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] defvals = cfg.get("mount_default_fields", defvals) diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 73b0e30d..86e1d25b 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -49,6 +49,8 @@ def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): def handle(_name, cfg, cloud, log, _args): + if cloud.is_excluded(_name): + return # Handle the old style + new config names update = _multi_cfg_bool_get(cfg, 'apt_update', 'package_update') upgrade = _multi_cfg_bool_get(cfg, 'package_upgrade', 'apt_upgrade') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 2e058ccd..b2246807 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -44,6 +44,8 @@ POST_LIST_ALL = [ # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id # def handle(name, cfg, cloud, log, args): + if cloud.is_excluded(name): + return if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index e3150808..12ac2b92 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -31,7 +31,8 @@ EXIT_FAIL = 254 def handle(_name, cfg, _cloud, log, _args): - + if _cloud.is_excluded(_name): + return try: (args, timeout) = load_power_state(cfg) if args is None: diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 471a1a8a..35fbb251 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -49,6 +49,8 @@ def _autostart_puppet(log): def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 56040fdd..73a9217a 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -52,6 +52,8 @@ NOBLOCK = "noblock" def handle(name, cfg, _cloud, log, args): + if _cloud.is_excluded(name): + return if len(args) != 0: resize_root = args[0] else: diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 879b62b1..707402b7 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -92,6 +92,8 @@ def handle(name, cfg, _cloud, log, _args): @param log: Pre-initialized Python logger object to use for logging. @param args: Any module arguments from cloud.cfg """ + if _cloud.is_excluded(name): + return if "manage_resolv_conf" not in cfg: log.debug(("Skipping module named %s," " no 'manage_resolv_conf' key in configuration"), name) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index c771728d..b07e7a3e 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -50,6 +50,8 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): + if cloud.is_excluded(name): + return try: ud = cloud.get_userdata_raw() except: diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 0c2c6880..89587d9f 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -35,6 +35,8 @@ def handle(name, cfg, cloud, log, _args): # *.* @@syslogd.example.com # process 'rsyslog' + if cloud.is_excluded(name): + return if not 'rsyslog' in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 53013dcb..307e6a48 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -22,6 +22,8 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 22a31f29..2b695ee7 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -39,6 +39,8 @@ def _decode(data, encoding=None): def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return if not cfg or "random_seed" not in cfg: log.debug(("Skipping module named %s, " "no 'random_seed' configuration found"), name) diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 5d7f4331..fe8e5e47 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -22,6 +22,8 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not setting the hostname in module %s"), name) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 579735ed..fc76edab 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -36,6 +36,8 @@ PW_SET = (letters.translate(None, 'loLOI') + def handle(_name, cfg, cloud, log, args): + if cloud.is_excluded(_name): + return if len(args) != 0: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 64a5e3cb..0dc1d15d 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -56,7 +56,8 @@ KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' def handle(_name, cfg, cloud, log, _args): - + if cloud.is_excluded(_name): + return # remove the static keys from the pristine image if cfg.get("ssh_deletekeys", True): key_pth = os.path.join("/etc/ssh/", "ssh_host_*key*") diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index be8083db..948bbba0 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -92,6 +92,8 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return if 'no_ssh_fingerprints' in cfg: log.debug(("Skipping module named %s, " "logging of ssh fingerprints disabled"), name) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 50d96e15..51e0bc0b 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -32,7 +32,8 @@ distros = ['ubuntu'] def handle(_name, cfg, cloud, log, args): - + if cloud.is_excluded(_name): + return # import for "user: XXXXX" if len(args) != 0: user = args[0] diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index b9eb85b2..da195b0a 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -26,6 +26,8 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, log, args): + if cloud.is_excluded(name): + return if len(args) != 0: timezone = args[0] else: diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index d3dd1f32..ce0a3ae8 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -27,6 +27,8 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index e396ba13..5ee38630 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -27,6 +27,8 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): + if cloud.is_excluded(name): + return if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not updating the hostname in module %s"), name) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index bf5b4581..7ddd9f2d 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -27,6 +27,8 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, _log, _args): + if cloud.is_excluded(name): + return (users, groups) = ds.normalize_users_groups(cfg, cloud.distro) for (name, members) in groups.items(): cloud.distro.create_group(name, members) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 5c273825..93b570a0 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -58,6 +58,8 @@ def _format_repository_config(repo_id, repo_config): def handle(name, cfg, _cloud, log, _args): + if _cloud.is_excluded(name): + return repos = cfg.get('yum_repos') if not repos: log.debug(("Skipping module named %s," diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 9c9ac384..a6d1e6c6 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -54,12 +54,20 @@ class Distro(object): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ['service'] # systemctl, service etc + exclude_modules = [] def __init__(self, name, cfg, paths): self._paths = paths self._cfg = cfg self.name = name + def is_excluded(self, name): + if name in self.excluded_modules: + distro = getattr(self, name, None) or getattr(self, 'osfamily') + LOG.debug(("Skipping module named %s, distro excluded"), name, + distro) + return True + @abc.abstractmethod def install_packages(self, pkglist): raise NotImplementedError() diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 565c68ed..fbd96b36 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -38,6 +38,7 @@ class Distro(distros.Distro): tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" init_cmd = [''] + exclude_modules = ['grub_dpkg', 'apt_configure'] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) -- cgit v1.2.3 From 79d1eccc9fa751325fcb574fd9385a14bf2bbba6 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 27 Jan 2014 16:34:35 -0600 Subject: Removed excessive is_excluded module calls --- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_apt_pipelining.py | 2 +- cloudinit/config/cc_byobu.py | 3 +-- cloudinit/config/cc_ca_certs.py | 3 +-- cloudinit/config/cc_chef.py | 3 +-- cloudinit/config/cc_emit_upstart.py | 2 +- cloudinit/config/cc_grub_dpkg.py | 2 +- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_landscape.py | 2 -- cloudinit/config/cc_locale.py | 3 +-- cloudinit/config/cc_mcollective.py | 3 +-- cloudinit/config/cc_migrator.py | 3 +-- cloudinit/config/cc_mounts.py | 3 +-- cloudinit/config/cc_package_update_upgrade_install.py | 3 +-- cloudinit/config/cc_phone_home.py | 3 +-- cloudinit/config/cc_power_state_change.py | 3 +-- cloudinit/config/cc_resizefs.py | 3 +-- cloudinit/config/cc_resolv_conf.py | 3 +-- cloudinit/config/cc_rightscale_userdata.py | 3 +-- cloudinit/config/cc_rsyslog.py | 3 +-- cloudinit/config/cc_salt_minion.py | 3 +-- cloudinit/config/cc_yum_add_repo.py | 2 +- 22 files changed, 21 insertions(+), 38 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a1be7514..ccb45bb9 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,7 +51,7 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): + if cloud.distro.is_excluded(name): return release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index 15e509ac..bd180e82 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -35,7 +35,7 @@ APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" def handle(_name, cfg, _cloud, log, _args): - if _cloud.is_excluded(_name): + if _cloud.distro.is_excluded(_name): return apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value_s = str(apt_pipe_value).lower().strip() diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index f708ca5f..4821693b 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -29,8 +29,7 @@ distros = ['ubuntu', 'debian'] def handle(name, cfg, cloud, log, args): - if cloud.is_excluded(name): - return + if len(args) != 0: value = args[0] else: diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 97af3705..7b339274 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -79,8 +79,7 @@ def handle(name, cfg, _cloud, log, _args): @param args: Any module arguments from cloud.cfg """ # If there isn't a ca-certs section in the configuration don't do anything - if _cloud.is_excluded(name): - return + if "ca-certs" not in cfg: log.debug(("Skipping module named %s," " no 'ca-certs' key in configuration"), name) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index a74c4d27..727769cd 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -40,8 +40,7 @@ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 4392c1f5..5c2f6a31 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -29,7 +29,7 @@ distros = ['ubuntu', 'debian'] def handle(name, _cfg, cloud, log, args): - if cloud.is_excluded(name): + if cloud.distro.is_excluded(name): return event_names = args if not event_names: diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index ada0f472..3fd92426 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -30,7 +30,7 @@ def handle(_name, cfg, _cloud, log, _args): idevs = None idevs_empty = None - if _cloud.is_excluded(_name): + if _cloud.distro.is_excluded(_name): return if "grub-dpkg" in cfg: idevs = util.get_cfg_option_str(cfg["grub-dpkg"], diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index ae47438a..c9563e25 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -30,7 +30,7 @@ HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints' def handle(name, cfg, _cloud, log, _args): - if _cloud.is_excluded(name): + if _cloud.distro.is_excluded(name): return if not os.path.exists(HELPER_TOOL): log.warn(("Unable to activate module %s," diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 645b059d..8a709677 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -56,8 +56,6 @@ def handle(_name, cfg, cloud, log, _args): ls_cloudcfg = cfg.get("landscape", {}) - if cloud.is_excluded(_name): - return if not isinstance(ls_cloudcfg, (dict)): raise RuntimeError(("'landscape' key existed in config," " but not a dictionary type," diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index a3f0933a..68399993 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -22,8 +22,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): - if cloud.is_excluded(name): - return + if len(args) != 0: locale = args[0] else: diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 76e149f5..b670390d 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -33,8 +33,7 @@ SERVER_CFG = '/etc/mcollective/server.cfg' def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: log.debug(("Skipping module named %s, " diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index b9a012f5..f7b2211a 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -75,8 +75,7 @@ def _migrate_legacy_sems(cloud, log): def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + do_migrate = util.get_cfg_option_str(cfg, "migrate", True) if not util.translate_bool(do_migrate): log.debug("Skipping module named %s, migration disabled", name) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1476d19f..c1ac76d4 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -76,8 +76,7 @@ def sanitize_devname(startname, transformer, log): def handle(_name, cfg, cloud, log, _args): - if cloud.is_excluded(_name): - return + # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] defvals = cfg.get("mount_default_fields", defvals) diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 86e1d25b..85bc0240 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -49,8 +49,7 @@ def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): def handle(_name, cfg, cloud, log, _args): - if cloud.is_excluded(_name): - return + # Handle the old style + new config names update = _multi_cfg_bool_get(cfg, 'apt_update', 'package_update') upgrade = _multi_cfg_bool_get(cfg, 'package_upgrade', 'apt_upgrade') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index b2246807..360fd83e 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -44,8 +44,7 @@ POST_LIST_ALL = [ # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id # def handle(name, cfg, cloud, log, args): - if cloud.is_excluded(name): - return + if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 12ac2b92..e3150808 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -31,8 +31,7 @@ EXIT_FAIL = 254 def handle(_name, cfg, _cloud, log, _args): - if _cloud.is_excluded(_name): - return + try: (args, timeout) = load_power_state(cfg) if args is None: diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 73a9217a..43cc9307 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -52,8 +52,7 @@ NOBLOCK = "noblock" def handle(name, cfg, _cloud, log, args): - if _cloud.is_excluded(name): - return + if len(args) != 0: resize_root = args[0] else: diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 707402b7..feacebcd 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -92,8 +92,7 @@ def handle(name, cfg, _cloud, log, _args): @param log: Pre-initialized Python logger object to use for logging. @param args: Any module arguments from cloud.cfg """ - if _cloud.is_excluded(name): - return + if "manage_resolv_conf" not in cfg: log.debug(("Skipping module named %s," " no 'manage_resolv_conf' key in configuration"), name) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index b07e7a3e..f99524d1 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -50,8 +50,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + try: ud = cloud.get_userdata_raw() except: diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 89587d9f..7f70167e 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -35,8 +35,7 @@ def handle(name, cfg, cloud, log, _args): # *.* @@syslogd.example.com # process 'rsyslog' - if cloud.is_excluded(name): - return + if not 'rsyslog' in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 307e6a48..b48762c8 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -22,8 +22,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 93b570a0..f63e3e08 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -58,7 +58,7 @@ def _format_repository_config(repo_id, repo_config): def handle(name, cfg, _cloud, log, _args): - if _cloud.is_excluded(name): + if _cloud.distro.is_excluded(name): return repos = cfg.get('yum_repos') if not repos: -- cgit v1.2.3 From dd95d5e0a90031f19a68b255510476fb176126a6 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Tue, 28 Jan 2014 08:48:47 -0600 Subject: exclude modules fix 1 --- cloudinit/config/cc_disk_setup.py | 3 +-- cloudinit/config/cc_growpart.py | 3 +-- cloudinit/config/cc_puppet.py | 3 +-- cloudinit/config/cc_seed_random.py | 3 +-- cloudinit/config/cc_set_hostname.py | 3 +-- cloudinit/config/cc_set_passwords.py | 3 +-- cloudinit/config/cc_ssh.py | 3 +-- cloudinit/config/cc_ssh_authkey_fingerprints.py | 3 +-- cloudinit/config/cc_ssh_import_id.py | 3 +-- cloudinit/config/cc_timezone.py | 3 +-- cloudinit/config/cc_update_etc_hosts.py | 3 +-- cloudinit/config/cc_update_hostname.py | 3 +-- cloudinit/config/cc_users_groups.py | 3 +-- cloudinit/distros/__init__.py | 2 +- 14 files changed, 14 insertions(+), 27 deletions(-) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 7b5e2f3d..d0274ba6 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -40,8 +40,7 @@ def handle(_name, cfg, cloud, log, _args): See doc/examples/cloud-config_disk-setup.txt for documentation on the format. """ - if cloud.is_excluded(_name): - return + disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): update_disk_setup_devices(disk_setup, cloud.device_name_to_device) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 91f472af..03b53a3c 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -213,8 +213,7 @@ def resize_devices(resizer, devices): def handle(_name, cfg, _cloud, log, _args): - if _cloud.is_excluded(_name): - return + if 'growpart' not in cfg: log.debug("No 'growpart' entry in cfg. Using default: %s" % DEFAULT_CONFIG) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 35fbb251..717734d1 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -49,8 +49,7 @@ def _autostart_puppet(log): def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 2b695ee7..1676dd0a 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -39,8 +39,7 @@ def _decode(data, encoding=None): def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + if not cfg or "random_seed" not in cfg: log.debug(("Skipping module named %s, " "no 'random_seed' configuration found"), name) diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index fe8e5e47..38246b57 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -22,8 +22,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not setting the hostname in module %s"), name) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index fc76edab..b9dc0cc0 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -36,8 +36,7 @@ PW_SET = (letters.translate(None, 'loLOI') + def handle(_name, cfg, cloud, log, args): - if cloud.is_excluded(_name): - return + if len(args) != 0: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 0dc1d15d..64a5e3cb 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -56,8 +56,7 @@ KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' def handle(_name, cfg, cloud, log, _args): - if cloud.is_excluded(_name): - return + # remove the static keys from the pristine image if cfg.get("ssh_deletekeys", True): key_pth = os.path.join("/etc/ssh/", "ssh_host_*key*") diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 948bbba0..8b2708b1 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -92,8 +92,7 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + if 'no_ssh_fingerprints' in cfg: log.debug(("Skipping module named %s, " "logging of ssh fingerprints disabled"), name) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 51e0bc0b..50d96e15 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -32,8 +32,7 @@ distros = ['ubuntu'] def handle(_name, cfg, cloud, log, args): - if cloud.is_excluded(_name): - return + # import for "user: XXXXX" if len(args) != 0: user = args[0] diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index da195b0a..bddcd0e9 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -26,8 +26,7 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, log, args): - if cloud.is_excluded(name): - return + if len(args) != 0: timezone = args[0] else: diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index ce0a3ae8..3e3b4228 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -27,8 +27,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 5ee38630..56f6ebb7 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -27,8 +27,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): - if cloud.is_excluded(name): - return + if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not updating the hostname in module %s"), name) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 7ddd9f2d..30bf455d 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -27,8 +27,7 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, _log, _args): - if cloud.is_excluded(name): - return + (users, groups) = ds.normalize_users_groups(cfg, cloud.distro) for (name, members) in groups.items(): cloud.distro.create_group(name, members) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index a6d1e6c6..674c7293 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -62,7 +62,7 @@ class Distro(object): self.name = name def is_excluded(self, name): - if name in self.excluded_modules: + if name in self.exclude_modules: distro = getattr(self, name, None) or getattr(self, 'osfamily') LOG.debug(("Skipping module named %s, distro excluded"), name, distro) -- cgit v1.2.3 From 8a49ce7dd2b628291e6eef8a826308ddbe4f0520 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Tue, 28 Jan 2014 15:04:22 -0600 Subject: Updated exclude modules for gentoo distro. --- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/gentoo.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 674c7293..e364080f 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -64,7 +64,7 @@ class Distro(object): def is_excluded(self, name): if name in self.exclude_modules: distro = getattr(self, name, None) or getattr(self, 'osfamily') - LOG.debug(("Skipping module named %s, distro excluded"), name, + LOG.debug(("Skipping module named %s, distro %s excluded"), name, distro) return True diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index fbd96b36..8b0355be 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -38,7 +38,7 @@ class Distro(distros.Distro): tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" init_cmd = [''] - exclude_modules = ['grub_dpkg', 'apt_configure'] + exclude_modules = ['grub-dpkg', 'apt-configure', 'apt-pipelining'] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) -- cgit v1.2.3 From c104d6dfa464a8906c16b4f09b4b76ab5bf2e4e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:48:55 -0800 Subject: Add a openstack specific datasource Openstack has a unique derivative datasource that is gaining usage. Previously the config drive datasource provided part of this functionality as well as the ec2 datasource, but since new functionality is being added to openstack is seems benefical to combine the used parts into one datasource just made for handling openstack deployments. This patch factors out the common logic shared between the config drive and the openstack metadata datasource and places that in a shared helper file and then creates a new openstack datasource that readers from the openstack metadata service and refactors the config drive datasource to use this common logic. --- cloudinit/ec2_utils.py | 24 +- cloudinit/sources/DataSourceConfigDrive.py | 463 +++++---------------- cloudinit/url_helper.py | 64 ++- cloudinit/util.py | 10 + .../unittests/test_datasource/test_configdrive.py | 35 +- tests/unittests/test_ec2_util.py | 35 +- 6 files changed, 214 insertions(+), 417 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 92a22747..80736a8f 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -20,9 +20,9 @@ from urlparse import (urlparse, urlunparse) import functools import json -import urllib from cloudinit import log as logging +from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -37,16 +37,6 @@ def maybe_json_object(text): return False -def combine_url(base, add_on): - base_parsed = list(urlparse(base)) - path = base_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += urllib.quote(str(add_on), safe="/:") - base_parsed[2] = path - return urlunparse(base_parsed) - - # See: http://bit.ly/TyoUQs # class MetadataMaterializer(object): @@ -118,14 +108,14 @@ class MetadataMaterializer(object): (leaves, children) = self._parse(blob) child_contents = {} for c in children: - child_url = combine_url(base_url, c) + child_url = url_helper.combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" child_blob = str(self._caller(child_url)) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): - leaf_url = combine_url(base_url, resource) + leaf_url = url_helper.combine_url(base_url, resource) leaf_blob = str(self._caller(leaf_url)) leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob) joined = {} @@ -141,8 +131,8 @@ class MetadataMaterializer(object): def get_instance_userdata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): - ud_url = combine_url(metadata_address, api_version) - ud_url = combine_url(ud_url, 'user-data') + ud_url = url_helper.combine_url(metadata_address, api_version) + ud_url = url_helper.combine_url(ud_url, 'user-data') try: response = util.read_file_or_url(ud_url, ssl_details=ssl_details, @@ -157,8 +147,8 @@ def get_instance_userdata(api_version='latest', def get_instance_metadata(api_version='latest', metadata_address='http://169.254.169.254', ssl_details=None, timeout=5, retries=5): - md_url = combine_url(metadata_address, api_version) - md_url = combine_url(md_url, 'meta-data') + md_url = url_helper.combine_url(metadata_address, api_version) + md_url = url_helper.combine_url(md_url, 'meta-data') caller = functools.partial(util.read_file_or_url, ssl_details=ssl_details, timeout=timeout, retries=retries) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 4f437244..c45a1119 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,181 +18,79 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import base64 -import json import os from cloudinit import log as logging from cloudinit import sources from cloudinit import util +from cloudinit.sources.helpers import openstack + LOG = logging.getLogger(__name__) # Various defaults/constants... DEFAULT_IID = "iid-dsconfigdrive" DEFAULT_MODE = 'pass' -CFG_DRIVE_FILES_V1 = [ - "etc/network/interfaces", - "root/.ssh/authorized_keys", - "meta.js", -] DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } VALID_DSMODES = ("local", "net", "pass", "disabled") +FS_TYPES = ('vfat', 'iso9660') +LABEL_TYPES = ('config-2',) +OPTICAL_DEVICES = tuple(('/dev/sr%s' % i for i in range(0, 2))) -class ConfigDriveHelper(object): - def __init__(self, distro): - self.distro = distro - - def on_first_boot(self, data): - if not data: - data = {} - if 'network_config' in data: - LOG.debug("Updating network interfaces from config drive") - self.distro.apply_network(data['network_config']) - files = data.get('files') - if files: - LOG.debug("Writing %s injected files", len(files)) - try: - write_files(files) - except IOError: - util.logexc(LOG, "Failed writing files") - - -class DataSourceConfigDrive(sources.DataSource): +class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) + super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None - self.helper = ConfigDriveHelper(distro) + self.files = {} def __str__(self): root = sources.DataSource.__str__(self) - mstr = "%s [%s,ver=%s]" % (root, - self.dsmode, - self.version) + mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) mstr += "[source=%s]" % (self.source) return mstr - def _ec2_name_to_device(self, name): - if not self.ec2_metadata: - return None - bdm = self.ec2_metadata.get('block-device-mapping', {}) - for (ent_name, device) in bdm.items(): - if name == ent_name: - return device - return None - - def _os_name_to_device(self, name): - device = None - try: - criteria = 'LABEL=%s' % (name) - if name in ['swap']: - criteria = 'TYPE=%s' % (name) - dev_entries = util.find_devs_with(criteria) - if dev_entries: - device = dev_entries[0] - except util.ProcessExecutionError: - pass - return device - - def _validate_device_name(self, device): - if not device: - return None - if not device.startswith("/"): - device = "/dev/%s" % device - if os.path.exists(device): - return device - # Durn, try adjusting the mapping - remapped = self._remap_device(os.path.basename(device)) - if remapped: - LOG.debug("Remapped device name %s => %s", device, remapped) - return remapped - return None - - def device_name_to_device(self, name): - # Translate a 'name' to a 'physical' device - if not name: - return None - # Try the ec2 mapping first - names = [name] - if name == 'root': - names.insert(0, 'ami') - if name == 'ami': - names.append('root') - device = None - LOG.debug("Using ec2 metadata lookup to find device %s", names) - for n in names: - device = self._ec2_name_to_device(n) - device = self._validate_device_name(device) - if device: - break - # Try the openstack way second - if not device: - LOG.debug("Using os lookup to find device %s", names) - for n in names: - device = self._os_name_to_device(n) - device = self._validate_device_name(device) - if device: - break - # Ok give up... - if not device: - return None - else: - LOG.debug("Using cfg drive lookup mapped to device %s", device) - return device - def get_data(self): found = None md = {} - results = {} if os.path.isdir(self.seed_dir): try: - results = read_config_drive_dir(self.seed_dir) + results = read_config_drive(self.seed_dir) found = self.seed_dir - except NonConfigDriveDir: + except openstack.NonReadable: util.logexc(LOG, "Failed reading config drive from %s", self.seed_dir) if not found: - devlist = find_candidate_devs() - for dev in devlist: + for dev in find_candidate_devs(): try: - results = util.mount_cb(dev, read_config_drive_dir) + results = util.mount_cb(dev, read_config_drive) found = dev - break - except (NonConfigDriveDir, util.MountFailedError): + except openstack.NonReadable: pass - except BrokenConfigDriveDir: - util.logexc(LOG, "broken config drive: %s", dev) - + except util.MountFailedError: + pass + except openstack.BrokenMetadata: + util.logexc(LOG, "Broken config drive: %s", dev) + if found: + break if not found: return False - md = results['metadata'] + md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - - # Perform some metadata 'fixups' - # - # OpenStack uses the 'hostname' key - # while most of cloud-init uses the metadata - # 'local-hostname' key instead so if it doesn't - # exist we need to make sure its copied over. - for (tgt, src) in [('local-hostname', 'hostname')]: - if tgt not in md and src in md: - md[tgt] = md[src] - user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s" % user_dsmode) + LOG.warn("User specified invalid mode: %s" % user_dsmode) user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['cfgdrive_ver'], + dsmode = get_ds_mode(cfgdrv_ver=results['version'], ds_cfg=self.ds_cfg.get('dsmode'), user=user_dsmode) @@ -209,7 +107,7 @@ class DataSourceConfigDrive(sources.DataSource): prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] if prev_iid != cur_iid and self.dsmode == "local": - self.helper.on_first_boot(results) + on_first_boot(results, distro=self.distro) # dsmode != self.dsmode here if: # * dsmode = "pass", pass means it should only copy files and then @@ -225,16 +123,11 @@ class DataSourceConfigDrive(sources.DataSource): self.metadata = md self.ec2_metadata = results.get('ec2-metadata') self.userdata_raw = results.get('userdata') - self.version = results['cfgdrive_ver'] - + self.version = results['version'] + self.files.update(results.get('files', {})) + self.vendordata_raw = results.get('vendordata') return True - def get_public_ssh_keys(self): - name = "public_keys" - if self.version == 1: - name = "public-keys" - return sources.normalize_pubkey_data(self.metadata.get(name)) - class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -242,220 +135,6 @@ class DataSourceConfigDriveNet(DataSourceConfigDrive): self.dsmode = 'net' -class NonConfigDriveDir(Exception): - pass - - -class BrokenConfigDriveDir(Exception): - pass - - -def find_candidate_devs(): - """Return a list of devices that may contain the config drive. - - The returned list is sorted by search order where the first item has - should be searched first (highest priority) - - config drive v1: - Per documentation, this is "associated as the last available disk on the - instance", and should be VFAT. - Currently, we do not restrict search list to "last available disk" - - config drive v2: - Disk should be: - * either vfat or iso9660 formated - * labeled with 'config-2' - """ - - # Query optical drive to get it in blkid cache for 2.6 kernels - util.find_devs_with(path="/dev/sr0") - util.find_devs_with(path="/dev/sr1") - - by_fstype = (util.find_devs_with("TYPE=vfat") + - util.find_devs_with("TYPE=iso9660")) - by_label = util.find_devs_with("LABEL=config-2") - - # give preference to "last available disk" (vdb over vda) - # note, this is not a perfect rendition of that. - by_fstype.sort(reverse=True) - by_label.sort(reverse=True) - - # combine list of items by putting by-label items first - # followed by fstype items, but with dupes removed - combined = (by_label + [d for d in by_fstype if d not in by_label]) - - # We are looking for block device (sda, not sda1), ignore partitions - combined = [d for d in combined if not util.is_partition(d)] - - return combined - - -def read_config_drive_dir(source_dir): - last_e = NonConfigDriveDir("Not found") - for finder in (read_config_drive_dir_v2, read_config_drive_dir_v1): - try: - data = finder(source_dir) - return data - except NonConfigDriveDir as exc: - last_e = exc - raise last_e - - -def read_config_drive_dir_v2(source_dir, version="2012-08-10"): - - if (not os.path.isdir(os.path.join(source_dir, "openstack", version)) and - os.path.isdir(os.path.join(source_dir, "openstack", "latest"))): - LOG.warn("version '%s' not available, attempting to use 'latest'" % - version) - version = "latest" - - datafiles = ( - ('metadata', - "openstack/%s/meta_data.json" % version, True, json.loads), - ('userdata', "openstack/%s/user_data" % version, False, None), - ('ec2-metadata', "ec2/latest/meta-data.json", False, json.loads), - ) - - results = {'userdata': None} - for (name, path, required, process) in datafiles: - fpath = os.path.join(source_dir, path) - data = None - found = False - if os.path.isfile(fpath): - try: - data = util.load_file(fpath) - except IOError: - raise BrokenConfigDriveDir("Failed to read: %s" % fpath) - found = True - elif required: - raise NonConfigDriveDir("Missing mandatory path: %s" % fpath) - - if found and process: - try: - data = process(data) - except Exception as exc: - raise BrokenConfigDriveDir(("Failed to process " - "path: %s") % fpath) - - if found: - results[name] = data - - # instance-id is 'uuid' for openstack. just copy it to instance-id. - if 'instance-id' not in results['metadata']: - try: - results['metadata']['instance-id'] = results['metadata']['uuid'] - except KeyError: - raise BrokenConfigDriveDir("No uuid entry in metadata") - - if 'random_seed' in results['metadata']: - random_seed = results['metadata']['random_seed'] - try: - results['metadata']['random_seed'] = base64.b64decode(random_seed) - except (ValueError, TypeError) as exc: - raise BrokenConfigDriveDir("Badly formatted random_seed: %s" % exc) - - def read_content_path(item): - # do not use os.path.join here, as content_path starts with / - cpath = os.path.sep.join((source_dir, "openstack", - "./%s" % item['content_path'])) - return util.load_file(cpath) - - files = {} - try: - for item in results['metadata'].get('files', {}): - files[item['path']] = read_content_path(item) - - # the 'network_config' item in metadata is a content pointer - # to the network config that should be applied. - # in folsom, it is just a '/etc/network/interfaces' file. - item = results['metadata'].get("network_config", None) - if item: - results['network_config'] = read_content_path(item) - except Exception as exc: - raise BrokenConfigDriveDir("Failed to read file %s: %s" % (item, exc)) - - # to openstack, user can specify meta ('nova boot --meta=key=value') and - # those will appear under metadata['meta']. - # if they specify 'dsmode' they're indicating the mode that they intend - # for this datasource to operate in. - try: - results['dsmode'] = results['metadata']['meta']['dsmode'] - except KeyError: - pass - - results['files'] = files - results['cfgdrive_ver'] = 2 - return results - - -def read_config_drive_dir_v1(source_dir): - """ - read source_dir, and return a tuple with metadata dict, user-data, - files and version (1). If not a valid dir, raise a NonConfigDriveDir - """ - - found = {} - for af in CFG_DRIVE_FILES_V1: - fn = os.path.join(source_dir, af) - if os.path.isfile(fn): - found[af] = fn - - if len(found) == 0: - raise NonConfigDriveDir("%s: %s" % (source_dir, "no files found")) - - md = {} - keydata = "" - if "etc/network/interfaces" in found: - fn = found["etc/network/interfaces"] - md['network_config'] = util.load_file(fn) - - if "root/.ssh/authorized_keys" in found: - fn = found["root/.ssh/authorized_keys"] - keydata = util.load_file(fn) - - meta_js = {} - if "meta.js" in found: - fn = found['meta.js'] - content = util.load_file(fn) - try: - # Just check if its really json... - meta_js = json.loads(content) - if not isinstance(meta_js, (dict)): - raise TypeError("Dict expected for meta.js root node") - except (ValueError, TypeError) as e: - raise NonConfigDriveDir("%s: %s, %s" % - (source_dir, "invalid json in meta.js", e)) - md['meta_js'] = content - - # keydata in meta_js is preferred over "injected" - keydata = meta_js.get('public-keys', keydata) - if keydata: - lines = keydata.splitlines() - md['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] - - # config-drive-v1 has no way for openstack to provide the instance-id - # so we copy that into metadata from the user input - if 'instance-id' in meta_js: - md['instance-id'] = meta_js['instance-id'] - - results = {'cfgdrive_ver': 1, 'metadata': md} - - # allow the user to specify 'dsmode' in a meta tag - if 'dsmode' in meta_js: - results['dsmode'] = meta_js['dsmode'] - - # config-drive-v1 has no way of specifying user-data, so the user has - # to cheat and stuff it in a meta tag also. - results['userdata'] = meta_js.get('user-data') - - # this implementation does not support files - # (other than network/interfaces and authorized_keys) - results['files'] = [] - - return results - - def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): """Determine what mode should be used. valid values are 'pass', 'disabled', 'local', 'net' @@ -481,6 +160,21 @@ def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): return "net" +def read_config_drive(source_dir, version="2012-08-10"): + reader = openstack.ConfigDriveReader(source_dir) + finders = [ + (reader.read_v2, [], {'version': version}), + (reader.read_v1, [], {}), + ] + excps = [] + for (functor, args, kwargs) in finders: + try: + return functor(*args, **kwargs) + except openstack.NonReadable as e: + excps.append(e) + raise excps[-1] + + def get_previous_iid(paths): # interestingly, for this purpose the "previous" instance-id is the current # instance-id. cloud-init hasn't moved them over yet as this datasource @@ -492,17 +186,76 @@ def get_previous_iid(paths): return None -def write_files(files): - for (name, content) in files.iteritems(): - if name[0] != os.sep: - name = os.sep + name - util.write_file(name, content, mode=0660) +def on_first_boot(data, distro=None): + """Performs any first-boot actions using data read from a config-drive.""" + if not isinstance(data, dict): + raise TypeError("Config-drive data expected to be a dict; not %s" + % (type(data))) + net_conf = data.get("network_config", '') + if net_conf and distro: + LOG.debug("Updating network interfaces from config drive") + distro.apply_network(net_conf) + files = data.get('files', {}) + if files: + LOG.debug("Writing %s injected files", len(files)) + for (filename, content) in files.iteritems(): + if not filename.startswith(os.sep): + filename = os.sep + filename + try: + util.write_file(filename, content, mode=0660) + except IOError: + util.logexc(LOG, "Failed writing file: %s", filename) + + +def find_candidate_devs(probe_optical=True): + """Return a list of devices that may contain the config drive. + + The returned list is sorted by search order where the first item has + should be searched first (highest priority) + + config drive v1: + Per documentation, this is "associated as the last available disk on the + instance", and should be VFAT. + Currently, we do not restrict search list to "last available disk" + + config drive v2: + Disk should be: + * either vfat or iso9660 formated + * labeled with 'config-2' + """ + # query optical drive to get it in blkid cache for 2.6 kernels + if probe_optical: + for device in OPTICAL_DEVICES: + try: + util.find_devs_with(path=device) + except util.ProcessExecutionError: + pass + + by_fstype = [] + for fs_type in FS_TYPES: + by_fstype.extend(util.find_devs_with("TYPE=%s" % (fs_type))) + + by_label = [] + for label in LABEL_TYPES: + by_label.extend(util.find_devs_with("LABEL=%s" % (label))) + + # give preference to "last available disk" (vdb over vda) + # note, this is not a perfect rendition of that. + by_fstype.sort(reverse=True) + by_label.sort(reverse=True) + + # combine list of items by putting by-label items first + # followed by fstype items, but with dupes removed + combined = (by_label + [d for d in by_fstype if d not in by_label]) + + # we are looking for block device (sda, not sda1), ignore partitions + return [d for d in combined if not util.is_partition(d)] # Used to match classes to dependencies datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), + (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 19a30409..5c33d1e4 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -21,6 +21,7 @@ # along with this program. If not, see . import time +import urllib import requests from requests import exceptions @@ -58,6 +59,23 @@ def _cleanurl(url): return urlunparse(parsed_url) +def combine_url(base, *add_ons): + + def combine_single(url, add_on): + url_parsed = list(urlparse(url)) + path = url_parsed[2] + if path and not path.endswith("/"): + path += "/" + path += urllib.quote(str(add_on), safe="/:") + url_parsed[2] = path + return urlunparse(url_parsed) + + url = base + for add_on in add_ons: + url = combine_single(url, add_on) + return url + + class UrlResponse(object): def __init__(self, response): self._response = response @@ -101,30 +119,52 @@ class UrlError(IOError): self.headers = {} -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True): - url = _cleanurl(url) - req_args = { - 'url': url, - } +def _get_ssl_args(url, ssl_details): + ssl_args = {} scheme = urlparse(url).scheme # pylint: disable=E1101 if scheme == 'https' and ssl_details: if not SSL_ENABLED: LOG.warn("SSL is not enabled, cert. verification can not occur!") else: if 'ca_certs' in ssl_details and ssl_details['ca_certs']: - req_args['verify'] = ssl_details['ca_certs'] + ssl_args['verify'] = ssl_details['ca_certs'] else: - req_args['verify'] = True + ssl_args['verify'] = True if 'cert_file' in ssl_details and 'key_file' in ssl_details: - req_args['cert'] = [ssl_details['cert_file'], + ssl_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] elif 'cert_file' in ssl_details: - req_args['cert'] = str(ssl_details['cert_file']) + ssl_args['cert'] = str(ssl_details['cert_file']) + return ssl_args + +def existsurl(url, ssl_details=None, timeout=None): + r = _readurl(url, ssl_details=ssl_details, timeout=timeout, + method='HEAD', check_status=False) + return r.ok() + + +def readurl(url, data=None, timeout=None, retries=0, sec_between=1, + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True): + return _readurl(url, data=data, timeout=timeout, retries=retries, + sec_between=sec_between, headers=headers, + headers_cb=headers_cb, ssl_details=ssl_details, + check_status=check_status, + allow_redirects=allow_redirects) + + +def _readurl(url, data=None, timeout=None, retries=0, sec_between=1, + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True, method='GET'): + url = _cleanurl(url) + req_args = { + 'url': url, + } + req_args.update(_get_ssl_args(url, ssl_details)) + scheme = urlparse(url).scheme # pylint: disable=E1101 req_args['allow_redirects'] = allow_redirects - req_args['method'] = 'GET' + req_args['method'] = method if timeout is not None: req_args['timeout'] = max(float(timeout), 0) if data: diff --git a/cloudinit/util.py b/cloudinit/util.py index 3ce54f28..9bafd5b3 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -31,6 +31,7 @@ import glob import grp import gzip import hashlib +import json import os import os.path import platform @@ -385,6 +386,15 @@ def multi_log(text, console=True, stderr=True, log.log(log_level, text) +def load_json(text, root_types=(dict,)): + decoded = json.loads(text) + if not isinstance(decoded, tuple(root_types)): + expected_types = ", ".join([str(t) for t in root_types]) + raise TypeError("(%s) root types expected, got %s instead" + % (expected_types, type(decoded))) + return decoded + + def is_ipv4(instr): """determine if input string is a ipv4 address. return boolean.""" toks = instr.split('.') diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index d5935294..bd6cdd5d 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -9,6 +9,7 @@ from mocker import MockerTestCase from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds +from cloudinit.sources.helpers import openstack from cloudinit import util from tests.unittests import helpers as unit_helpers @@ -71,7 +72,7 @@ class TestConfigDriveDataSource(MockerTestCase): def test_ec2_metadata(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) self.assertTrue('ec2-metadata' in found) ec2_md = found['ec2-metadata'] self.assertEqual(EC2_META, ec2_md) @@ -81,7 +82,7 @@ class TestConfigDriveDataSource(MockerTestCase): cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) cfg_ds.metadata = found['metadata'] name_tests = { 'ami': '/dev/vda1', @@ -112,7 +113,7 @@ class TestConfigDriveDataSource(MockerTestCase): cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) os_md = found['metadata'] cfg_ds.metadata = os_md name_tests = { @@ -140,7 +141,7 @@ class TestConfigDriveDataSource(MockerTestCase): cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) ec2_md = found['ec2-metadata'] os_md = found['metadata'] cfg_ds.ec2_metadata = ec2_md @@ -165,13 +166,13 @@ class TestConfigDriveDataSource(MockerTestCase): my_mock.replay() device = cfg_ds.device_name_to_device(name) self.assertEquals(dev_name, device) - + def test_dev_ec2_map(self): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) exists_mock = self.mocker.replace(os.path.exists, spec=False, passthrough=False) exists_mock(mocker.ARGS) @@ -200,10 +201,11 @@ class TestConfigDriveDataSource(MockerTestCase): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) expected_md = copy(OSTACK_META) expected_md['instance-id'] = expected_md['uuid'] + expected_md['local-hostname'] = expected_md['hostname'] self.assertEqual(USER_DATA, found['userdata']) self.assertEqual(expected_md, found['metadata']) @@ -219,10 +221,11 @@ class TestConfigDriveDataSource(MockerTestCase): populate_dir(self.tmp, data) - found = ds.read_config_drive_dir(self.tmp) + found = ds.read_config_drive(self.tmp) expected_md = copy(OSTACK_META) expected_md['instance-id'] = expected_md['uuid'] + expected_md['local-hostname'] = expected_md['hostname'] self.assertEqual(expected_md, found['metadata']) @@ -235,8 +238,8 @@ class TestConfigDriveDataSource(MockerTestCase): populate_dir(self.tmp, data) - self.assertRaises(ds.BrokenConfigDriveDir, - ds.read_config_drive_dir, self.tmp) + self.assertRaises(openstack.BrokenMetadata, + ds.read_config_drive, self.tmp) def test_seed_dir_no_configdrive(self): """Verify that no metadata raises NonConfigDriveDir.""" @@ -247,14 +250,14 @@ class TestConfigDriveDataSource(MockerTestCase): data["openstack/latest/random-file.txt"] = "random-content" data["content/foo"] = "foocontent" - self.assertRaises(ds.NonConfigDriveDir, - ds.read_config_drive_dir, my_d) + self.assertRaises(openstack.NonReadable, + ds.read_config_drive, my_d) def test_seed_dir_missing(self): """Verify that missing seed_dir raises NonConfigDriveDir.""" my_d = os.path.join(self.tmp, "nonexistantdirectory") - self.assertRaises(ds.NonConfigDriveDir, - ds.read_config_drive_dir, my_d) + self.assertRaises(openstack.NonReadable, + ds.read_config_drive, my_d) def test_find_candidates(self): devs_with_answers = {} @@ -303,7 +306,7 @@ class TestConfigDriveDataSource(MockerTestCase): def cfg_ds_from_dir(seed_d): - found = ds.read_config_drive_dir(seed_d) + found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, helpers.Paths({})) populate_ds_from_read_config(cfg_ds, seed_d, found) @@ -318,7 +321,7 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.metadata = results.get('metadata') cfg_ds.ec2_metadata = results.get('ec2-metadata') cfg_ds.userdata_raw = results.get('userdata') - cfg_ds.version = results.get('cfgdrive_ver') + cfg_ds.version = results.get('version') def populate_dir(seed_dir, files): diff --git a/tests/unittests/test_ec2_util.py b/tests/unittests/test_ec2_util.py index dd588aca..18d36d86 100644 --- a/tests/unittests/test_ec2_util.py +++ b/tests/unittests/test_ec2_util.py @@ -1,6 +1,7 @@ from tests.unittests import helpers from cloudinit import ec2_utils as eu +from cloudinit import url_helper as uh import httpretty as hp @@ -40,11 +41,11 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'ami-launch-index'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'ami-launch-index'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'ami-launch-index'), status=200, body='1') md = eu.get_instance_metadata(self.VERSION, retries=0) self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') @@ -58,14 +59,14 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'public-keys/'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'), status=200, body='0=my-public-key') hp.register_uri(hp.GET, - eu.combine_url(base_url, 'public-keys/0/openssh-key'), + uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') @@ -79,18 +80,18 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'public-keys/'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'public-keys/'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'public-keys/'), status=200, body="\n".join(['0=my-public-key', '1=my-other-key'])) hp.register_uri(hp.GET, - eu.combine_url(base_url, 'public-keys/0/openssh-key'), + uh.combine_url(base_url, 'public-keys/0/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-public-key') hp.register_uri(hp.GET, - eu.combine_url(base_url, 'public-keys/1/openssh-key'), + uh.combine_url(base_url, 'public-keys/1/openssh-key'), status=200, body='ssh-rsa AAAA.....wZEf my-other-key') md = eu.get_instance_metadata(self.VERSION, retries=0, timeout=0.1) self.assertEquals(md['hostname'], 'ec2.fake.host.name.com') @@ -104,20 +105,20 @@ class TestEc2Util(helpers.TestCase): body="\n".join(['hostname', 'instance-id', 'block-device-mapping/'])) - hp.register_uri(hp.GET, eu.combine_url(base_url, 'hostname'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'hostname'), status=200, body='ec2.fake.host.name.com') - hp.register_uri(hp.GET, eu.combine_url(base_url, 'instance-id'), + hp.register_uri(hp.GET, uh.combine_url(base_url, 'instance-id'), status=200, body='123') hp.register_uri(hp.GET, - eu.combine_url(base_url, 'block-device-mapping/'), + uh.combine_url(base_url, 'block-device-mapping/'), status=200, body="\n".join(['ami', 'ephemeral0'])) hp.register_uri(hp.GET, - eu.combine_url(base_url, 'block-device-mapping/ami'), + uh.combine_url(base_url, 'block-device-mapping/ami'), status=200, body="sdb") hp.register_uri(hp.GET, - eu.combine_url(base_url, + uh.combine_url(base_url, 'block-device-mapping/ephemeral0'), status=200, body="sdc") -- cgit v1.2.3 From 810df2c55c108e7e4064263e508d9786d8b1dc8e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:58:58 -0800 Subject: Don't forget the rest of the files! --- cloudinit/sources/DataSourceOpenStack.py | 163 +++++++++ cloudinit/sources/helpers/__init__.py | 14 + cloudinit/sources/helpers/openstack.py | 420 ++++++++++++++++++++++ tests/unittests/test_datasource/test_openstack.py | 122 +++++++ 4 files changed, 719 insertions(+) create mode 100644 cloudinit/sources/DataSourceOpenStack.py create mode 100644 cloudinit/sources/helpers/__init__.py create mode 100644 cloudinit/sources/helpers/openstack.py create mode 100644 tests/unittests/test_datasource/test_openstack.py diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py new file mode 100644 index 00000000..44889f4e --- /dev/null +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -0,0 +1,163 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import time + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +from cloudinit.sources.helpers import openstack + +LOG = logging.getLogger(__name__) + +# Various defaults/constants... +DEF_MD_URL = "http://169.254.169.254" +DEFAULT_IID = "iid-dsopenstack" +DEFAULT_METADATA = { + "instance-id": DEFAULT_IID, +} +VALID_DSMODES = ("net", "disabled") + + +class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) + self.dsmode = 'net' + self.metadata_address = None + self.ssl_details = util.fetch_ssl_details(self.paths) + self.version = None + self.files = {} + + def __str__(self): + root = sources.DataSource.__str__(self) + mstr = "%s [%s,ver=%s]" % (root, self.dsmode, self.version) + return mstr + + def _get_url_settings(self): + # TODO(harlowja): this is shared with ec2 datasource, we should just + # move it to a shared location instead... + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + max_wait = 120 + try: + max_wait = int(ds_cfg.get("max_wait", max_wait)) + except Exception: + util.logexc(LOG, "Failed to get max wait. using %s", max_wait) + + timeout = 50 + try: + timeout = max(0, int(ds_cfg.get("timeout", timeout))) + except Exception: + util.logexc(LOG, "Failed to get timeout, using %s", timeout) + return (max_wait, timeout) + + def wait_for_metadata_service(self): + ds_cfg = self.ds_cfg + if not ds_cfg: + ds_cfg = {} + urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + filtered = [x for x in urls if util.is_resolvable_url(x)] + if set(filtered) != set(urls): + LOG.debug("Removed the following from metadata urls: %s", + list((set(urls) - set(filtered)))) + if len(filtered): + urls = filtered + else: + LOG.warn("Empty metadata url list! using default list") + urls = [DEF_MD_URL] + + md_urls = [] + url2base = {} + for url in urls: + md_url = url_helper.combine_url(url, 'openstack', + openstack.OS_LATEST, + 'meta_data.json') + md_urls.append(md_url) + url2base[md_url] = url + + (max_wait, timeout) = self._get_url_settings() + if max_wait <= 0: + return False + start_time = time.time() + avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, + timeout=timeout, + status_cb=LOG.warn) + if avail_url: + LOG.debug("Using metadata source: '%s'", url2base[avail_url]) + else: + LOG.critical("Giving up on md from %s after %s seconds", + md_urls, int(time.time() - start_time)) + + self.metadata_address = url2base.get(avail_url) + return bool(avail_url) + + def get_data(self): + try: + if not self.wait_for_metadata_service(): + return False + except IOError: + return False + + try: + results = util.log_time(LOG.debug, + 'Crawl of openstack metadata service', + read_metadata_service, + args=[self.metadata_address], + kwargs={'ssl_details': self.ssl_details}) + except openstack.NonReadable: + return False + except openstack.BrokenMetadata: + util.logexc(LOG, "Broken metadata address %s", + self.metadata_address) + return False + + user_dsmode = results.get('dsmode', None) + if user_dsmode not in VALID_DSMODES + (None,): + LOG.warn("User specified invalid mode: %s" % user_dsmode) + user_dsmode = None + if user_dsmode == 'disabled': + return False + + md = results.get('metadata', {}) + md = util.mergemanydict([md, DEFAULT_METADATA]) + self.metadata = md + self.ec2_metadata = results.get('ec2-metadata') + self.userdata_raw = results.get('userdata') + self.version = results['version'] + self.files.update(results.get('files', {})) + self.vendordata_raw = results.get('vendordata') + return True + + +def read_metadata_service(base_url, version=None, ssl_details=None): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) + return reader.read_v2(version=version) + + +# Used to match classes to dependencies +datasources = [ + (DataSourceOpenStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/helpers/__init__.py b/cloudinit/sources/helpers/__init__.py new file mode 100644 index 00000000..2cf99ec8 --- /dev/null +++ b/cloudinit/sources/helpers/__init__.py @@ -0,0 +1,14 @@ +# vi: ts=4 expandtab +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py new file mode 100644 index 00000000..9dbef677 --- /dev/null +++ b/cloudinit/sources/helpers/openstack.py @@ -0,0 +1,420 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import abc +import base64 +import copy +import functools +import os + +from cloudinit import ec2_utils +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import url_helper +from cloudinit import util + +# For reference: http://tinyurl.com/laora4c + +LOG = logging.getLogger(__name__) + +FILES_V1 = { + # Path <-> (metadata key name, translator function, default value) + 'etc/network/interfaces': ('network_config', lambda x: x, ''), + 'meta.js': ('meta_js', util.load_json, {}), + "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''), +} +KEY_COPIES = ( + # Cloud-init metadata names <-> (metadata key, is required) + ('local-hostname', 'hostname', False), + ('instance-id', 'uuid', True), +) +OS_VERSIONS = ( + '2012-08-10', # folsom + '2013-04-04', # grizzly + '2013-10-17', # havana +) +OS_LATEST = 'latest' + + +class NonReadable(IOError): + pass + + +class BrokenMetadata(IOError): + pass + + +class SourceMixin(object): + def _ec2_name_to_device(self, name): + if not self.ec2_metadata: + return None + bdm = self.ec2_metadata.get('block-device-mapping', {}) + for (ent_name, device) in bdm.items(): + if name == ent_name: + return device + return None + + def get_public_ssh_keys(self): + name = "public_keys" + if self.version == 1: + name = "public-keys" + return sources.normalize_pubkey_data(self.metadata.get(name)) + + def _os_name_to_device(self, name): + device = None + try: + criteria = 'LABEL=%s' % (name) + if name == 'swap': + criteria = 'TYPE=%s' % (name) + dev_entries = util.find_devs_with(criteria) + if dev_entries: + device = dev_entries[0] + except util.ProcessExecutionError: + pass + return device + + def _validate_device_name(self, device): + if not device: + return None + if not device.startswith("/"): + device = "/dev/%s" % device + if os.path.exists(device): + return device + # Durn, try adjusting the mapping + remapped = self._remap_device(os.path.basename(device)) + if remapped: + LOG.debug("Remapped device name %s => %s", device, remapped) + return remapped + return None + + def device_name_to_device(self, name): + # Translate a 'name' to a 'physical' device + if not name: + return None + # Try the ec2 mapping first + names = [name] + if name == 'root': + names.insert(0, 'ami') + if name == 'ami': + names.append('root') + device = None + LOG.debug("Using ec2 style lookup to find device %s", names) + for n in names: + device = self._ec2_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Try the openstack way second + if not device: + LOG.debug("Using openstack style lookup to find device %s", names) + for n in names: + device = self._os_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Ok give up... + if not device: + return None + else: + LOG.debug("Mapped %s to device %s", name, device) + return device + + +class BaseReader(object): + __metaclass__ = abc.ABCMeta + + def __init__(self, base_path): + self.base_path = base_path + + @abc.abstractmethod + def _path_join(self, base, *add_ons): + pass + + @abc.abstractmethod + def _path_exists(self, path): + pass + + @abc.abstractmethod + def _path_read(self, path): + pass + + @abc.abstractmethod + def _read_ec2_metadata(self): + pass + + def _read_content_path(self, item): + path = item.get('content_path', '').lstrip("/") + path_pieces = path.split("/") + valid_pieces = [p for p in path_pieces if len(p)] + if not valid_pieces: + raise BrokenMetadata("Item %s has no valid content path" % (item)) + path = self._path_join(self.base_path, "openstack", *path_pieces) + return self._path_read(path) + + def _find_working_version(self, version): + search_versions = [version] + list(OS_VERSIONS) + for potential_version in search_versions: + if not potential_version: + continue + path = self._path_join(self.base_path, "openstack", + potential_version) + if self._path_exists(path): + if potential_version != version: + LOG.warn("Version '%s' not available, attempting to use" + " version '%s' instead", version, + potential_version) + return potential_version + LOG.warn("Version '%s' not available, attempting to use '%s'" + " instead", version, OS_LATEST) + return OS_LATEST + + def read_v2(self, version=None): + """Reads a version 2 formatted location. + + Return a dict with metadata, userdata, ec2-metadata, dsmode, + network_config, files and version (2). + + If not a valid location, raise a NonReadable exception. + """ + + def datafiles(version): + files = {} + files['metadata'] = ( + # File path to read + self._path_join("openstack", version, 'meta_data.json'), + # Is it required? + True, + # Translator function (applied after loading) + util.load_json, + ) + files['userdata'] = ( + self._path_join("openstack", version, 'user_data'), + False, + lambda x: x, + ) + files['vendordata'] = ( + self._path_join("openstack", version, 'vendor_data.json'), + False, + util.load_json, + ) + return files + + version = self._find_working_version(version) + results = { + 'userdata': '', + 'version': 2, + } + data = datafiles(version) + for (name, (path, required, translator)) in data.iteritems(): + path = self._path_join(self.base_path, path) + data = None + found = False + if self._path_exists(path): + try: + data = self._path_read(path) + except IOError: + raise NonReadable("Failed to read: %s" % path) + found = True + else: + if required: + raise NonReadable("Missing mandatory path: %s" % path) + if found and translator: + try: + data = translator(data) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + if found: + results[name] = data + + metadata = results['metadata'] + if 'random_seed' in metadata: + random_seed = metadata['random_seed'] + try: + metadata['random_seed'] = base64.b64decode(random_seed) + except (ValueError, TypeError) as e: + raise BrokenMetadata("Badly formatted metadata" + " random_seed entry: %s" % e) + + # load any files that were provided + files = {} + metadata_files = metadata.get('files', []) + for item in metadata_files: + if 'path' not in item: + continue + path = item['path'] + try: + files[path] = self._read_content_path(item) + except Exception as e: + raise BrokenMetadata("Failed to read provided " + "file %s: %s" % (path, e)) + results['files'] = files + + # The 'network_config' item in metadata is a content pointer + # to the network config that should be applied. It is just a + # ubuntu/debian '/etc/network/interfaces' file. + net_item = metadata.get("network_config", None) + if net_item: + try: + results['network_config'] = self._read_content_path(net_item) + except IOError as e: + raise BrokenMetadata("Failed to read network" + " configuration: %s" % (e)) + + # To openstack, user can specify meta ('nova boot --meta=key=value') + # and those will appear under metadata['meta']. + # if they specify 'dsmode' they're indicating the mode that they intend + # for this datasource to operate in. + try: + results['dsmode'] = metadata['meta']['dsmode'] + except KeyError: + pass + + # Read any ec2-metadata (if applicable) + results['ec2-metadata'] = self._read_ec2_metadata() + + # Perform some misc. metadata key renames... + for (target_key, source_key, is_required) in KEY_COPIES: + if is_required and source_key not in metadata: + raise BrokenMetadata("No '%s' entry in metadata" % source_key) + if source_key in metadata: + metadata[target_key] = metadata.get(source_key) + return results + + +class ConfigDriveReader(BaseReader): + def __init__(self, base_path): + super(ConfigDriveReader, self).__init__(base_path) + + def _path_join(self, base, *add_ons): + components = [base] + list(add_ons) + return os.path.join(*components) + + def _path_exists(self, path): + return os.path.exists(path) + + def _path_read(self, path): + return util.load_file(path) + + def _read_ec2_metadata(self): + path = self._path_join(self.base_path, + 'ec2', 'latest', 'meta-data.json') + if not self._path_exists(path): + return {} + else: + try: + return util.load_json(self._path_read(path)) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + + def read_v1(self): + """Reads a version 1 formatted location. + + Return a dict with metadata, userdata, dsmode, files and version (1). + + If not a valid path, raise a NonReadable exception. + """ + + found = {} + for name in FILES_V1.keys(): + path = self._path_join(self.base_path, name) + if self._path_exists(path): + found[name] = path + if len(found) == 0: + raise NonReadable("%s: no files found" % (self.base_path)) + + md = {} + for (name, (key, translator, default)) in FILES_V1.iteritems(): + if name in found: + path = found[name] + try: + contents = self._path_read(path) + except IOError: + raise BrokenMetadata("Failed to read: %s" % path) + try: + md[key] = translator(contents) + except Exception as e: + raise BrokenMetadata("Failed to process " + "path %s: %s" % (path, e)) + else: + md[key] = copy.deepcopy(default) + + keydata = md['authorized_keys'] + meta_js = md['meta_js'] + + # keydata in meta_js is preferred over "injected" + keydata = meta_js.get('public-keys', keydata) + if keydata: + lines = keydata.splitlines() + md['public-keys'] = [l for l in lines + if len(l) and not l.startswith("#")] + + # config-drive-v1 has no way for openstack to provide the instance-id + # so we copy that into metadata from the user input + if 'instance-id' in meta_js: + md['instance-id'] = meta_js['instance-id'] + + results = { + 'version': 1, + 'metadata': md, + } + + # allow the user to specify 'dsmode' in a meta tag + if 'dsmode' in meta_js: + results['dsmode'] = meta_js['dsmode'] + + # config-drive-v1 has no way of specifying user-data, so the user has + # to cheat and stuff it in a meta tag also. + results['userdata'] = meta_js.get('user-data', '') + + # this implementation does not support files other than + # network/interfaces and authorized_keys... + results['files'] = {} + + return results + + +class MetadataReader(BaseReader): + def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): + super(MetadataReader, self).__init__(base_url) + self._url_reader = functools.partial(url_helper.readurl, + retries=retries, + ssl_details=ssl_details, + timeout=timeout) + self._url_checker = functools.partial(url_helper.existsurl, + ssl_details=ssl_details, + timeout=timeout) + self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, + ssl_details=ssl_details, + timeout=timeout, + retries=retries) + + def _path_read(self, path): + return str(self._url_reader(path)) + + def _path_exists(self, path): + return self._url_checker(path) + + def _path_join(self, base, *add_ons): + return url_helper.combine_url(base, *add_ons) + + def _read_ec2_metadata(self): + return self._ec2_reader() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py new file mode 100644 index 00000000..7d93f1d3 --- /dev/null +++ b/tests/unittests/test_datasource/test_openstack.py @@ -0,0 +1,122 @@ +import re +import json + +from StringIO import StringIO + +from urlparse import urlparse + +from tests.unittests import helpers + +from cloudinit.sources import DataSourceOpenStack as ds +from cloudinit.sources.helpers import openstack +from cloudinit import util + +import httpretty as hp + +BASE_URL = "http://169.254.169.254" +PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' +EC2_META = { + 'ami-id': 'ami-00000001', + 'ami-launch-index': 0, + 'ami-manifest-path': 'FIXME', + 'hostname': 'sm-foo-test.novalocal', + 'instance-action': 'none', + 'instance-id': 'i-00000001', + 'instance-type': 'm1.tiny', + 'local-hostname': 'sm-foo-test.novalocal', + 'local-ipv4': '0.0.0.0', + 'public-hostname': 'sm-foo-test.novalocal', + 'public-ipv4': '0.0.0.1', + 'reservation-id': 'r-iru5qm4m', +} +USER_DATA = '#!/bin/sh\necho This is user data\n' +VENDOR_DATA = { + 'magic': '', +} +OSTACK_META = { + 'availability_zone': 'nova', + 'files': [{'content_path': '/content/0000', 'path': '/etc/foo.cfg'}, + {'content_path': '/content/0001', 'path': '/etc/bar/bar.cfg'}], + 'hostname': 'sm-foo-test.novalocal', + 'meta': {'dsmode': 'local', 'my-meta': 'my-value'}, + 'name': 'sm-foo-test', + 'public_keys': {'mykey': PUBKEY}, + 'uuid': 'b0fa911b-69d4-4476-bbe2-1c92bff6535c', +} +CONTENT_0 = 'This is contents of /etc/foo.cfg\n' +CONTENT_1 = '# this is /etc/bar/bar.cfg\n' +OS_FILES = { + 'openstack/2012-08-10/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2012-08-10/user_data': USER_DATA, + 'openstack/content/0000': CONTENT_0, + 'openstack/content/0001': CONTENT_1, + 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/vendor_data.json': json.dumps(VENDOR_DATA), +} +EC2_FILES = { + 'latest/user-data': USER_DATA, +} + + +def _register_uris(version): + + def match_ec2_url(uri, headers): + path = uri.path.lstrip("/") + if path in EC2_FILES: + return (200, headers, EC2_FILES.get(path)) + if path == 'latest/meta-data': + buf = StringIO() + for (k, v) in EC2_META.items(): + if isinstance(v, (list, tuple)): + buf.write("%s/" % (k)) + else: + buf.write("%s" % (k)) + buf.write("\n") + return (200, headers, buf.getvalue()) + if path.startswith('latest/meta-data'): + value = None + pieces = path.split("/") + if path.endswith("/"): + pieces = pieces[2:-1] + value = util.get_cfg_by_path(EC2_META, pieces) + else: + pieces = pieces[2:] + value = util.get_cfg_by_path(EC2_META, pieces) + if value is not None: + return (200, headers, str(value)) + return (404, headers, '') + + def get_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + if path in OS_FILES: + return (200, headers, OS_FILES.get(path)) + return match_ec2_url(uri, headers) + + def head_request_callback(method, uri, headers): + uri = urlparse(uri) + path = uri.path.lstrip("/") + for key in OS_FILES.keys(): + if key.startswith(path): + return (200, headers, '') + return (404, headers, '') + + hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), + body=get_request_callback) + + hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), + body=head_request_callback) + + +class TestOpenStackDataSource(helpers.TestCase): + VERSION = 'latest' + + @hp.activate + def test_fetch(self): + _register_uris(self.VERSION) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(USER_DATA, f.get('userdata')) -- cgit v1.2.3 From bc77232588dd587849e8b3b7fc599387b6c905fa Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Sun, 2 Feb 2014 17:11:59 -0600 Subject: Net interface up updates and override ssh_svcname to init scripts --- cloudinit/distros/gentoo.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 8b0355be..74431bf6 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -37,8 +37,13 @@ class Distro(distros.Distro): network_conf_fn = "/etc/conf.d/net" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" - init_cmd = [''] - exclude_modules = ['grub-dpkg', 'apt-configure', 'apt-pipelining'] + init_cmd = [''] # init scripts + exclude_modules = [ + 'grub-dpkg', + 'apt-configure', + 'apt-pipelining', + 'yum-add-repo', + ] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) @@ -47,6 +52,8 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'gentoo' + # Fix sshd restarts + cfg['system_info']['ssh_svcname'] = '/etc/init.d/sshd' def apply_locale(self, locale, out_fn=None): if not out_fn: @@ -68,9 +75,8 @@ class Distro(distros.Distro): util.write_file(self.network_conf_fn, settings) return ['all'] - # TODO(NateH): Update to use init scripts def _bring_up_interface(self, device_name): - cmd = ['ifup', device_name] + cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: @@ -82,14 +88,24 @@ class Distro(distros.Distro): util.logexc(LOG, "Running interface command %s failed", cmd) return False - # TODO(NateH): Refactor for gentoo net init scripts def _bring_up_interfaces(self, device_names): use_all = False for d in device_names: if d == 'all': use_all = True if use_all: - return distros.Distro._bring_up_interface(self, '--all') + # Grab device names from init scripts + cmd = ['ls', '/etc/init.d/net.*'] + try: + (_out, err) = util.subp(cmd) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", cmd, + err) + except util.ProcessExecutionError: + util.logexc(LOG, "Running interface command %s failed", cmd) + return False + devices = [x.split('.')[2] for x in _out.split(' ')] + return distros.Distro._bring_up_interfaces(self, devices) else: return distros.Distro._bring_up_interfaces(self, device_names) -- cgit v1.2.3 From 29b838af062549ef5e2f12c3df74721d294dea37 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Mon, 3 Feb 2014 11:28:50 +0000 Subject: Add Google Compute Engine data source support. --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceGCE.py | 100 +++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 cloudinit/sources/DataSourceGCE.py diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 7be2199a..7c598bf9 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -36,6 +36,7 @@ CFG_BUILTIN = { 'AltCloud', 'OVF', 'MAAS', + 'GCE', 'Ec2', 'CloudStack', 'SmartOS', diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py new file mode 100644 index 00000000..b6a82245 --- /dev/null +++ b/cloudinit/sources/DataSourceGCE.py @@ -0,0 +1,100 @@ +# vi: ts=4 expandtab +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import requests + +from cloudinit import log as logging +from cloudinit import sources + +LOG = logging.getLogger(__name__) + +MD_URL = 'http://metadata/computeMetadata/v1/' + + +class DataSourceGCE(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.metadata_address = MD_URL + self.metadata = {} + + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + + def get_data(self): + # GCE metadata server requires a custom header since v1 + headers = {'X-Google-Metadata-Request': True} + + url_map = { + 'instance-id': self.metadata_address + 'instance/id', + 'availability-zone': self.metadata_address + 'instance/zone', + 'public-keys': self.metadata_address + 'project/attributes/sshKeys', + 'local-hostname': self.metadata_address + 'instance/hostname', + } + + with requests.Session() as s: + for mkey in url_map.iterkeys(): + try: + r = s.get(url_map[mkey], headers=headers) + except requests.exceptions.ConnectionError: + return False + if r.ok: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in r.text.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = r.text + else: + self.metadata[mkey] = None + return False + return True + + @property + def launch_index(self): + # GCE does not provide lauch_index property + return None + + def get_instance_id(self): + return self.metadata['instance-id'] + + def get_public_ssh_keys(self): + return self.metadata['public-keys'] + + def get_hostname(self, fqdn=False): + return self.metadata['local-hostname'] + + def get_userdata_raw(self): + return None + + @property + def availability_zone(self): + return self.metadata['instance-zone'] + +# Used to match classes to dependencies +datasources = [ + (DataSourceGCE, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) -- cgit v1.2.3 From b997a8cab8b3b10f9be7e915a5ab4c2cc91effb8 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 3 Feb 2014 11:37:36 -0600 Subject: Fix sshd restart --- cloudinit/distros/gentoo.py | 10 +++------- cloudinit/util.py | 1 + 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 74431bf6..e8778e15 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -1,12 +1,8 @@ # vi: ts=4 expandtab # -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. +# Copyright (C) 2014 Rackspace, US Inc. # -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow +# Author: Nate House # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -53,7 +49,7 @@ class Distro(distros.Distro): self._runner = helpers.Runners(paths) self.osfamily = 'gentoo' # Fix sshd restarts - cfg['system_info']['ssh_svcname'] = '/etc/init.d/sshd' + cfg['ssh_svcname'] = '/etc/init.d/sshd' def apply_locale(self, locale, out_fn=None): if not out_fn: diff --git a/cloudinit/util.py b/cloudinit/util.py index 3ce54f28..54fdad20 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1483,6 +1483,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, logstring=False): if rcs is None: rcs = [0] + args = filter(None, args) # Remove empty arguments try: if not logstring: -- cgit v1.2.3 From 4a0e460f18d8cbf2651286565efec1f00cbb20cd Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 3 Feb 2014 15:58:43 -0600 Subject: Update yum unittest --- cloudinit/config/cc_set_passwords.py | 3 ++- cloudinit/util.py | 1 - tests/unittests/test_handler/test_handler_yum_add_repo.py | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index b9dc0cc0..ab0ed5ad 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -137,9 +137,10 @@ def handle(_name, cfg, cloud, log, args): util.write_file(ssh_util.DEF_SSHD_CFG, "\n".join(lines)) try: - cmd = cloud.distro.init_cmd + cmd = cloud.distro.init_cmd # Default service cmd.append(cloud.distro.get_option('ssh_svcname', 'ssh')) cmd.append('restart') + cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") except: diff --git a/cloudinit/util.py b/cloudinit/util.py index 54fdad20..3ce54f28 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1483,7 +1483,6 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, logstring=False): if rcs is None: rcs = [0] - args = filter(None, args) # Remove empty arguments try: if not logstring: diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 8df592f9..ddf0ef9c 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -2,6 +2,7 @@ from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_yum_add_repo +from cloudinit.distros import rhel from tests.unittests import helpers @@ -18,6 +19,8 @@ class TestConfig(helpers.FilesystemMockingTestCase): def setUp(self): super(TestConfig, self).setUp() self.tmp = self.makeDir(prefix="unittest_") + self.cloud = type('', (), {})() + self.cloud.distro = rhel.Distro('test', {}, None) def test_bad_config(self): cfg = { @@ -34,7 +37,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) self.assertRaises(IOError, util.load_file, "/etc/yum.repos.d/epel_testing.repo") @@ -52,7 +55,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") contents = configobj.ConfigObj(StringIO(contents)) expected = { -- cgit v1.2.3 From 0efeb26736ddae2967c14a9440088594da32070d Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Mon, 3 Feb 2014 16:53:31 -0600 Subject: Added is_excluded tests and updated gentoo init script --- setup.py | 2 -- sysvinit/gentoo/cloud-init-local | 1 - tests/unittests/test_distros/test_is_excluded.py | 15 +++++++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 tests/unittests/test_distros/test_is_excluded.py diff --git a/setup.py b/setup.py index b0766970..9118e5f6 100755 --- a/setup.py +++ b/setup.py @@ -39,14 +39,12 @@ def is_f(p): INITSYS_FILES = { 'sysvinit': [f for f in glob('sysvinit/redhat/*') if is_f(f)], 'sysvinit_deb': [f for f in glob('sysvinit/debian/*') if is_f(f)], - 'sysvinit_gentoo': [f for f in glob('sysvinit/gentoo/*') if is_f(f)], 'systemd': [f for f in glob('systemd/*') if is_f(f)], 'upstart': [f for f in glob('upstart/*') if is_f(f)], } INITSYS_ROOTS = { 'sysvinit': '/etc/rc.d/init.d', 'sysvinit_deb': '/etc/init.d', - 'sysvinit_gentoo': '/etc/init.d', 'systemd': '/etc/systemd/system/', 'upstart': '/etc/init/', } diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local index 1d22a79d..8c9968d8 100644 --- a/sysvinit/gentoo/cloud-init-local +++ b/sysvinit/gentoo/cloud-init-local @@ -1,7 +1,6 @@ #!/sbin/runscript depend() { - after net # remove after nova-agent fix before cloud-init provide cloud-init-local } diff --git a/tests/unittests/test_distros/test_is_excluded.py b/tests/unittests/test_distros/test_is_excluded.py new file mode 100644 index 00000000..53a4445c --- /dev/null +++ b/tests/unittests/test_distros/test_is_excluded.py @@ -0,0 +1,15 @@ +from cloudinit.distros import gentoo +import unittest + + +class TestIsExcluded(unittest.TestCase): + + def setUp(self): + self.distro = gentoo.Distro('gentoo', {}, None) + self.distro.exclude_modules = ['test-module'] + + def test_is_excluded_success(self): + self.assertEqual(self.distro.is_excluded('test-module'), True) + + def test_is_excluded_fail(self): + self.assertEqual(self.distro.is_excluded('missing'), None) -- cgit v1.2.3 From 79741c91372c1101dac30b0c1fee32f6dabaff51 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Tue, 4 Feb 2014 12:32:57 -0600 Subject: Initial arch distro add. --- cloudinit/distros/__init__.py | 3 +- cloudinit/distros/arch.py | 186 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 cloudinit/distros/arch.py diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 46b67fa3..b58c8460 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -40,7 +40,8 @@ OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['fedora', 'rhel'], 'freebsd': ['freebsd'], - 'suse': ['sles'] + 'suse': ['sles'], + 'arch': ['arch'], } LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py new file mode 100644 index 00000000..808f9c14 --- /dev/null +++ b/cloudinit/distros/arch.py @@ -0,0 +1,186 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Rackspace, US Inc. +# +# Author: Nate House +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import distros +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util + +from cloudinit.distros.parsers.hostname import HostnameConf + +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + + +class Distro(distros.Distro): + locale_conf_fn = "/etc/locale.gen" + network_conf_dir = "/etc/netctl" + tz_conf_fn = "/etc/timezone" + tz_local_fn = "/etc/localtime" + init_cmd = ['systemctl'] # init scripts + exclude_modules = [ + 'grub-dpkg', + 'apt-configure', + 'apt-pipelining', + 'yum-add-repo', + ] + + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + self.osfamily = 'arch' + + def apply_locale(self, locale, out_fn=None): + if not out_fn: + out_fn = self.locale_conf_fn + util.subp(['locale-gen', '-G', locale], capture=False) + # "" provides trailing newline during join + lines = [ + util.make_header(), + 'LANG="%s"' % (locale), + "", + ] + util.write_file(out_fn, "\n".join(lines)) + + def install_packages(self, pkglist): + self.update_package_sources() + self.package_command('', pkgs=pkglist) + + def _write_network(self, settings): + util.write_file(self.network_conf_fn, settings) + return ['all'] + + # TODO(NateH): Fix for Arch's multi-file net config + def _bring_up_interface(self, device_name): + cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] + LOG.debug("Attempting to run bring up interface %s using command %s", + device_name, cmd) + try: + (_out, err) = util.subp(cmd) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", cmd, err) + return True + except util.ProcessExecutionError: + util.logexc(LOG, "Running interface command %s failed", cmd) + return False + + # TODO(NateH): Fix for Arch's multi-file net config + def _bring_up_interfaces(self, device_names): + use_all = False + for d in device_names: + if d == 'all': + use_all = True + if use_all: + # Grab device names from init scripts + cmd = ['ls', '/etc/init.d/net.*'] + try: + (_out, err) = util.subp(cmd) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", cmd, + err) + except util.ProcessExecutionError: + util.logexc(LOG, "Running interface command %s failed", cmd) + return False + devices = [x.split('.')[2] for x in _out.split(' ')] + return distros.Distro._bring_up_interfaces(self, devices) + else: + return distros.Distro._bring_up_interfaces(self, device_names) + + def _select_hostname(self, hostname, fqdn): + # Prefer the short hostname over the long + # fully qualified domain name + if not hostname: + return fqdn + return hostname + + def _write_hostname(self, your_hostname, out_fn): + conf = None + try: + # Try to update the previous one + # so lets see if we can read it first. + conf = self._read_hostname_conf(out_fn) + except IOError: + pass + if not conf: + conf = HostnameConf('') + conf.set_hostname(your_hostname) + util.write_file(out_fn, str(conf), 0644) + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) + return (self.hostname_conf_fn, sys_hostname) + + def _read_hostname_conf(self, filename): + conf = HostnameConf(util.load_file(filename)) + conf.parse() + return conf + + def _read_hostname(self, filename, default=None): + hostname = None + try: + conf = self._read_hostname_conf(filename) + hostname = conf.hostname + except IOError: + pass + if not hostname: + return default + return hostname + + def set_timezone(self, tz): + tz_file = self._find_tz_file(tz) + # Note: "" provides trailing newline during join + tz_lines = [ + util.make_header(), + str(tz), + "", + ] + util.write_file(self.tz_conf_fn, "\n".join(tz_lines)) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) + + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + + cmd = ['pacman'] + # Redirect output + cmd.append("-Sy") + cmd.append("--quiet") + cmd.append("--noconfirm") + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): + cmd.extend(args) + + if command: + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + + # Allow the output of this to flow outwards (ie not be captured) + util.subp(cmd, capture=False) + + def update_package_sources(self): + self._runner.run("update-sources", self.package_command, + ["-y"], freq=PER_INSTANCE) -- cgit v1.2.3 From 6922fc8294e38ee0780e9d74da7d3ec010a3cd3c Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Tue, 4 Feb 2014 15:59:44 -0600 Subject: Update ssh_svcname in init --- cloudinit/distros/arch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 808f9c14..d67a7bde 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -48,6 +48,7 @@ class Distro(distros.Distro): # should only happen say once per instance...) self._runner = helpers.Runners(paths) self.osfamily = 'arch' + cfg['ssh_svcname'] = 'sshd' def apply_locale(self, locale, out_fn=None): if not out_fn: -- cgit v1.2.3 From b541b4edf648e4ef0c5f66344459a0287ba36c60 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Wed, 5 Feb 2014 09:36:47 -0600 Subject: Cleanup and cloud-init-local local/net mount dep fix --- cloudinit/config/cc_byobu.py | 1 - cloudinit/config/cc_ca_certs.py | 1 - cloudinit/config/cc_disk_setup.py | 1 - cloudinit/config/cc_growpart.py | 1 - cloudinit/config/cc_grub_dpkg.py | 1 - cloudinit/config/cc_locale.py | 1 - cloudinit/config/cc_migrator.py | 1 - cloudinit/config/cc_mounts.py | 1 - cloudinit/config/cc_package_update_upgrade_install.py | 1 - cloudinit/config/cc_phone_home.py | 1 - cloudinit/config/cc_puppet.py | 1 - cloudinit/config/cc_resizefs.py | 1 - cloudinit/config/cc_resolv_conf.py | 1 - cloudinit/config/cc_rightscale_userdata.py | 1 - cloudinit/config/cc_rsyslog.py | 1 - cloudinit/config/cc_salt_minion.py | 1 - cloudinit/config/cc_seed_random.py | 1 - cloudinit/config/cc_set_hostname.py | 1 - cloudinit/config/cc_set_passwords.py | 1 - cloudinit/config/cc_ssh_authkey_fingerprints.py | 1 - cloudinit/config/cc_timezone.py | 1 - cloudinit/config/cc_update_etc_hosts.py | 1 - cloudinit/config/cc_update_hostname.py | 1 - cloudinit/config/cc_users_groups.py | 1 - sysvinit/gentoo/cloud-init-local | 1 + 25 files changed, 1 insertion(+), 24 deletions(-) diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 4821693b..92d428b7 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -29,7 +29,6 @@ distros = ['ubuntu', 'debian'] def handle(name, cfg, cloud, log, args): - if len(args) != 0: value = args[0] else: diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 7b339274..4f2a46a1 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -79,7 +79,6 @@ def handle(name, cfg, _cloud, log, _args): @param args: Any module arguments from cloud.cfg """ # If there isn't a ca-certs section in the configuration don't do anything - if "ca-certs" not in cfg: log.debug(("Skipping module named %s," " no 'ca-certs' key in configuration"), name) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d0274ba6..0b970e4e 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -40,7 +40,6 @@ def handle(_name, cfg, cloud, log, _args): See doc/examples/cloud-config_disk-setup.txt for documentation on the format. """ - disk_setup = cfg.get("disk_setup") if isinstance(disk_setup, dict): update_disk_setup_devices(disk_setup, cloud.device_name_to_device) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 4c5997fa..f52c41f0 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -255,7 +255,6 @@ def resize_devices(resizer, devices): def handle(_name, cfg, _cloud, log, _args): - if 'growpart' not in cfg: log.debug("No 'growpart' entry in cfg. Using default: %s" % DEFAULT_CONFIG) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 3fd92426..03cdd98c 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -26,7 +26,6 @@ distros = ['ubuntu', 'debian'] def handle(_name, cfg, _cloud, log, _args): - idevs = None idevs_empty = None diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 68399993..6feaae9d 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -22,7 +22,6 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): - if len(args) != 0: locale = args[0] else: diff --git a/cloudinit/config/cc_migrator.py b/cloudinit/config/cc_migrator.py index f7b2211a..facaa538 100644 --- a/cloudinit/config/cc_migrator.py +++ b/cloudinit/config/cc_migrator.py @@ -75,7 +75,6 @@ def _migrate_legacy_sems(cloud, log): def handle(name, cfg, cloud, log, _args): - do_migrate = util.get_cfg_option_str(cfg, "migrate", True) if not util.translate_bool(do_migrate): log.debug("Skipping module named %s, migration disabled", name) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index c1ac76d4..80590118 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -76,7 +76,6 @@ def sanitize_devname(startname, transformer, log): def handle(_name, cfg, cloud, log, _args): - # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] defvals = cfg.get("mount_default_fields", defvals) diff --git a/cloudinit/config/cc_package_update_upgrade_install.py b/cloudinit/config/cc_package_update_upgrade_install.py index 85bc0240..73b0e30d 100644 --- a/cloudinit/config/cc_package_update_upgrade_install.py +++ b/cloudinit/config/cc_package_update_upgrade_install.py @@ -49,7 +49,6 @@ def _fire_reboot(log, wait_attempts=6, initial_sleep=1, backoff=2): def handle(_name, cfg, cloud, log, _args): - # Handle the old style + new config names update = _multi_cfg_bool_get(cfg, 'apt_update', 'package_update') upgrade = _multi_cfg_bool_get(cfg, 'package_upgrade', 'apt_upgrade') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 360fd83e..2e058ccd 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -44,7 +44,6 @@ POST_LIST_ALL = [ # post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id # def handle(name, cfg, cloud, log, args): - if len(args) != 0: ph_cfg = util.read_conf(args[0]) else: diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 717734d1..471a1a8a 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -49,7 +49,6 @@ def _autostart_puppet(log): def handle(name, cfg, cloud, log, _args): - # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 9d767873..be406034 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -76,7 +76,6 @@ def rootdev_from_cmdline(cmdline): def handle(name, cfg, _cloud, log, args): - if len(args) != 0: resize_root = args[0] else: diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index feacebcd..879b62b1 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -92,7 +92,6 @@ def handle(name, cfg, _cloud, log, _args): @param log: Pre-initialized Python logger object to use for logging. @param args: Any module arguments from cloud.cfg """ - if "manage_resolv_conf" not in cfg: log.debug(("Skipping module named %s," " no 'manage_resolv_conf' key in configuration"), name) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index f99524d1..c771728d 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -50,7 +50,6 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): - try: ud = cloud.get_userdata_raw() except: diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 7f70167e..0c2c6880 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -35,7 +35,6 @@ def handle(name, cfg, cloud, log, _args): # *.* @@syslogd.example.com # process 'rsyslog' - if not 'rsyslog' in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index b48762c8..53013dcb 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -22,7 +22,6 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): - # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: log.debug(("Skipping module named %s," diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 1676dd0a..22a31f29 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -39,7 +39,6 @@ def _decode(data, encoding=None): def handle(name, cfg, cloud, log, _args): - if not cfg or "random_seed" not in cfg: log.debug(("Skipping module named %s, " "no 'random_seed' configuration found"), name) diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 38246b57..5d7f4331 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -22,7 +22,6 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not setting the hostname in module %s"), name) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index ab0ed5ad..646c3f8b 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -36,7 +36,6 @@ PW_SET = (letters.translate(None, 'loLOI') + def handle(_name, cfg, cloud, log, args): - if len(args) != 0: # if run from command line, and give args, wipe the chpasswd['list'] password = args[0] diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 8b2708b1..be8083db 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -92,7 +92,6 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5', def handle(name, cfg, cloud, log, _args): - if 'no_ssh_fingerprints' in cfg: log.debug(("Skipping module named %s, " "logging of ssh fingerprints disabled"), name) diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index bddcd0e9..b9eb85b2 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -26,7 +26,6 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, log, args): - if len(args) != 0: timezone = args[0] else: diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 3e3b4228..d3dd1f32 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -27,7 +27,6 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): - manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 56f6ebb7..e396ba13 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -27,7 +27,6 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): - if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," " not updating the hostname in module %s"), name) diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 30bf455d..bf5b4581 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -27,7 +27,6 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, _log, _args): - (users, groups) = ds.normalize_users_groups(cfg, cloud.distro) for (name, members) in groups.items(): cloud.distro.create_group(name, members) diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local index 8c9968d8..fef2cbb9 100644 --- a/sysvinit/gentoo/cloud-init-local +++ b/sysvinit/gentoo/cloud-init-local @@ -1,6 +1,7 @@ #!/sbin/runscript depend() { + after localmount netmount before cloud-init provide cloud-init-local } -- cgit v1.2.3 From d57f684e62b00d3fc4f3b7f018a566d7804b7440 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Wed, 5 Feb 2014 11:21:46 -0600 Subject: Init script after consistency --- sysvinit/gentoo/cloud-init-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local index fef2cbb9..9d47263e 100644 --- a/sysvinit/gentoo/cloud-init-local +++ b/sysvinit/gentoo/cloud-init-local @@ -1,7 +1,8 @@ #!/sbin/runscript depend() { - after localmount netmount + after localmount + after netmount before cloud-init provide cloud-init-local } -- cgit v1.2.3 From 805ac503531d27651f0ad4b1a590a488545a0887 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 6 Feb 2014 09:59:04 -0600 Subject: Removed exclude bits from yum module as its not a default --- cloudinit/config/cc_yum_add_repo.py | 2 -- cloudinit/distros/gentoo.py | 2 +- tests/unittests/test_handler/test_handler_yum_add_repo.py | 7 ++----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index f63e3e08..5c273825 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -58,8 +58,6 @@ def _format_repository_config(repo_id, repo_config): def handle(name, cfg, _cloud, log, _args): - if _cloud.distro.is_excluded(name): - return repos = cfg.get('yum_repos') if not repos: log.debug(("Skipping module named %s," diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index e8778e15..0087908a 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -181,4 +181,4 @@ class Distro(distros.Distro): def update_package_sources(self): self._runner.run("update-sources", self.package_command, - ["-u", "world", "--quiet"], freq=PER_INSTANCE) + ["-u", "world"], freq=PER_INSTANCE) diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index ddf0ef9c..8df592f9 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -2,7 +2,6 @@ from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_yum_add_repo -from cloudinit.distros import rhel from tests.unittests import helpers @@ -19,8 +18,6 @@ class TestConfig(helpers.FilesystemMockingTestCase): def setUp(self): super(TestConfig, self).setUp() self.tmp = self.makeDir(prefix="unittest_") - self.cloud = type('', (), {})() - self.cloud.distro = rhel.Distro('test', {}, None) def test_bad_config(self): cfg = { @@ -37,7 +34,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) self.assertRaises(IOError, util.load_file, "/etc/yum.repos.d/epel_testing.repo") @@ -55,7 +52,7 @@ class TestConfig(helpers.FilesystemMockingTestCase): }, } self.patchUtils(self.tmp) - cc_yum_add_repo.handle('yum_add_repo', cfg, self.cloud, LOG, []) + cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") contents = configobj.ConfigObj(StringIO(contents)) expected = { -- cgit v1.2.3 From 122085edf26508d7f9cdf930c8a31f089159ccf5 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 6 Feb 2014 15:35:34 -0600 Subject: Added arch network config --- cloudinit/distros/arch.py | 82 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 22 deletions(-) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index d67a7bde..fc5eee66 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -21,6 +21,7 @@ from cloudinit import helpers from cloudinit import log as logging from cloudinit import util +from cloudinit.distros import net_util from cloudinit.distros.parsers.hostname import HostnameConf from cloudinit.settings import PER_INSTANCE @@ -33,6 +34,7 @@ class Distro(distros.Distro): network_conf_dir = "/etc/netctl" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" + resolv_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts exclude_modules = [ 'grub-dpkg', @@ -67,12 +69,46 @@ class Distro(distros.Distro): self.package_command('', pkgs=pkglist) def _write_network(self, settings): - util.write_file(self.network_conf_fn, settings) - return ['all'] + entries = net_util.translate_network(settings) + LOG.debug("Translated ubuntu style network settings %s into %s", + settings, entries) + dev_names = entries.keys() + # Format for netctl + for (dev, info) in entries.iteritems(): + nameservers = [] + net_fn = self.network_conf_dir + dev + net_cfg = { + 'Connection': 'ethernet', + 'Interface': dev, + 'IP': info.get('bootproto'), + 'Address': "('%s/%s')" % (info.get('address'), + info.get('netmask')), + 'Gateway': info.get('gateway'), + 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') + } + util.write_file(net_fn, convert_netctl(net_cfg)) + if info.get('auto'): + self._enable_interface(dev) + if 'dns-nameservers' in info: + nameservers.extend(info['dns-nameservers']) + + if nameservers: + util.write_file(self.resolve_conf_fn, + convert_resolv_conf(nameservers)) + + return dev_names + + def _enable_interface(self, device_name): + cmd = ['netctl', 'reenable', device_name] + try: + (_out, err) = util.subp(cmd) + if len(err): + LOG.warn("Running %s resulted in stderr output: %s", cmd, err) + except util.ProcessExecutionError: + util.logexc(LOG, "Running interface command %s failed", cmd) - # TODO(NateH): Fix for Arch's multi-file net config def _bring_up_interface(self, device_name): - cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] + cmd = ['netctl', 'restart', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", device_name, cmd) try: @@ -84,27 +120,11 @@ class Distro(distros.Distro): util.logexc(LOG, "Running interface command %s failed", cmd) return False - # TODO(NateH): Fix for Arch's multi-file net config def _bring_up_interfaces(self, device_names): - use_all = False for d in device_names: - if d == 'all': - use_all = True - if use_all: - # Grab device names from init scripts - cmd = ['ls', '/etc/init.d/net.*'] - try: - (_out, err) = util.subp(cmd) - if len(err): - LOG.warn("Running %s resulted in stderr output: %s", cmd, - err) - except util.ProcessExecutionError: - util.logexc(LOG, "Running interface command %s failed", cmd) + if not self._bring_up_interface(d): return False - devices = [x.split('.')[2] for x in _out.split(' ')] - return distros.Distro._bring_up_interfaces(self, devices) - else: - return distros.Distro._bring_up_interfaces(self, device_names) + return True def _select_hostname(self, hostname, fqdn): # Prefer the short hostname over the long @@ -185,3 +205,21 @@ class Distro(distros.Distro): def update_package_sources(self): self._runner.run("update-sources", self.package_command, ["-y"], freq=PER_INSTANCE) + + +def convert_netctl(settings): + """Returns a settings string formatted for netctl.""" + result = '' + if isinstance(settings, dict): + for k, v in settings.items(): + result = result + '%s=%s\n' % (k, v) + return result + + +def convert_resolv_conf(settings): + """Returns a settings string formatted for resolv.conf.""" + result = '' + if isinstance(settings, list): + for ns in list: + result = result + 'nameserver %s\n' % ns + return result -- cgit v1.2.3 From 003d3903dcd2c023a37f99ca74a7ecc56243b449 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 6 Feb 2014 15:42:51 -0600 Subject: Removed exclude conditional on keys-to-console --- cloudinit/config/cc_keys_to_console.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index c9563e25..ed7af690 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -30,8 +30,6 @@ HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints' def handle(name, cfg, _cloud, log, _args): - if _cloud.distro.is_excluded(name): - return if not os.path.exists(HELPER_TOOL): log.warn(("Unable to activate module %s," " helper tool not found at %s"), name, HELPER_TOOL) -- cgit v1.2.3 From 65da76341796a00b7bbdca514167b89f99d5a599 Mon Sep 17 00:00:00 2001 From: "Nate House nathan.house@rackspace.com" <> Date: Thu, 6 Feb 2014 15:49:23 -0600 Subject: Removed yum exclude module entry from gentoo distro --- cloudinit/distros/gentoo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 0087908a..0a95fa23 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -38,7 +38,6 @@ class Distro(distros.Distro): 'grub-dpkg', 'apt-configure', 'apt-pipelining', - 'yum-add-repo', ] def __init__(self, name, cfg, paths): -- cgit v1.2.3 From e1710e06fd09c3f643fdca851c011da3a3eb7a98 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 09:43:32 +0000 Subject: use url_helper instead of requests --- cloudinit/sources/DataSourceGCE.py | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index b6a82245..f9cdc1da 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import requests from cloudinit import log as logging from cloudinit import sources +from cloudinit import url_helper LOG = logging.getLogger(__name__) @@ -30,16 +30,6 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} - # GCE takes sshKeys attribute in the format of ':' - # so we have to trim each key to remove the username part - def _trim_key(self, public_key): - try: - index = public_key.index(':') - if index > 0: - return public_key[(index + 1):] - except: - return public_key - def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} @@ -51,22 +41,18 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } - with requests.Session() as s: - for mkey in url_map.iterkeys(): - try: - r = s.get(url_map[mkey], headers=headers) - except requests.exceptions.ConnectionError: - return False - if r.ok: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in r.text.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = r.text + for mkey in url_map.iterkeys(): + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + if resp.ok(): + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys else: - self.metadata[mkey] = None - return False - return True + self.metadata[mkey] = resp.contents + else: + self.metadata[mkey] = None + return False + return True @property def launch_index(self): -- cgit v1.2.3 From 64b35a9c3b201aed5073823fd4a15ecb15195aea Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Fri, 7 Feb 2014 13:12:48 +0000 Subject: Forgot to include _trim_key function Got removed somehow --- cloudinit/sources/DataSourceGCE.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index f9cdc1da..8a46f933 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -30,6 +30,16 @@ class DataSourceGCE(sources.DataSource): self.metadata_address = MD_URL self.metadata = {} + # GCE takes sshKeys attribute in the format of ':' + # so we have to trim each key to remove the username part + def _trim_key(self, public_key): + try: + index = public_key.index(':') + if index > 0: + return public_key[(index + 1):] + except: + return public_key + def get_data(self): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} -- cgit v1.2.3 From f9582464b157ffb0087b18910af11aa6ed49abcd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:34:04 -0800 Subject: Add a bunch of new tests --- cloudinit/sources/DataSourceOpenStack.py | 3 +- tests/unittests/helpers.py | 6 + tests/unittests/test_datasource/test_openstack.py | 197 ++++++++++++++++++++-- 3 files changed, 190 insertions(+), 16 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 44889f4e..621572de 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -121,7 +121,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'version': openstack.OS_LATEST}) except openstack.NonReadable: return False except openstack.BrokenMetadata: diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 5b4f4208..945c1500 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -29,6 +29,12 @@ if (_PY_MAJOR, _PY_MINOR) <= (2, 6): standardMsg = standardMsg % (member, container) self.fail(self._formatMessage(msg, standardMsg)) + def assertIsNone(self, value, msg=None): + if value is not None: + standardMsg = '%r is not None' + standardMsg = standardMsg % (value) + self.fail(self._formatMessage(msg, standardMsg)) + else: class TestCase(unittest.TestCase): pass diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 7d93f1d3..c8cc40db 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -1,12 +1,33 @@ -import re +# vi: ts=4 expandtab +# +# Copyright (C) 2014 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import copy import json +import re from StringIO import StringIO from urlparse import urlparse -from tests.unittests import helpers +from tests.unittests import helpers as test_helpers +from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceOpenStack as ds from cloudinit.sources.helpers import openstack from cloudinit import util @@ -17,7 +38,7 @@ BASE_URL = "http://169.254.169.254" PUBKEY = u'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460\n' EC2_META = { 'ami-id': 'ami-00000001', - 'ami-launch-index': 0, + 'ami-launch-index': '0', 'ami-manifest-path': 'FIXME', 'hostname': 'sm-foo-test.novalocal', 'instance-action': 'none', @@ -59,15 +80,17 @@ EC2_FILES = { } -def _register_uris(version): +def _register_uris(version, ec2_files, ec2_meta, os_files): + """Registers a set of url patterns into httpretty that will mimic the + same data returned by the openstack metadata service (and ec2 service).""" def match_ec2_url(uri, headers): path = uri.path.lstrip("/") - if path in EC2_FILES: - return (200, headers, EC2_FILES.get(path)) + if path in ec2_files: + return (200, headers, ec2_files.get(path)) if path == 'latest/meta-data': buf = StringIO() - for (k, v) in EC2_META.items(): + for (k, v) in ec2_meta.items(): if isinstance(v, (list, tuple)): buf.write("%s/" % (k)) else: @@ -79,10 +102,10 @@ def _register_uris(version): pieces = path.split("/") if path.endswith("/"): pieces = pieces[2:-1] - value = util.get_cfg_by_path(EC2_META, pieces) + value = util.get_cfg_by_path(ec2_meta, pieces) else: pieces = pieces[2:] - value = util.get_cfg_by_path(EC2_META, pieces) + value = util.get_cfg_by_path(ec2_meta, pieces) if value is not None: return (200, headers, str(value)) return (404, headers, '') @@ -90,14 +113,14 @@ def _register_uris(version): def get_request_callback(method, uri, headers): uri = urlparse(uri) path = uri.path.lstrip("/") - if path in OS_FILES: - return (200, headers, OS_FILES.get(path)) + if path in os_files: + return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) def head_request_callback(method, uri, headers): uri = urlparse(uri) path = uri.path.lstrip("/") - for key in OS_FILES.keys(): + for key in os_files.keys(): if key.startswith(path): return (200, headers, '') return (404, headers, '') @@ -109,14 +132,158 @@ def _register_uris(version): body=head_request_callback) -class TestOpenStackDataSource(helpers.TestCase): +class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' @hp.activate - def test_fetch(self): - _register_uris(self.VERSION) + def test_successful(self): + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertEquals(2, len(f['files'])) + self.assertEquals(USER_DATA, f.get('userdata')) + self.assertEquals(EC2_META, f.get('ec2-metadata')) + self.assertEquals(2, f.get('version')) + metadata = f['metadata'] + self.assertEquals('nova', metadata.get('availability_zone')) + self.assertEquals('sm-foo-test.novalocal', metadata.get('hostname')) + self.assertEquals('sm-foo-test.novalocal', + metadata.get('local-hostname')) + self.assertEquals('sm-foo-test', metadata.get('name')) + self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('uuid')) + self.assertEquals('b0fa911b-69d4-4476-bbe2-1c92bff6535c', + metadata.get('instance-id')) + + @hp.activate + def test_no_ec2(self): + _register_uris(self.VERSION, {}, {}, OS_FILES) f = ds.read_metadata_service(BASE_URL, version=self.VERSION) self.assertEquals(VENDOR_DATA, f.get('vendordata')) self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertEquals(USER_DATA, f.get('userdata')) + self.assertEquals({}, f.get('ec2-metadata')) + self.assertEquals(2, f.get('version')) + + @hp.activate + def test_bad_metadata(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.NonReadable, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_bad_uuid(self): + os_files = copy.deepcopy(OS_FILES) + os_meta = copy.deepcopy(OSTACK_META) + os_meta.pop('uuid') + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = json.dumps(os_meta) + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_userdata_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('user_data'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(VENDOR_DATA, f.get('vendordata')) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('userdata')) + + @hp.activate + def test_vendordata_empty(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data.json'): + os_files.pop(k, None) + _register_uris(self.VERSION, {}, {}, os_files) + f = ds.read_metadata_service(BASE_URL, version=self.VERSION) + self.assertEquals(CONTENT_0, f['files']['/etc/foo.cfg']) + self.assertEquals(CONTENT_1, f['files']['/etc/bar/bar.cfg']) + self.assertFalse(f.get('vendordata')) + + @hp.activate + def test_vendordata_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('vendor_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_metadata_invalid(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, + BASE_URL, version=self.VERSION) + + @hp.activate + def test_datasource(self): + _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertTrue(found) + self.assertEquals(2, ds_os.version) + md = dict(ds_os.metadata) + md.pop('instance-id', None) + md.pop('local-hostname', None) + self.assertEquals(OSTACK_META, md) + self.assertEquals(EC2_META, ds_os.ec2_metadata) + self.assertEquals(USER_DATA, ds_os.userdata_raw) + self.assertEquals(2, len(ds_os.files)) + self.assertEquals(VENDOR_DATA, ds_os.vendordata_raw) + + @hp.activate + def test_bad_datasource_meta(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = '{' # some invalid json + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) + + @hp.activate + def test_no_datasource(self): + os_files = copy.deepcopy(OS_FILES) + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files.pop(k) + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + ds_os.ds_cfg = { + 'max_wait': 0, + 'timeout': 0, + } + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 6e40098626531397a339d3a231d821b738e69175 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:40:51 -0800 Subject: Adjust detection of python versions and variables exposed --- tests/unittests/helpers.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 945c1500..5bed13cc 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -12,10 +12,27 @@ from cloudinit import util import shutil -# Handle how 2.6 doesn't have the assertIn or assertNotIn +# Used for detecting different python versions +PY2 = False +PY26 = False +PY27 = False +PY3 = False + _PY_VER = sys.version_info _PY_MAJOR, _PY_MINOR = _PY_VER[0:2] if (_PY_MAJOR, _PY_MINOR) <= (2, 6): + if (_PY_MAJOR, _PY_MINOR) == (2, 6): + PY26 = True + if (_PY_MAJOR, _PY_MINOR) >= (2, 0): + PY2 = True +else: + if (_PY_MAJOR, _PY_MINOR) == (2, 7): + PY27 = True + PY2 = True + if (_PY_MAJOR, _PY_MINOR) >= (3, 0): + PY3 = True + +if PY26: # For now add these on, taken from python 2.7 + slightly adjusted class TestCase(unittest.TestCase): def assertIn(self, member, container, msg=None): -- cgit v1.2.3 From 7fb9f75e1bd8b8ef36398c7adeb8d18a4fe9745e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 7 Feb 2014 16:46:32 -0800 Subject: Add test for disabled dsmode --- tests/unittests/test_datasource/test_openstack.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index c8cc40db..3fcf8bc9 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -287,3 +287,26 @@ class TestOpenStackDataSource(test_helpers.TestCase): found = ds_os.get_data() self.assertFalse(found) self.assertIsNone(ds_os.version) + + @hp.activate + def test_disabled_datasource(self): + os_files = copy.deepcopy(OS_FILES) + os_meta = copy.deepcopy(OSTACK_META) + os_meta['meta'] = { + 'dsmode': 'disabled', + } + for k in list(os_files.keys()): + if k.endswith('meta_data.json'): + os_files[k] = json.dumps(os_meta) + _register_uris(self.VERSION, {}, {}, os_files) + ds_os = ds.DataSourceOpenStack(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + ds_os.ds_cfg = { + 'max_wait': 0, + 'timeout': 0, + } + self.assertIsNone(ds_os.version) + found = ds_os.get_data() + self.assertFalse(found) + self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 098a74e6207f5d91f515fac63e970375d52795c0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 12:20:33 -0800 Subject: Remove HEAD usage and other small adjustments --- cloudinit/ec2_utils.py | 4 +-- cloudinit/sources/DataSourceOpenStack.py | 1 + cloudinit/sources/helpers/openstack.py | 41 ++++++++++++++--------- cloudinit/url_helper.py | 23 ++----------- tests/unittests/test_datasource/test_openstack.py | 11 ------ 5 files changed, 30 insertions(+), 50 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 91cba20f..a7c9c9ab 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,10 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httplib -from urlparse import (urlparse, urlunparse) - import functools +import httplib import json from cloudinit import log as logging diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 621572de..2c50ed84 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -44,6 +44,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None self.files = {} + self.ec2_metadata = None def __str__(self): root = sources.DataSource.__str__(self) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 9dbef677..09fb4ad8 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,7 +21,6 @@ import abc import base64 import copy -import functools import os from cloudinit import ec2_utils @@ -395,26 +394,38 @@ class ConfigDriveReader(BaseReader): class MetadataReader(BaseReader): def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): super(MetadataReader, self).__init__(base_url) - self._url_reader = functools.partial(url_helper.readurl, - retries=retries, - ssl_details=ssl_details, - timeout=timeout) - self._url_checker = functools.partial(url_helper.existsurl, - ssl_details=ssl_details, - timeout=timeout) - self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, - ssl_details=ssl_details, - timeout=timeout, - retries=retries) + self.ssl_details = ssl_details + self.timeout = float(timeout) + self.retries = int(retries) def _path_read(self, path): - return str(self._url_reader(path)) + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout) + return response.contents def _path_exists(self, path): - return self._url_checker(path) + + def should_retry_cb(request, cause): + if cause.code >= 400: + return False + return True + + try: + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout, + exception_cb=should_retry_cb) + return response.ok() + except IOError: + return False def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) def _read_ec2_metadata(self): - return self._ec2_reader() + return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, + timeout=self.timeout, + retries=self.retries) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 76a8e29b..a477b185 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -166,35 +166,16 @@ def _get_ssl_args(url, ssl_details): return ssl_args -def existsurl(url, ssl_details=None, timeout=None): - r = _readurl(url, ssl_details=ssl_details, timeout=timeout, - method='HEAD', check_status=False) - return r.ok() - - def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): - return _readurl(url, data=data, timeout=timeout, retries=retries, - sec_between=sec_between, headers=headers, - headers_cb=headers_cb, ssl_details=ssl_details, - check_status=check_status, - allow_redirects=allow_redirects, - exception_cb=exception_cb) - - -def _readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None, - method='GET'): + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, } req_args.update(_get_ssl_args(url, ssl_details)) - scheme = urlparse(url).scheme # pylint: disable=E1101 req_args['allow_redirects'] = allow_redirects - req_args['method'] = method + req_args['method'] = 'GET' if timeout is not None: req_args['timeout'] = max(float(timeout), 0) if data: diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 3fcf8bc9..3a64430a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -117,20 +117,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) - def head_request_callback(method, uri, headers): - uri = urlparse(uri) - path = uri.path.lstrip("/") - for key in os_files.keys(): - if key.startswith(path): - return (200, headers, '') - return (404, headers, '') - hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), body=get_request_callback) - hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), - body=head_request_callback) - class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' -- cgit v1.2.3 From aa1d0328da6ae1ace03e327feafabcf84f91b2b5 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Sat, 8 Feb 2014 20:59:52 +0000 Subject: wrap url get call in try/except clause --- cloudinit/sources/DataSourceGCE.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 8a46f933..c96cfffd 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -52,7 +52,10 @@ class DataSourceGCE(sources.DataSource): } for mkey in url_map.iterkeys(): - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + try: + resp = url_helper.readurl(url=url_map[mkey], headers=headers) + except IOError: + return False if resp.ok(): if mkey == 'public-keys': pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] -- cgit v1.2.3 From ea05883f88cedd5ed099aed19d4c5df563525097 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 13:16:07 -0800 Subject: Capture IOError and use LOG better --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceOpenStack.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 1d30fe08..142e0eb8 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -87,7 +87,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = util.mergemanydict([md, DEFAULT_METADATA]) user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s" % user_dsmode) + LOG.warn("User specified invalid mode: %s", user_dsmode) user_dsmode = None dsmode = get_ds_mode(cfgdrv_ver=results['version'], diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 2c50ed84..69807798 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -126,14 +126,14 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'version': openstack.OS_LATEST}) except openstack.NonReadable: return False - except openstack.BrokenMetadata: + except (openstack.BrokenMetadata, IOError): util.logexc(LOG, "Broken metadata address %s", self.metadata_address) return False user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s" % user_dsmode) + LOG.warn("User specified invalid mode: %s", user_dsmode) user_dsmode = None if user_dsmode == 'disabled': return False -- cgit v1.2.3 From 1edc8d3697d05f66195b9a425771f60cf6f9c27e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 13:28:07 -0800 Subject: Update requests ssl not supported message --- cloudinit/url_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index a477b185..43e879d2 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -39,6 +39,7 @@ NOT_FOUND = httplib.NOT_FOUND # Check if requests has ssl support (added in requests >= 0.8.8) SSL_ENABLED = False CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) +_REQ_VER = None try: from distutils.version import LooseVersion import pkg_resources @@ -152,7 +153,8 @@ def _get_ssl_args(url, ssl_details): scheme = urlparse(url).scheme # pylint: disable=E1101 if scheme == 'https' and ssl_details: if not SSL_ENABLED: - LOG.warn("SSL is not enabled, cert. verification can not occur!") + LOG.warn("SSL is not supported in requests v%s, " + "cert. verification can not occur!", _REQ_VER) else: if 'ca_certs' in ssl_details and ssl_details['ca_certs']: ssl_args['verify'] = ssl_details['ca_certs'] -- cgit v1.2.3 From 1597aca9b0606d02c045549afce4395370502231 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 13:39:27 -0800 Subject: Spacing and comment cleanup --- cloudinit/url_helper.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 43e879d2..c116a484 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -169,8 +169,8 @@ def _get_ssl_args(url, ssl_details): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, @@ -206,12 +206,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, def _cb(url): return headers headers_cb = _cb - if data: - # Do this after the log (it might be large) req_args['data'] = data if sec_between is None: sec_between = -1 + excps = [] # Handle retrying ourselves since the built-in support # doesn't handle sleeping between tries... -- cgit v1.2.3 From 5788cd903f6e4a9bab2ad32e9c1d2eb13b485ac3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 17:29:52 -0800 Subject: Handle code not existing in older requests versions --- cloudinit/sources/helpers/openstack.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 09fb4ad8..a17148d3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -408,8 +408,13 @@ class MetadataReader(BaseReader): def _path_exists(self, path): def should_retry_cb(request, cause): - if cause.code >= 400: - return False + try: + code = int(cause.code) + if code >= 400: + return False + except (TypeError, ValueError): + # Older versions of requests didn't have a code. + pass return True try: -- cgit v1.2.3 From 4ada54b39a852d0e4afea09b71ad248c90eb1501 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 12:19:13 -0500 Subject: cc_emit_upstart: do not bother filtering this module should "work" everywhere, in that it will only do anything if /sbin/initctl exists (which is going to be upstart). --- cloudinit/config/cc_emit_upstart.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 5c2f6a31..6d376184 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -29,8 +29,6 @@ distros = ['ubuntu', 'debian'] def handle(name, _cfg, cloud, log, args): - if cloud.distro.is_excluded(name): - return event_names = args if not event_names: # Default to the 'cloud-config' -- cgit v1.2.3 From 8d117d37e2945369abaa66d1e30f153e483c3faf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 14:44:41 -0500 Subject: fix pylint warning (and real bug) in bad spelling of resolve_conf_fn --- cloudinit/distros/arch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index fc5eee66..27dcaa88 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -34,7 +34,7 @@ class Distro(distros.Distro): network_conf_dir = "/etc/netctl" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" - resolv_conf_fn = "/etc/resolv.conf" + resolve_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts exclude_modules = [ 'grub-dpkg', -- cgit v1.2.3 From 20305aea1eac724069e0bfaaf976ec5caa8c2439 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 14:56:55 -0500 Subject: drop 'is_excluded'. for now, this the mechanism just doesn't seem right. I think i'd rather have the module declare supported distros than have distros declare [un]supported modules. --- cloudinit/config/cc_apt_configure.py | 2 -- cloudinit/config/cc_apt_pipelining.py | 2 -- cloudinit/config/cc_grub_dpkg.py | 2 -- cloudinit/distros/__init__.py | 8 -------- cloudinit/distros/arch.py | 6 ------ cloudinit/distros/gentoo.py | 5 ----- tests/unittests/test_distros/test_is_excluded.py | 15 --------------- 7 files changed, 40 deletions(-) delete mode 100644 tests/unittests/test_distros/test_is_excluded.py diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ccb45bb9..29c13a3d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,8 +51,6 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): - if cloud.distro.is_excluded(name): - return release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index bd180e82..503a1485 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -35,8 +35,6 @@ APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" def handle(_name, cfg, _cloud, log, _args): - if _cloud.distro.is_excluded(_name): - return apt_pipe_value = util.get_cfg_option_str(cfg, "apt_pipelining", False) apt_pipe_value_s = str(apt_pipe_value).lower().strip() diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 03cdd98c..b3ce6fb6 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -29,8 +29,6 @@ def handle(_name, cfg, _cloud, log, _args): idevs = None idevs_empty = None - if _cloud.distro.is_excluded(_name): - return if "grub-dpkg" in cfg: idevs = util.get_cfg_option_str(cfg["grub-dpkg"], "grub-pc/install_devices", None) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8fc0da9f..55d6bcbc 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -56,20 +56,12 @@ class Distro(object): hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" init_cmd = ['service'] # systemctl, service etc - exclude_modules = [] def __init__(self, name, cfg, paths): self._paths = paths self._cfg = cfg self.name = name - def is_excluded(self, name): - if name in self.exclude_modules: - distro = getattr(self, name, None) or getattr(self, 'osfamily') - LOG.debug(("Skipping module named %s, distro %s excluded"), name, - distro) - return True - @abc.abstractmethod def install_packages(self, pkglist): raise NotImplementedError() diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 27dcaa88..310c3dff 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -36,12 +36,6 @@ class Distro(distros.Distro): tz_local_fn = "/etc/localtime" resolve_conf_fn = "/etc/resolv.conf" init_cmd = ['systemctl'] # init scripts - exclude_modules = [ - 'grub-dpkg', - 'apt-configure', - 'apt-pipelining', - 'yum-add-repo', - ] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 0a95fa23..09f8d8ea 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -34,11 +34,6 @@ class Distro(distros.Distro): tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" init_cmd = [''] # init scripts - exclude_modules = [ - 'grub-dpkg', - 'apt-configure', - 'apt-pipelining', - ] def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/tests/unittests/test_distros/test_is_excluded.py b/tests/unittests/test_distros/test_is_excluded.py deleted file mode 100644 index 53a4445c..00000000 --- a/tests/unittests/test_distros/test_is_excluded.py +++ /dev/null @@ -1,15 +0,0 @@ -from cloudinit.distros import gentoo -import unittest - - -class TestIsExcluded(unittest.TestCase): - - def setUp(self): - self.distro = gentoo.Distro('gentoo', {}, None) - self.distro.exclude_modules = ['test-module'] - - def test_is_excluded_success(self): - self.assertEqual(self.distro.is_excluded('test-module'), True) - - def test_is_excluded_fail(self): - self.assertEqual(self.distro.is_excluded('missing'), None) -- cgit v1.2.3 From 065287bd56fe7f63a8dc41fa1457be4439f20efd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Feb 2014 19:58:38 -0500 Subject: support configuration of MD_URL, disable if not resolvable. this allows the metadata url to be configured by setting: datasource: GCE: metadata_url: Then also, if its not resolvable, we just deactivate the datasource quickly. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index c96cfffd..6eb3da4d 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -16,19 +16,25 @@ from cloudinit import log as logging +from cloudinit import util from cloudinit import sources from cloudinit import url_helper LOG = logging.getLogger(__name__) -MD_URL = 'http://metadata/computeMetadata/v1/' +BUILTIN_DS_CONFIG = { + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' +} class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata_address = MD_URL self.metadata = {} + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), + BUILTIN_DS_CONFIG]) + self.metadata_address = self.ds_cfg['metadata_url'] # GCE takes sshKeys attribute in the format of ':' # so we have to trim each key to remove the username part @@ -51,6 +57,11 @@ class DataSourceGCE(sources.DataSource): 'local-hostname': self.metadata_address + 'instance/hostname', } + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug("%s is not resolvable", self.metadata_address) + return False + for mkey in url_map.iterkeys(): try: resp = url_helper.readurl(url=url_map[mkey], headers=headers) -- cgit v1.2.3 From 18b8d40d96d2e19f1a949b06b55a3dac54595b23 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:39:39 -0500 Subject: cloudsigma: change default dsmode to 'net' Previously this had 'local' as the default datasource mode, meaning that user-data code such as boot hooks and such would not be guaranteed to have network access. That would be out of sync with the expectation on other platforms where the default is 'network up'. The user can still specify 'dsmode' as local if necessary and the local datasource will claim itself found. --- cloudinit/sources/DataSourceCloudSigma.py | 27 ++++++++++++++++++++------- doc/sources/cloudsigma/README.rst | 6 +++--- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 78acd8a4..e734d7e5 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -45,18 +45,25 @@ class DataSourceCloudSigma(sources.DataSource): Metadata is the whole server context and /meta/cloud-config is used as userdata. """ + dsmode = None try: server_context = self.cepko.all().result server_meta = server_context['meta'] - self.userdata_raw = server_meta.get('cloudinit-user-data', "") - self.metadata = server_context - self.ssh_public_key = server_meta['ssh_public_key'] - - if server_meta.get('cloudinit-dsmode') in VALID_DSMODES: - self.dsmode = server_meta['cloudinit-dsmode'] except: util.logexc(LOG, "Failed reading from the serial port") return False + + dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) + if dsmode not in VALID_DSMODES: + LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) + dsmode = 'net' + if dsmode == "disabled" or dsmode != self.dsmode: + return False + + self.userdata_raw = server_meta.get('cloudinit-user-data', "") + self.metadata = server_context + self.ssh_public_key = server_meta['ssh_public_key'] + return True def get_hostname(self, fqdn=False, resolve_ip=False): @@ -76,11 +83,17 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] +class DataSourceCloudSigmaNet(DataSourceCloudSigma): + def __init__(self, sys_cfg, distro, paths): + DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) + self.dsmode = 'net' + + # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigma, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/doc/sources/cloudsigma/README.rst b/doc/sources/cloudsigma/README.rst index 8cb2b0fe..1d9160a2 100644 --- a/doc/sources/cloudsigma/README.rst +++ b/doc/sources/cloudsigma/README.rst @@ -23,9 +23,9 @@ You can provide user-data to the VM using the dedicated `meta field`_ in the `se header could be omitted. However since this is a raw-text field you could provide any of the valid `config formats`_. -If your user-data needs an internet connection you have to create a `meta field`_ in the `server context`_ -``cloudinit-dsmode`` and set "net" as value. If this field does not exist the default value is "local". - +If your user-data does not need an internet connection you can create a +`meta field`_ in the `server context`_ ``cloudinit-dsmode`` and set "local" as value. +If this field does not exist the default value is "net". .. _CloudSigma: http://cloudsigma.com/ -- cgit v1.2.3 From 64856d7f245cc329a11b263c4eef8bf76bdee0e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 10:58:08 -0500 Subject: add 'user-data' support. This just adds user-data in 'instance/attributes/user-data'. Also turns retries to 0 on all other things. --- cloudinit/sources/DataSourceGCE.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 6eb3da4d..95a410ba 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -25,6 +25,7 @@ LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } +REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): @@ -55,6 +56,7 @@ class DataSourceGCE(sources.DataSource): 'availability-zone': self.metadata_address + 'instance/zone', 'public-keys': self.metadata_address + 'project/attributes/sshKeys', 'local-hostname': self.metadata_address + 'instance/hostname', + 'user-data': self.metadata_address + 'instance/attributes/user-data', } # if we cannot resolve the metadata server, then no point in trying @@ -64,7 +66,8 @@ class DataSourceGCE(sources.DataSource): for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=url_map[mkey], headers=headers, + retries=0) except IOError: return False if resp.ok(): @@ -74,8 +77,15 @@ class DataSourceGCE(sources.DataSource): else: self.metadata[mkey] = resp.contents else: + if mkey in REQUIRED_FIELDS: + LOG.warn("required metadata '%s' not found in metadata", + url_map[mkey]) + return False + self.metadata[mkey] = None return False + + self.user_data_raw = self.metadata['user-data'] return True @property @@ -92,9 +102,6 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] - def get_userdata_raw(self): - return None - @property def availability_zone(self): return self.metadata['instance-zone'] -- cgit v1.2.3 From f82e9552145ff468763727d7e5a53f56dc9f5b20 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 11:13:52 -0500 Subject: add Openstack to default datasources --- cloudinit/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 0d8d8ec5..37d4958b 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -37,6 +37,7 @@ CFG_BUILTIN = { 'OVF', 'MAAS', 'GCE', + 'OpenStack' 'Ec2', 'CloudSigma', 'CloudStack', -- cgit v1.2.3 From 87d0fa867f27f101e93006ba8dc8a395098e8df1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 12:13:42 -0500 Subject: wait less for the metadata service (by default) Waiting around for a metadata service in a given datasource means that if its not there all the subsequent datasources have to wait, and boot is slowed down. As it is right now, EC2 is the only one that has the right to wait. In the past, we had to wait around for the EC2 metadata service. I really do not want to extend that courtesy to other cloud platforms. A network based metadata service should be up as soon as networking is up. --- cloudinit/sources/DataSourceOpenStack.py | 28 +++++++++++++--------------- cloudinit/url_helper.py | 1 + 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 69807798..7fafa3f7 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -45,6 +45,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = None self.files = {} self.ec2_metadata = None + if not self.ds_cfg: + self.ds_cfg = {} def __str__(self): root = sources.DataSource.__str__(self) @@ -54,27 +56,25 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def _get_url_settings(self): # TODO(harlowja): this is shared with ec2 datasource, we should just # move it to a shared location instead... - ds_cfg = self.ds_cfg - if not ds_cfg: - ds_cfg = {} - max_wait = 120 + # Note: the defaults here are different though. + + # max_wait < 0 indicates do not wait + max_wait = -1 + timeout = 10 + try: - max_wait = int(ds_cfg.get("max_wait", max_wait)) + max_wait = int(self.ds_cfg.get("max_wait", max_wait)) except Exception: util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - timeout = 50 try: - timeout = max(0, int(ds_cfg.get("timeout", timeout))) + timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) except Exception: util.logexc(LOG, "Failed to get timeout, using %s", timeout) return (max_wait, timeout) def wait_for_metadata_service(self): - ds_cfg = self.ds_cfg - if not ds_cfg: - ds_cfg = {} - urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL]) filtered = [x for x in urls if util.is_resolvable_url(x)] if set(filtered) != set(urls): LOG.debug("Removed the following from metadata urls: %s", @@ -95,8 +95,6 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() - if max_wait <= 0: - return False start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, timeout=timeout, @@ -104,8 +102,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if avail_url: LOG.debug("Using metadata source: '%s'", url2base[avail_url]) else: - LOG.critical("Giving up on md from %s after %s seconds", - md_urls, int(time.time() - start_time)) + LOG.debug("Giving up on OpenStack md from %s after %s seconds", + md_urls, int(time.time() - start_time)) self.metadata_address = url2base.get(avail_url) return bool(avail_url) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index c116a484..4a83169a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -266,6 +266,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. + a number <= 0 will always result in only one try timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers -- cgit v1.2.3 From 5d5b8f90a10549bf54cf2000f514eb9c6ea420f7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 12:50:59 -0500 Subject: do not warn on waiting for url --- cloudinit/sources/DataSourceOpenStack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 7fafa3f7..5edbb448 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -97,8 +97,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): (max_wait, timeout) = self._get_url_settings() start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, - timeout=timeout, - status_cb=LOG.warn) + timeout=timeout) if avail_url: LOG.debug("Using metadata source: '%s'", url2base[avail_url]) else: -- cgit v1.2.3 From b8f1c6f27345d1aa0ef550a72bd34c2d7bd4ed41 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 13:53:08 -0500 Subject: make 'Loaded datasource' a info message --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 7acd3355..58349ffc 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -234,7 +234,7 @@ class Init(object): copy.deepcopy(self.ds_deps), cfg_list, pkg_list) - LOG.debug("Loaded datasource %s - %s", dsname, ds) + LOG.info("Loaded datasource %s - %s", dsname, ds) self.datasource = ds # Ensure we adjust our path members datasource # now that we have one (thus allowing ipath to be used) -- cgit v1.2.3 From 046352fc69e94f4c2f2d209ff5638fc455ff4b97 Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 21:24:41 +0000 Subject: GCE: add unit tests, user-data support and few other fixes --- cloudinit/sources/DataSourceGCE.py | 49 ++++++------ tests/unittests/test_datasource/test_gce.py | 112 ++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 24 deletions(-) create mode 100644 tests/unittests/test_datasource/test_gce.py diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 95a410ba..2d5ccad5 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' + 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -31,7 +31,7 @@ REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.metadata = {} + self.metadata = dict() self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, ["datasource", "GCE"], {}), BUILTIN_DS_CONFIG]) @@ -59,33 +59,31 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # if we cannot resolve the metadata server, then no point in trying - if not util.is_resolvable(self.metadata_address): - LOG.debug("%s is not resolvable", self.metadata_address) + # try to connect to metadata service or fail otherwise + try: + url_helper.readurl(url=url_map['instance-id'], headers=headers) + except url_helper.UrlError as e: + LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( + e)) return False + # iterate over url_map keys to get metadata items for mkey in url_map.iterkeys(): try: - resp = url_helper.readurl(url=url_map[mkey], headers=headers, - retries=0) - except IOError: - return False - if resp.ok(): - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys + resp = url_helper.readurl( + url=url_map[mkey], headers=headers) + if resp.code == 200: + if mkey == 'public-keys': + pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] + self.metadata[mkey] = pub_keys + else: + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = resp.contents - else: - if mkey in REQUIRED_FIELDS: - LOG.warn("required metadata '%s' not found in metadata", - url_map[mkey]) - return False - + self.metadata[mkey] = None + except url_helper.UrlError as e: self.metadata[mkey] = None - return False - - self.user_data_raw = self.metadata['user-data'] + LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( + mkey, e)) return True @property @@ -102,9 +100,12 @@ class DataSourceGCE(sources.DataSource): def get_hostname(self, fqdn=False): return self.metadata['local-hostname'] + def get_userdata_raw(self): + return self.metadata['user-data'] + @property def availability_zone(self): - return self.metadata['instance-zone'] + return self.metadata['availability-zone'] # Used to match classes to dependencies datasources = [ diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py new file mode 100644 index 00000000..0c58be52 --- /dev/null +++ b/tests/unittests/test_datasource/test_gce.py @@ -0,0 +1,112 @@ +# +# Copyright (C) 2014 Vaidas Jablonskis +# +# Author: Vaidas Jablonskis +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import unittest +import httpretty +import re + +from urlparse import urlparse + +from cloudinit import settings +from cloudinit import helpers +from cloudinit.sources import DataSourceGCE + +GCE_META = { + 'instance/id': '123', + 'instance/zone': 'foo/bar', + 'project/attributes/sshKeys': 'user:ssh-rsa AA2..+aRD0fyVw== root@server', + 'instance/hostname': 'server.project-name.local', + 'instance/attributes/user-data': '/bin/echo foo\n', +} + +GCE_META_PARTIAL = { + 'instance/id': '123', + 'instance/hostname': 'server.project-name.local', +} + +HEADERS = {'X-Google-Metadata-Request': 'True'} +MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') + + +def _request_callback(method, uri, headers): + url_path = urlparse(uri).path + if url_path.startswith('/computeMetadata/v1/'): + path = url_path.split('/computeMetadata/v1/')[1:][0] + else: + path = None + if path in GCE_META: + #return (200, headers, GCE_META.get(path)) + return (200, headers, GCE_META.get(path)) + else: + return (404, headers, '') + + +class TestDataSourceGCE(unittest.TestCase): + + def setUp(self): + self.ds = DataSourceGCE.DataSourceGCE( + settings.CFG_BUILTIN, None, + helpers.Paths({})) + + @httpretty.activate + def test_connection(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + + success = self.ds.get_data() + self.assertTrue(success) + + req_header = httpretty.last_request().headers + self.assertDictContainsSubset(HEADERS, req_header) + + @httpretty.activate + def test_metadata(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META.get('instance/hostname'), + self.ds.get_hostname()) + + self.assertEqual(GCE_META.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META.get('instance/zone'), + self.ds.availability_zone) + + self.assertEqual(GCE_META.get('instance/attributes/user-data'), + self.ds.get_userdata_raw()) + + # we expect a list of public ssh keys with user names stripped + self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], + self.ds.get_public_ssh_keys()) + + # test partial metadata (missing user-data in particular) + @httpretty.activate + def test_metadata_partial(self): + httpretty.register_uri( + httpretty.GET, MD_URL_RE, + body=_request_callback) + self.ds.get_data() + + self.assertEqual(GCE_META_PARTIAL.get('instance/id'), + self.ds.get_instance_id()) + + self.assertEqual(GCE_META_PARTIAL.get('instance/hostname'), + self.ds.get_hostname()) -- cgit v1.2.3 From f40bdb4869567124dbc48feaa0574cc83df26e3a Mon Sep 17 00:00:00 2001 From: Vaidas Jablonskis Date: Thu, 13 Feb 2014 22:03:12 +0000 Subject: GCE: use dns name instead of IP address --- cloudinit/sources/DataSourceGCE.py | 11 ++++------- tests/unittests/test_datasource/test_gce.py | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 2d5ccad5..e2be15b0 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -23,7 +23,7 @@ from cloudinit import url_helper LOG = logging.getLogger(__name__) BUILTIN_DS_CONFIG = { - 'metadata_url': 'http://169.254.169.254/computeMetadata/v1/' + 'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/' } REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') @@ -59,12 +59,9 @@ class DataSourceGCE(sources.DataSource): 'user-data': self.metadata_address + 'instance/attributes/user-data', } - # try to connect to metadata service or fail otherwise - try: - url_helper.readurl(url=url_map['instance-id'], headers=headers) - except url_helper.UrlError as e: - LOG.debug('Failed to connect to metadata service. Reason: {0}'.format( - e)) + # if we cannot resolve the metadata server, then no point in trying + if not util.is_resolvable(self.metadata_address): + LOG.debug('{0} is not resolvable'.format(self.metadata_address)) return False # iterate over url_map keys to get metadata items diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 0c58be52..d91bd531 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -39,7 +39,7 @@ GCE_META_PARTIAL = { } HEADERS = {'X-Google-Metadata-Request': 'True'} -MD_URL_RE = re.compile(r'http://169.254.169.254/computeMetadata/v1/.*') +MD_URL_RE = re.compile(r'http://metadata.google.internal./computeMetadata/v1/.*') def _request_callback(method, uri, headers): @@ -49,7 +49,6 @@ def _request_callback(method, uri, headers): else: path = None if path in GCE_META: - #return (200, headers, GCE_META.get(path)) return (200, headers, GCE_META.get(path)) else: return (404, headers, '') -- cgit v1.2.3 From 6baa9166aa8b150566003d2d4cf56cfd03fe6952 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 21:09:47 -0500 Subject: some style changes, some pylint, be less noisy this changes url_map to a list and adds 'required' information. * If we've not already found an entry, and this is required, then debug log (ie, this is just not GCE). * if we already found an entry and this is required: warn split the keys fixing out of the loop. --- cloudinit/sources/DataSourceGCE.py | 65 ++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index e2be15b0..c993293f 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -51,37 +51,60 @@ class DataSourceGCE(sources.DataSource): # GCE metadata server requires a custom header since v1 headers = {'X-Google-Metadata-Request': True} - url_map = { - 'instance-id': self.metadata_address + 'instance/id', - 'availability-zone': self.metadata_address + 'instance/zone', - 'public-keys': self.metadata_address + 'project/attributes/sshKeys', - 'local-hostname': self.metadata_address + 'instance/hostname', - 'user-data': self.metadata_address + 'instance/attributes/user-data', - } + # url_map: (our-key, path, required) + url_map = [ + ('instance-id', 'instance/id', True), + ('availability-zone', 'instance/zone', True), + ('local-hostname', 'instance/hostname', True), + ('public-keys', 'project/attributes/sshKeys', False), + ('user-data', 'instance/attributes/user-data', False), + ] # if we cannot resolve the metadata server, then no point in trying if not util.is_resolvable(self.metadata_address): - LOG.debug('{0} is not resolvable'.format(self.metadata_address)) + LOG.debug("%s is not resolvable", self.metadata_address) return False # iterate over url_map keys to get metadata items - for mkey in url_map.iterkeys(): + found = False + for (mkey, path, required) in url_map: try: - resp = url_helper.readurl( - url=url_map[mkey], headers=headers) + resp = url_helper.readurl(url=self.metadata_address + path, + headers=headers) if resp.code == 200: - if mkey == 'public-keys': - pub_keys = [self._trim_key(k) for k in resp.contents.splitlines()] - self.metadata[mkey] = pub_keys - else: - self.metadata[mkey] = resp.contents + found = True + self.metadata[mkey] = resp.contents else: - self.metadata[mkey] = None + if required: + msg = "required url %s returned code %s. not GCE" + if not found: + LOG.debug(msg, path, resp.code) + else: + LOG.warn(msg, path, resp.code) + return False + else: + self.metadata[mkey] = None except url_helper.UrlError as e: + if required: + msg = "required url %s raised exception %s. not GCE" + if not found: + LOG.debug(msg, path, e) + else: + LOG.warn(msg, path, e) + return False + msg = "Failed to get %s metadata item: %s." + if found: + LOG.warn(msg, path, e) + else: + LOG.debug(msg, path, e) + self.metadata[mkey] = None - LOG.warn('Failed to get {0} metadata item. Reason {1}.'.format( - mkey, e)) - return True + + if self.metadata['public-keys']: + lines = self.metadata['public-keys'].splitlines() + self.metadata['public-keys'] = [self._trim_key(k) for k in lines] + + return found @property def launch_index(self): @@ -94,7 +117,7 @@ class DataSourceGCE(sources.DataSource): def get_public_ssh_keys(self): return self.metadata['public-keys'] - def get_hostname(self, fqdn=False): + def get_hostname(self, fqdn=False, _resolve_ip=False): return self.metadata['local-hostname'] def get_userdata_raw(self): -- cgit v1.2.3