From 361738c6a9a14e32bd2123828fab8d8b70c6bc3a Mon Sep 17 00:00:00 2001 From: ctracey Date: Tue, 15 Jan 2013 16:08:43 -0500 Subject: add support for operating system families often it is convenient to classify a distro as being part of an operating system family. for instance, file templates may be identical for both debian and ubuntu, but to support this under the current templating code, one would need multiple templates for the same code. similarly, configuration handlers often fall into the same bucket: the configuraton is known to work/has been tested on a particular family of operating systems. right now this is handled with a declaration like: distros = ['fedora', 'rhel'] this fix seeks to address both of these issues. it allows for the simplification of the above line to: osfamilies = ['redhat'] and provides a mechanism for operating system family templates. --- cloudinit/distros/debian.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 7422f4f0..49b73477 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -48,6 +48,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'debian' def apply_locale(self, locale, out_fn=None): if not out_fn: -- cgit v1.2.3 From dc3ebfe2416028b78b6a846e939201d894b2c9b6 Mon Sep 17 00:00:00 2001 From: Craig Tracey Date: Sun, 27 Jan 2013 21:48:03 -0500 Subject: Adding package versioning logic to package_command This change adds the ability to provide specific package versions to Distro.install_packages and subsequently Distro.package_command. In order to effectively use Distro.install_packages, one is now able to pass a variety of formats in order to easily manage package requirements. These are examples of what can be passed: - "package" - ["package1","package2"] - ("package",) - ("package", "version") - [("package1",)("package2",)] - [("package1", "version1"),("package2","version2")] This change also adds the option to install a specific version for the puppet configuration module. This is especially important here as successful puppet deployments are highly reliant on specific puppet versions. --- cloudinit/config/cc_landscape.py | 2 +- cloudinit/config/cc_puppet.py | 10 ++++++++-- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/distros/debian.py | 17 +++++++++++++---- cloudinit/distros/rhel.py | 16 ++++++++++++---- cloudinit/util.py | 23 +++++++++++++++++++++++ 6 files changed, 58 insertions(+), 12 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 02610dd0..2efdff79 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -62,7 +62,7 @@ def handle(_name, cfg, cloud, log, _args): if not ls_cloudcfg: return - cloud.distro.install_packages(["landscape-client"]) + cloud.distro.install_packages(('landscape-client',)) merge_data = [ LSC_BUILTIN_CFG, diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index e9a0a0f4..471a1a8a 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -59,8 +59,14 @@ def handle(name, cfg, cloud, log, _args): # Start by installing the puppet package if necessary... install = util.get_cfg_option_bool(puppet_cfg, 'install', True) - if install: - cloud.distro.install_packages(["puppet"]) + version = util.get_cfg_option_str(puppet_cfg, 'version', None) + if not install and version: + log.warn(("Puppet install set false but version supplied," + " doing nothing.")) + elif install: + log.debug(("Attempting to install puppet %s,"), + version if version else 'latest') + cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration if 'conf' in puppet_cfg: diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index f3eede18..53013dcb 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -31,7 +31,7 @@ def handle(name, cfg, cloud, log, _args): salt_cfg = cfg['salt_minion'] # Start by installing the salt package ... - cloud.distro.install_packages(["salt-minion"]) + cloud.distro.install_packages(('salt-minion',)) # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 49b73477..1a8e927b 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -65,7 +65,7 @@ class Distro(distros.Distro): def install_packages(self, pkglist): self.update_package_sources() - self.package_command('install', pkglist) + self.package_command('install', pkgs=pkglist) def _write_network(self, settings): util.write_file(self.network_conf_fn, settings) @@ -142,15 +142,24 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None): + def package_command(self, command, args=None, pkgs=[]): e = os.environ.copy() # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', '--quiet', command] - if args: + '--assume-yes', '--quiet'] + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s=%s', pkgs) + cmd.extend(pkglist) + # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, env=e, capture=False) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index e65be8d7..2f91e386 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -63,7 +63,7 @@ class Distro(distros.Distro): self.osfamily = 'redhat' def install_packages(self, pkglist): - self.package_command('install', pkglist) + self.package_command('install', pkgs=pkglist) def _adjust_resolve(self, dns_servers, search_servers): try: @@ -208,7 +208,7 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None): + def package_command(self, command, args=None, pkgs=[]): cmd = ['yum'] # If enabled, then yum will be tolerant of errors on the command line # with regard to packages. @@ -219,9 +219,17 @@ class Distro(distros.Distro): # Determines whether or not yum prompts for confirmation # of critical actions. We don't want to prompt... cmd.append("-y") - cmd.append(command) - if args: + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, capture=False) diff --git a/cloudinit/util.py b/cloudinit/util.py index c0ea8d91..c9c5f794 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1560,3 +1560,26 @@ def is_partition(device): device = device[5:] return os.path.isfile("/sys/class/block/%s/partition" % device) + + +def expand_package_list(version_fmt, pkgs): + # we will accept tuples, lists of tuples, or just plain lists + if not isinstance(pkgs, list): + pkgs = [pkgs] + + pkglist = [] + for pkg in pkgs: + if isinstance(pkg, str): + pkglist.append(pkg) + continue + + if len(pkg) < 1 or len(pkg) > 2: + raise RuntimeError("Invalid package_command tuple.") + + if len(pkg) == 2 and pkg[1]: + pkglist.append(version_fmt % pkg) + continue + + pkglist.append(pkg[0]) + + return pkglist -- cgit v1.2.3 From dca9b6c94e10f9f42ad0f129ae6fd38ebb44f4b5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Mar 2013 14:54:25 -0500 Subject: pep8 and pylint fixes --- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/distros/__init__.py | 6 +++--- cloudinit/distros/debian.py | 5 ++++- cloudinit/distros/rhel.py | 5 ++++- cloudinit/ssh_util.py | 10 ++++------ cloudinit/util.py | 2 +- doc/rtd/conf.py | 8 ++++---- tests/unittests/helpers.py | 1 + tests/unittests/test_datasource/test_nocloud.py | 2 +- .../test_handler/test_handler_growpart.py | 22 +++++++++++----------- tests/unittests/test_sshutil.py | 5 +++-- 11 files changed, 37 insertions(+), 31 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index aefa3aff..de0c0bbd 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -75,7 +75,7 @@ def load_power_state(cfg): ','.join(opt_map.keys())) delay = pstate.get("delay", "now") - if delay != "now" and not re.match("\+[0-9]+", delay): + if delay != "now" and not re.match(r"\+[0-9]+", delay): raise TypeError("power_state[delay] must be 'now' or '+m' (minutes).") args = ["shutdown", opt_map[mode], delay] diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 0db4aac7..2a2d8216 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -73,7 +73,7 @@ class Distro(object): self._apply_hostname(hostname) @abc.abstractmethod - def package_command(self, cmd, args=None): + def package_command(self, cmd, args=None, pkgs=None): raise NotImplementedError() @abc.abstractmethod @@ -370,7 +370,7 @@ class Distro(object): # Import SSH keys if 'ssh_authorized_keys' in kwargs: keys = set(kwargs['ssh_authorized_keys']) or [] - ssh_util.setup_user_keys(keys, name, key_prefix=None) + ssh_util.setup_user_keys(keys, name, options=None) return True @@ -776,7 +776,7 @@ def normalize_users_groups(cfg, distro): # Just add it on at the end... base_users.append({'name': 'default'}) elif isinstance(base_users, (dict)): - base_users['default'] = base_users.get('default', True) + base_users['default'] = dict(base_users).get('default', True) elif isinstance(base_users, (str, basestring)): # Just append it on to be re-parsed later base_users += ",default" diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1a8e927b..1f2848d2 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -142,7 +142,10 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None, pkgs=[]): + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + e = os.environ.copy() # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 2f91e386..9fee5fd1 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -208,7 +208,10 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None, pkgs=[]): + def package_command(self, command, args=None, pkgs=None): + if pkgs is None: + pkgs = [] + cmd = ['yum'] # If enabled, then yum will be tolerant of errors on the command line # with regard to packages. diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 65fab117..95133236 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -19,9 +19,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO - -import csv import os import pwd @@ -42,6 +39,7 @@ VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", "ecdsa-sha2-nistp384-cert-v01@openssh.com", "ecdsa-sha2-nistp521-cert-v01@openssh.com") + class AuthKeyLine(object): def __init__(self, source, keytype=None, base64=None, comment=None, options=None): @@ -141,14 +139,14 @@ class AuthKeyLineParser(object): ent = line.strip() try: (keytype, base64, comment) = parse_ssh_key(ent) - except TypeError as e: + except TypeError: (keyopts, remain) = self._extract_options(ent) if options is None: options = keyopts - + try: (keytype, base64, comment) = parse_ssh_key(remain) - except TypeError as e: + except TypeError: return AuthKeyLine(src_line) return AuthKeyLine(src_line, keytype=keytype, base64=base64, diff --git a/cloudinit/util.py b/cloudinit/util.py index d0a6f81c..afde2066 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1530,7 +1530,7 @@ def get_proc_env(pid): fn = os.path.join("/proc/", str(pid), "environ") try: contents = load_file(fn) - toks = contents.split("\0") + toks = contents.split("\x00") for tok in toks: if tok == "": continue diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py index 87fc40ab..c9ae79f4 100644 --- a/doc/rtd/conf.py +++ b/doc/rtd/conf.py @@ -17,13 +17,13 @@ from cloudinit import version # General information about the project. project = 'Cloud-Init' -# -- General configuration ----------------------------------------------------- +# -- General configuration ---------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.intersphinx', ] @@ -55,7 +55,7 @@ exclude_patterns = [] # output. They are ignored by default. show_authors = False -# -- Options for HTML output --------------------------------------------------- +# -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 91a50e18..904677f1 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -183,6 +183,7 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): setattr(mod, f, trap_func) self.patched_funcs.append((mod, f, func)) + def populate_dir(path, files): os.makedirs(path) for (name, content) in files.iteritems(): diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 28e0a472..62fc5358 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -1,7 +1,7 @@ from cloudinit import helpers -from tests.unittests.helpers import populate_dir from cloudinit.sources import DataSourceNoCloud from cloudinit import util +from tests.unittests.helpers import populate_dir from mocker import MockerTestCase import os diff --git a/tests/unittests/test_handler/test_handler_growpart.py b/tests/unittests/test_handler/test_handler_growpart.py index 74c254e0..325244f2 100644 --- a/tests/unittests/test_handler/test_handler_growpart.py +++ b/tests/unittests/test_handler/test_handler_growpart.py @@ -1,7 +1,6 @@ from mocker import MockerTestCase from cloudinit import cloud -from cloudinit import helpers from cloudinit import util from cloudinit.config import cc_growpart @@ -9,9 +8,7 @@ from cloudinit.config import cc_growpart import errno import logging import os -import mocker import re -import stat # growpart: # mode: auto # off, on, auto, 'growpart', 'parted' @@ -85,6 +82,7 @@ growpart disk partition Resize partition 1 on /dev/sda """ + class TestDisabled(MockerTestCase): def setUp(self): super(TestDisabled, self).setUp() @@ -106,6 +104,7 @@ class TestDisabled(MockerTestCase): self.handle(self.name, config, self.cloud_init, self.log, self.args) + class TestConfig(MockerTestCase): def setUp(self): super(TestConfig, self).setUp() @@ -125,9 +124,9 @@ class TestConfig(MockerTestCase): def test_no_resizers_auto_is_fine(self): subp = self.mocker.replace(util.subp, passthrough=False) subp(['parted', '--help'], env={'LANG': 'C'}) - self.mocker.result((HELP_PARTED_NO_RESIZE,"")) + self.mocker.result((HELP_PARTED_NO_RESIZE, "")) subp(['growpart', '--help'], env={'LANG': 'C'}) - self.mocker.result((HELP_GROWPART_NO_RESIZE,"")) + self.mocker.result((HELP_GROWPART_NO_RESIZE, "")) self.mocker.replay() config = {'growpart': {'mode': 'auto'}} @@ -136,7 +135,7 @@ class TestConfig(MockerTestCase): def test_no_resizers_mode_growpart_is_exception(self): subp = self.mocker.replace(util.subp, passthrough=False) subp(['growpart', '--help'], env={'LANG': 'C'}) - self.mocker.result((HELP_GROWPART_NO_RESIZE,"")) + self.mocker.result((HELP_GROWPART_NO_RESIZE, "")) self.mocker.replay() config = {'growpart': {'mode': "growpart"}} @@ -146,7 +145,7 @@ class TestConfig(MockerTestCase): def test_mode_auto_prefers_parted(self): subp = self.mocker.replace(util.subp, passthrough=False) subp(['parted', '--help'], env={'LANG': 'C'}) - self.mocker.result((HELP_PARTED_RESIZE,"")) + self.mocker.result((HELP_PARTED_RESIZE, "")) self.mocker.replay() ret = cc_growpart.resizer_factory(mode="auto") @@ -173,7 +172,7 @@ class TestConfig(MockerTestCase): self.handle(self.name, {}, self.cloud_init, self.log, self.args) finally: cc_growpart.RESIZERS = orig_resizers - + class TestResize(MockerTestCase): def setUp(self): @@ -196,7 +195,7 @@ class TestResize(MockerTestCase): real_stat = os.stat resize_calls = [] - class myresizer(): + class myresizer(object): def resize(self, diskdev, partnum, partdev): resize_calls.append((diskdev, partnum, partdev)) if partdev == "/dev/YYda2": @@ -224,7 +223,7 @@ class TestResize(MockerTestCase): if f[0] == name: return f return None - + self.assertEqual(cc_growpart.RESIZE.NOCHANGE, find("/dev/XXda1", resized)[1]) self.assertEqual(cc_growpart.RESIZE.CHANGED, @@ -244,7 +243,8 @@ def simple_device_part_info(devpath): ret = re.search("([^0-9]*)([0-9]*)$", devpath) x = (ret.group(1), ret.group(2)) return x - + + class Bunch: def __init__(self, **kwds): self.__dict__.update(kwds) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 2415d06f..d8662cac 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -1,5 +1,5 @@ -from unittest import TestCase from cloudinit import ssh_util +from unittest import TestCase VALID_CONTENT = { @@ -34,6 +34,7 @@ TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'command="echo \'Please login as the user \"ubuntu\" rather than the' 'user \"root\".\';echo;sleep 10"') + class TestAuthKeyLineParser(TestCase): def test_simple_parse(self): # test key line with common 3 fields (keytype, base64, comment) @@ -61,7 +62,7 @@ class TestAuthKeyLineParser(TestCase): self.assertFalse(key.options) self.assertFalse(key.comment) self.assertEqual(key.keytype, ktype) - + def test_parse_with_keyoptions(self): # test key line with options in it parser = ssh_util.AuthKeyLineParser() -- cgit v1.2.3 From 6586b35f348ba089bba00e6bebb4ca1b14f41a19 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Mar 2013 15:30:38 -0500 Subject: allow customization of apt-get command, add --force-unsafe-io This allows the customization of the apt-get command used for installing packages, and also adds '--force-unsafe-io'. Because this is spawned from cloud-init, it seems to make sense as a first boot package installation option. --- ChangeLog | 1 + cloudinit/distros/debian.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/ChangeLog b/ChangeLog index 5ff305a1..d035a7a3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -47,6 +47,7 @@ - upstart: cloud-init-nonet.conf trap the TERM signal, so that dmesg or other output does not get a 'killed by TERM signal' message. - support resizing partitions via growpart or parted (LP: #1136936) + - allow specifying apt-get command in distro config ('apt_get_command') 0.7.1: - sysvinit: fix missing dependency in cloud-init job for RHEL 5.6 diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 1f2848d2..4b779d57 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -33,6 +33,10 @@ from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) +APT_GET_COMMAND = ('apt-get', '--option=Dpkg::Options::=--force-confold', + '--option=Dpkg::options::=--force-unsafe-io', + '--assume-yes', '--quiet') + class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" @@ -150,8 +154,7 @@ class Distro(distros.Distro): # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' - cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', '--quiet'] + cmd = list(self.get_option("apt_get_command", APT_GET_COMMAND)) if args and isinstance(args, str): cmd.append(args) -- cgit v1.2.3