diff options
| -rw-r--r-- | cloudinit/config/cc_apt_configure.py | 4 | ||||
| -rw-r--r-- | cloudinit/config/cc_emit_upstart.py | 28 | ||||
| -rw-r--r-- | cloudinit/config/cc_grub_dpkg.py | 21 | ||||
| -rw-r--r-- | cloudinit/config/cc_locale.py | 6 | ||||
| -rw-r--r-- | cloudinit/config/cc_snappy.py | 133 | ||||
| -rw-r--r-- | cloudinit/settings.py | 2 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceCloudSigma.py | 2 | ||||
| -rw-r--r-- | cloudinit/stages.py | 21 | ||||
| -rw-r--r-- | cloudinit/user_data.py | 4 | ||||
| -rw-r--r-- | cloudinit/util.py | 77 | ||||
| -rw-r--r-- | config/cloud.cfg | 1 | ||||
| -rw-r--r-- | doc/examples/cloud-config.txt | 2 | ||||
| -rw-r--r-- | tests/unittests/test_util.py | 96 |
13 files changed, 313 insertions, 84 deletions
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index de72903f..2c51d116 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,6 +51,10 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): + if util.is_false(cfg.get('apt_configure_enabled', True)): + log.debug("Skipping module named %s, disabled by config.", name) + return + release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 6d376184..e1b9a4c2 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -21,11 +21,32 @@ import os from cloudinit.settings import PER_ALWAYS +from cloudinit import log as logging from cloudinit import util frequency = PER_ALWAYS distros = ['ubuntu', 'debian'] +LOG = logging.getLogger(__name__) + + +def is_upstart_system(): + if not os.path.isfile("/sbin/initctl"): + LOG.debug(("Skipping module named %s," + " no /sbin/initctl located"), name) + return False + + myenv = os.environ.copy() + if 'UPSTART_SESSION' in myenv: + del myenv['UPSTART_SESSION'] + check_cmd = ['initctl', 'version'] + try: + (out, err) = util.subp(check_cmd, env=myenv) + return 'upstart' in out + except util.ProcessExecutionError as e: + LOG.debug("'%s' returned '%s', not using upstart", + ' '.join(check_cmd), e.exit_code) + return False def handle(name, _cfg, cloud, log, args): @@ -34,10 +55,11 @@ def handle(name, _cfg, cloud, log, args): # Default to the 'cloud-config' # event for backwards compat. event_names = ['cloud-config'] - if not os.path.isfile("/sbin/initctl"): - log.debug(("Skipping module named %s," - " no /sbin/initctl located"), name) + + if not is_upstart_system(): + log.debug("not upstart system, '%s' disabled") return + cfgpath = cloud.paths.get_ipath_cur("cloud_config") for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index e3219e81..456597af 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -25,15 +25,20 @@ from cloudinit import util distros = ['ubuntu', 'debian'] -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None +def handle(name, cfg, _cloud, log, _args): - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) + mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) + if not mycfg: + mycfg = {} + + enabled = mycfg.get('enabled', True) + if util.is_false(enabled): + log.debug("%s disabled by config grub_dpkg/enabled=%s", name, enabled) + return + + idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(mycfg, + "grub-pc/install_devices_empty", None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or (os.path.exists("/dev/xvda1") diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 6feaae9d..bbe5fcae 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -27,9 +27,9 @@ def handle(name, cfg, cloud, log, args): else: locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - if not locale: - log.debug(("Skipping module named %s, " - "no 'locale' configuration found"), name) + if util.is_false(locale): + log.debug("Skipping module named %s, disabled by config: %s", + name, locale) return log.debug("Setting locale to %s", locale) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py new file mode 100644 index 00000000..1588443f --- /dev/null +++ b/cloudinit/config/cc_snappy.py @@ -0,0 +1,133 @@ +# vi: ts=4 expandtab +# + +from cloudinit import log as logging +from cloudinit import templater +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +import glob +import os + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" + +CI_SNAPPY_CFG = { + 'env_file_path': SNAPPY_ENV_PATH, + 'packages': [], + 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'ssh_enabled': False +} + +""" +snappy: + ssh_enabled: True + packages: + - etcd + - {'name': 'pkg1', 'config': "wark"} +""" + + +def flatten(data, fill=None, tok="_", prefix='', recurse=True): + if fill is None: + fill = {} + for key, val in data.items(): + key = key.replace("-", "_") + if isinstance(val, dict) and recurse: + flatten(val, fill, tok=tok, prefix=prefix + key + tok, + recurse=recurse) + elif isinstance(key, str): + fill[prefix + key] = val + return fill + + +def render2env(data, tok="_", prefix=''): + flat = flatten(data, tok=tok, prefix=prefix) + ret = ["%s='%s'" % (key, val) for key, val in flat.items()] + return '\n'.join(ret) + '\n' + + +def install_package(pkg_name, config=None): + cmd = ["snappy", "install"] + if config: + if os.path.isfile(config): + cmd.append("--config-file=" + config) + else: + cmd.append("--config=" + config) + cmd.append(pkg_name) + util.subp(cmd) + + +def install_packages(package_dir, packages): + local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) + LOG.debug("installing local packages %s" % local_pkgs) + if local_pkgs: + for pkg in local_pkgs: + cfg = pkg.replace(".click", ".config") + if not os.path.isfile(cfg): + cfg = None + install_package(pkg, config=cfg) + + LOG.debug("installing click packages") + if packages: + for pkg in packages: + if not pkg: + continue + if isinstance(pkg, str): + name = pkg + config = None + elif pkg: + name = pkg.get('name', pkg) + config = pkg.get('config') + install_package(pkg_name=name, config=config) + + +def disable_enable_ssh(enabled): + LOG.debug("setting enablement of ssh to: %s", enabled) + # do something here that would enable or disable + not_to_be_run = "/etc/ssh/sshd_not_to_be_run" + if enabled: + util.del_file(not_to_be_run) + # this is an indempotent operation + util.subp(["systemctl", "start", "ssh"]) + else: + # this is an indempotent operation + util.subp(["systemctl", "stop", "ssh"]) + util.write_file(not_to_be_run, "cloud-init\n") + + +def handle(name, cfg, cloud, log, args): + mycfg = cfg.get('snappy', {'ssh_enabled': False}) + + if not mycfg: + LOG.debug("%s: no top level found", name) + return + + # take out of 'cfg' the cfg keys that cloud-init uses, so + # mycfg has only content external to cloud-init. + ci_cfg = CI_SNAPPY_CFG.copy() + for i in ci_cfg: + if i in mycfg: + ci_cfg[i] = mycfg[i] + del mycfg[i] + + # render the flattened environment variable style file to a path + # this was useful for systemd config environment files. given: + # snappy: + # foo: + # bar: wark + # cfg1: + # key1: value + # you get the following in env_file_path. + # foo_bar=wark + # foo_cfg1_key1=value + contents = render2env(mycfg) + header = '# for internal use only, not a guaranteed interface\n' + util.write_file(ci_cfg['env_file_path'], header + render2env(mycfg)) + + install_packages(ci_cfg['packages_dir'], + ci_cfg['packages']) + + disable_enable_ssh(ci_cfg.get('ssh_enabled', False)) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 5efcb0b0..b61e5613 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -47,7 +47,7 @@ CFG_BUILTIN = { ], 'def_log_file': '/var/log/cloud-init.log', 'log_cfgs': [], - 'syslog_fix_perms': 'syslog:adm', + 'syslog_fix_perms': ['syslog:adm', 'root:adm'], 'system_info': { 'paths': { 'cloud_dir': '/var/lib/cloud', diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 76597116..f8f94759 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -59,7 +59,7 @@ class DataSourceCloudSigma(sources.DataSource): LOG.warn("failed to get hypervisor product name via dmi data") return False else: - LOG.debug("detected hypervisor as {}".format(sys_product_name)) + LOG.debug("detected hypervisor as %s", sys_product_name) return 'cloudsigma' in sys_product_name.lower() LOG.warn("failed to query dmi data for system product name") diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 45d64823..d28e765b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -148,16 +148,25 @@ class Init(object): def _initialize_filesystem(self): util.ensure_dirs(self._initial_subdirs()) log_file = util.get_cfg_option_str(self.cfg, 'def_log_file') - perms = util.get_cfg_option_str(self.cfg, 'syslog_fix_perms') if log_file: util.ensure_file(log_file) - if perms: - u, g = util.extract_usergroup(perms) + perms = self.cfg.get('syslog_fix_perms') + if not perms: + perms = {} + if not isinstance(perms, list): + perms = [perms] + + error = None + for perm in perms: + u, g = util.extract_usergroup(perm) try: util.chownbyname(log_file, u, g) - except OSError: - util.logexc(LOG, "Unable to change the ownership of %s to " - "user %s, group %s", log_file, u, g) + return + except OSError as e: + error = e + + LOG.warn("Failed changing perms on '%s'. tried: %s. %s", + log_file, ','.join(perms), error) def read_cfg(self, extra_fns=None): # None check so that we don't keep on re-loading if empty diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 663a9048..eb3c7336 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -22,8 +22,6 @@ import os -import email - from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart @@ -338,7 +336,7 @@ def convert_string(raw_data, headers=None): headers = {} data = util.decode_binary(util.decomp_gzip(raw_data)) if "mime-version:" in data[0:4096].lower(): - msg = email.message_from_string(data) + msg = util.message_from_string(data) for (key, val) in headers.items(): _replace_header(msg, key, val) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index cc20305c..971c1c2d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -23,6 +23,7 @@ import contextlib import copy as obj_copy import ctypes +import email import errno import glob import grp @@ -128,6 +129,28 @@ def fully_decoded_payload(part): # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" +# dmidecode and /sys/class/dmi/id/* use different names for the same value, +# this allows us to refer to them by one canonical name +DMIDECODE_TO_DMI_SYS_MAPPING = { + 'baseboard-asset-tag': 'board_asset_tag', + 'baseboard-manufacturer': 'board_vendor', + 'baseboard-product-name': 'board_name', + 'baseboard-serial-number': 'board_serial', + 'baseboard-version': 'board_version', + 'bios-release-date': 'bios_date', + 'bios-vendor': 'bios_vendor', + 'bios-version': 'bios_version', + 'chassis-asset-tag': 'chassis_asset_tag', + 'chassis-manufacturer': 'chassis_vendor', + 'chassis-serial-number': 'chassis_serial', + 'chassis-version': 'chassis_version', + 'system-manufacturer': 'sys_vendor', + 'system-product-name': 'product_name', + 'system-serial-number': 'product_serial', + 'system-uuid': 'product_uuid', + 'system-version': 'product_version', +} + class ProcessExecutionError(IOError): @@ -2103,24 +2126,26 @@ def _read_dmi_syspath(key): """ Reads dmi data with from /sys/class/dmi/id """ - - dmi_key = "{0}/{1}".format(DMI_SYS_PATH, key) - LOG.debug("querying dmi data {0}".format(dmi_key)) + if key not in DMIDECODE_TO_DMI_SYS_MAPPING: + return None + mapped_key = DMIDECODE_TO_DMI_SYS_MAPPING[key] + dmi_key_path = "{0}/{1}".format(DMI_SYS_PATH, mapped_key) + LOG.debug("querying dmi data %s", dmi_key_path) try: - if not os.path.exists(dmi_key): - LOG.debug("did not find {0}".format(dmi_key)) + if not os.path.exists(dmi_key_path): + LOG.debug("did not find %s", dmi_key_path) return None - key_data = load_file(dmi_key) + key_data = load_file(dmi_key_path) if not key_data: - LOG.debug("{0} did not return any data".format(key)) + LOG.debug("%s did not return any data", dmi_key_path) return None - LOG.debug("dmi data {0} returned {0}".format(dmi_key, key_data)) + LOG.debug("dmi data %s returned %s", dmi_key_path, key_data) return key_data.strip() except Exception as e: - logexc(LOG, "failed read of {0}".format(dmi_key), e) + logexc(LOG, "failed read of %s", dmi_key_path, e) return None @@ -2132,26 +2157,40 @@ def _call_dmidecode(key, dmidecode_path): try: cmd = [dmidecode_path, "--string", key] (result, _err) = subp(cmd) - LOG.debug("dmidecode returned '{0}' for '{0}'".format(result, key)) + LOG.debug("dmidecode returned '%s' for '%s'", result, key) return result - except OSError as _err: - LOG.debug('failed dmidecode cmd: {0}\n{0}'.format(cmd, _err.message)) + except (IOError, OSError) as _err: + LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message) return None def read_dmi_data(key): """ - Wrapper for reading DMI data. This tries to determine whether the DMI - Data can be read directly, otherwise it will fallback to using dmidecode. + Wrapper for reading DMI data. + + This will do the following (returning the first that produces a + result): + 1) Use a mapping to translate `key` from dmidecode naming to + sysfs naming and look in /sys/class/dmi/... for a value. + 2) Use `key` as a sysfs key directly and look in /sys/class/dmi/... + 3) Fall-back to passing `key` to `dmidecode --string`. + + If all of the above fail to find a value, None will be returned. """ - if os.path.exists(DMI_SYS_PATH): - return _read_dmi_syspath(key) + syspath_value = _read_dmi_syspath(key) + if syspath_value is not None: + return syspath_value dmidecode_path = which('dmidecode') if dmidecode_path: return _call_dmidecode(key, dmidecode_path) - LOG.warn("did not find either path {0} or dmidecode command".format( - DMI_SYS_PATH)) - + LOG.warn("did not find either path %s or dmidecode command", + DMI_SYS_PATH) return None + + +def message_from_string(string): + if sys.version_info[:2] < (2, 7): + return email.message_from_file(six.StringIO(string)) + return email.message_from_string(string) diff --git a/config/cloud.cfg b/config/cloud.cfg index 200050d3..e96e1781 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -48,6 +48,7 @@ cloud_config_modules: - ssh-import-id - locale - set-passwords + - snappy - grub-dpkg - apt-pipelining - apt-configure diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1c59c2cf..1236796c 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -536,6 +536,8 @@ timezone: US/Eastern # # to remedy this situation, 'def_log_file' can be set to a filename # and syslog_fix_perms to a string containing "<user>:<group>" +# if syslog_fix_perms is a list, it will iterate through and use the +# first pair that does not raise error. # # the default values are '/var/log/cloud-init.log' and 'syslog:adm' # the value of 'def_log_file' should match what is configured in logging diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 33c191a9..1619b5d2 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -323,58 +323,67 @@ class TestMountinfoParsing(helpers.ResourceUsingTestCase): class TestReadDMIData(helpers.FilesystemMockingTestCase): - def _patchIn(self, root): - self.patchOS(root) - self.patchUtils(root) + def setUp(self): + super(TestReadDMIData, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + self.patchOS(self.new_root) + self.patchUtils(self.new_root) - def _write_key(self, key, content): - """Mocks the sys path found on Linux systems.""" - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self._patchIn(new_root) + def _create_sysfs_parent_directory(self): util.ensure_dir(os.path.join('sys', 'class', 'dmi', 'id')) + def _create_sysfs_file(self, key, content): + """Mocks the sys path found on Linux systems.""" + self._create_sysfs_parent_directory() dmi_key = "/sys/class/dmi/id/{0}".format(key) util.write_file(dmi_key, content) - def _no_syspath(self, key, content): + def _configure_dmidecode_return(self, key, content, error=None): """ In order to test a missing sys path and call outs to dmidecode, this function fakes the results of dmidecode to test the results. """ - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self._patchIn(new_root) - self.real_which = util.which - self.real_subp = util.subp - - def _which(key): - return True - util.which = _which - - def _cdd(_key, error=None): + def _dmidecode_subp(cmd): + if cmd[-1] != key: + raise util.ProcessExecutionError() return (content, error) - util.subp = _cdd - - def test_key(self): - key_content = "TEST-KEY-DATA" - self._write_key("key", key_content) - self.assertEquals(key_content, util.read_dmi_data("key")) - def test_key_mismatch(self): - self._write_key("test", "ABC") - self.assertNotEqual("123", util.read_dmi_data("test")) - - def test_no_key(self): - self._no_syspath(None, None) - self.assertFalse(util.read_dmi_data("key")) - - def test_callout_dmidecode(self): - """test to make sure that dmidecode is used when no syspath""" - self._no_syspath("key", "stuff") - self.assertEquals("stuff", util.read_dmi_data("key")) - self._no_syspath("key", None) - self.assertFalse(None, util.read_dmi_data("key")) + self.patched_funcs.enter_context( + mock.patch.object(util, 'which', lambda _: True)) + self.patched_funcs.enter_context( + mock.patch.object(util, 'subp', _dmidecode_subp)) + + def patch_mapping(self, new_mapping): + self.patched_funcs.enter_context( + mock.patch('cloudinit.util.DMIDECODE_TO_DMI_SYS_MAPPING', + new_mapping)) + + def test_sysfs_used_with_key_in_mapping_and_file_on_disk(self): + self.patch_mapping({'mapped-key': 'mapped-value'}) + expected_dmi_value = 'sys-used-correctly' + self._create_sysfs_file('mapped-value', expected_dmi_value) + self._configure_dmidecode_return('mapped-key', 'wrong-wrong-wrong') + self.assertEqual(expected_dmi_value, util.read_dmi_data('mapped-key')) + + def test_dmidecode_used_if_no_sysfs_file_on_disk(self): + self.patch_mapping({}) + self._create_sysfs_parent_directory() + expected_dmi_value = 'dmidecode-used' + self._configure_dmidecode_return('use-dmidecode', expected_dmi_value) + self.assertEqual(expected_dmi_value, + util.read_dmi_data('use-dmidecode')) + + def test_none_returned_if_neither_source_has_data(self): + self.patch_mapping({}) + self._configure_dmidecode_return('key', 'value') + self.assertEqual(None, util.read_dmi_data('expect-fail')) + + def test_none_returned_if_dmidecode_not_in_path(self): + self.patched_funcs.enter_context( + mock.patch.object(util, 'which', lambda _: False)) + self.patch_mapping({}) + self.assertEqual(None, util.read_dmi_data('expect-fail')) class TestMultiLog(helpers.FilesystemMockingTestCase): @@ -443,4 +452,11 @@ class TestMultiLog(helpers.FilesystemMockingTestCase): util.multi_log('message', log=log, log_level=log_level) self.assertEqual((log_level, mock.ANY), log.log.call_args[0]) + +class TestMessageFromString(helpers.TestCase): + + def test_unicode_not_messed_up(self): + roundtripped = util.message_from_string(u'\n').as_string() + self.assertNotIn('\x00', roundtripped) + # vi: ts=4 expandtab |