From b898aa96412367fb0cb0e920bb7d3b24933df414 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Jul 2013 00:48:54 -0700 Subject: Use the same method for registering custom vs default. Instead of having a register default handler and a register custom handler, just use the same function to do both but provide a parameter to affect how overwritting of previously existing content-types (which default handlers use to not overwrite custom ones). --- cloudinit/helpers.py | 9 +++++++-- cloudinit/stages.py | 11 ++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 1c46efde..e500220a 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -292,11 +292,16 @@ class ContentHandlers(object): def is_registered(self, content_type): return content_type in self.registered - def register(self, mod, initialized=False): + def register(self, mod, initialized=False, overwrite=True): types = set() for t in mod.list_types(): + if overwrite: + types.add(t) + else: + if not self.is_registered(t): + types.add(t) + for t in types: self.registered[t] = mod - types.add(t) if initialized and mod not in self.initialized: self.initialized.append(mod) return types diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 3e49e8c5..d7211eb6 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -375,7 +375,8 @@ class Init(object): mod = importer.import_module(mod_locs[0]) mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) - LOG.debug("Added handler for %s from %s", types, fname) + LOG.debug("Added custom handler for %s from %s", types, + fname) except Exception: util.logexc(LOG, "Failed to register handler from %s", fname) @@ -386,10 +387,10 @@ class Init(object): # Register any other handlers that come from the default set. This # is done after the cloud-dir handlers so that the cdir modules can # take over the default user-data handler content-types. - def_handlers = self._default_userdata_handlers() - applied_def_handlers = c_handlers.register_defaults(def_handlers) - if applied_def_handlers: - LOG.debug("Registered default handlers: %s", applied_def_handlers) + for mod in self._default_userdata_handlers(): + types = c_handlers.register(mod, overwrite=False) + if types: + LOG.debug("Added default handler for %s from %s", types, mod) # Form our cloud interface data = self.cloudify() -- cgit v1.2.3 From 18a625f958aaecf82a82db24b31290408d27882a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Jul 2013 00:51:14 -0700 Subject: Only do the debug log if types registered. --- cloudinit/stages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d7211eb6..07c55802 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -375,8 +375,9 @@ class Init(object): mod = importer.import_module(mod_locs[0]) mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) - LOG.debug("Added custom handler for %s from %s", types, - fname) + if types: + LOG.debug("Added custom handler for %s from %s", + types, fname) except Exception: util.logexc(LOG, "Failed to register handler from %s", fname) -- cgit v1.2.3 From d04b027003faf526028740d34421693396fcabb1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 24 Jul 2013 00:53:43 -0700 Subject: Remove the old register_defaults function. --- cloudinit/helpers.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index e500220a..e5eac6a7 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -315,15 +315,6 @@ class ContentHandlers(object): def iteritems(self): return self.registered.iteritems() - def register_defaults(self, defs): - registered = set() - for mod in defs: - for t in mod.list_types(): - if not self.is_registered(t): - self.registered[t] = mod - registered.add(t) - return registered - class Paths(object): def __init__(self, path_cfgs, ds=None): -- cgit v1.2.3 From ee9e56a9d286d34b20bbe3de200872d1eee1af38 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Sep 2013 22:43:51 -0700 Subject: Add config drive support for random_seed A new field in the metadata has emerged, one that provides a way to seed the linux random generator. Add support for writing the seed and rewrite parts of the on_boot code to use a little helper class. LP: #1198297 --- cloudinit/distros/__init__.py | 5 +++ cloudinit/sources/DataSourceConfigDrive.py | 49 +++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 74e95797..49b129ae 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -52,6 +52,7 @@ class Distro(object): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" + random_seed_fn = "/var/lib/random-seed" def __init__(self, name, cfg, paths): self._paths = paths @@ -169,6 +170,10 @@ class Distro(object): distros.extend(OSFAMILIES[family]) return distros + def set_random_seed(self, seed): + if self.random_seed_fn: + util.write_file(self.random_seed_fn, seed, mode=0600) + def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 835f2a9a..23f791aa 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import base64 import json import os @@ -41,6 +42,30 @@ DEFAULT_METADATA = { VALID_DSMODES = ("local", "net", "pass", "disabled") +class ConfigDriveHelper(object): + def __init__(self, distro): + self.distro = distro + + def on_first_boot(self, data): + if 'network_config' in data: + LOG.debug("Updating network interfaces from config drive") + self.distro.apply_network(data['network_config']) + files = data.get('files') + if files: + LOG.debug("Writing %s injected files", len(files)) + try: + write_files(files) + except IOError: + util.logexc(LOG, "Failed writing files") + random_seed = util.get_cfg_by_path(data, ('metadata', 'random_seed')) + if random_seed is not None: + LOG.debug("Writing random seed") + try: + self.distro.set_random_seed(random_seed) + except IOError: + util.logexc(LOG, "Failed writing random seed") + + class DataSourceConfigDrive(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -49,6 +74,7 @@ class DataSourceConfigDrive(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None + self.helper = ConfigDriveHelper(distro) def __str__(self): root = sources.DataSource.__str__(self) @@ -187,20 +213,8 @@ class DataSourceConfigDrive(sources.DataSource): # instance-id prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - - if ('network_config' in results and self.dsmode == "local" and - prev_iid != cur_iid): - LOG.debug("Updating network interfaces from config drive (%s)", - dsmode) - self.distro.apply_network(results['network_config']) - - # file writing occurs in local mode (to be as early as possible) - if self.dsmode == "local" and prev_iid != cur_iid and results['files']: - LOG.debug("writing injected files") - try: - write_files(results['files']) - except: - util.logexc(LOG, "Failed writing files") + if prev_iid != cur_iid and self.dsmode == "local": + self.helper.on_first_boot(results) # dsmode != self.dsmode here if: # * dsmode = "pass", pass means it should only copy files and then @@ -338,6 +352,13 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"): except KeyError: raise BrokenConfigDriveDir("No uuid entry in metadata") + if 'random_seed' in results['metadata']: + random_seed = results['metadata']['random_seed'] + try: + results['metadata']['random_seed'] = base64.b64decode(random_seed) + except (ValueError, TypeError) as exc: + raise BrokenConfigDriveDir("Badly formatted random_seed: %s" % exc) + def read_content_path(item): # do not use os.path.join here, as content_path starts with / cpath = os.path.sep.join((source_dir, "openstack", -- cgit v1.2.3 From 73863ed53d87e0eb3c24449ef164089531bc9679 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Sep 2013 22:46:03 -0700 Subject: Ensure data is initialized to a dict if its empty/none LP: #1198297 --- cloudinit/sources/DataSourceConfigDrive.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 23f791aa..af205d86 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -47,6 +47,8 @@ class ConfigDriveHelper(object): self.distro = distro def on_first_boot(self, data): + if not data: + data = {} if 'network_config' in data: LOG.debug("Updating network interfaces from config drive") self.distro.apply_network(data['network_config']) -- cgit v1.2.3 From af2264dd823d2861a470b566c558ac764f255ef4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Sep 2013 22:52:28 -0700 Subject: Save only 512 bytes and use different file paths for debian/rhel LP: #1198297 --- cloudinit/distros/__init__.py | 5 +++-- cloudinit/distros/debian.py | 1 + cloudinit/distros/rhel.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 49b129ae..3c536f2b 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -52,7 +52,7 @@ class Distro(object): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" - random_seed_fn = "/var/lib/random-seed" + random_seed_fn = None def __init__(self, name, cfg, paths): self._paths = paths @@ -172,7 +172,8 @@ class Distro(object): def set_random_seed(self, seed): if self.random_seed_fn: - util.write_file(self.random_seed_fn, seed, mode=0600) + # Ensure we only write 512 bytes worth + util.write_file(self.random_seed_fn, seed[0:512], mode=0600) def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 8fe49cbe..db4afc76 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -44,6 +44,7 @@ class Distro(distros.Distro): network_conf_fn = "/etc/network/interfaces" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" + random_seed_fn = "/var/lib/urandom/random-seed" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 30195384..96df9ae2 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -49,6 +49,7 @@ class Distro(distros.Distro): network_script_tpl = '/etc/sysconfig/network-scripts/ifcfg-%s' resolve_conf_fn = "/etc/resolv.conf" tz_local_fn = "/etc/localtime" + random_seed_fn = "/var/lib/random-seed" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) -- cgit v1.2.3 From 5252152361e0902658f4eb3ded732228a4f96128 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 2 Sep 2013 22:56:48 -0700 Subject: Raise when no seed filename provided LP: #1198297 --- cloudinit/distros/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3c536f2b..8cffb0ee 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -171,9 +171,11 @@ class Distro(object): return distros def set_random_seed(self, seed): - if self.random_seed_fn: - # Ensure we only write 512 bytes worth - util.write_file(self.random_seed_fn, seed[0:512], mode=0600) + if not self.random_seed_fn: + raise IOError("No random seed filename provided for %s" + % (self.name)) + # Ensure we only write 512 bytes worth + util.write_file(self.random_seed_fn, seed[0:512], mode=0600) def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname -- cgit v1.2.3 From c3e070de802ebc0f44722d4238f5447b93cc9fac Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 3 Sep 2013 23:51:51 -0700 Subject: Review adjustments. --- cloudinit/config/cc_seed_random.py | 36 ++++++++++++++++++++++++++++++ cloudinit/distros/__init__.py | 8 ++++--- cloudinit/distros/debian.py | 1 - cloudinit/distros/rhel.py | 1 - cloudinit/sources/DataSourceConfigDrive.py | 7 ------ config/cloud.cfg | 1 + 6 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 cloudinit/config/cc_seed_random.py diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py new file mode 100644 index 00000000..5d9890d5 --- /dev/null +++ b/cloudinit/config/cc_seed_random.py @@ -0,0 +1,36 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.settings import PER_INSTANCE + +frequency = PER_INSTANCE + + +def handle(name, cfg, cloud, log, _args): + random_seed = None + # Prefer metadata over cfg for random_seed + for src in (cloud.datasource.metadata, cfg): + if not src: + continue + tmp_random_seed = src.get('random_seed') + if tmp_random_seed and isinstance(tmp_random_seed, (str, basestring)): + random_seed = tmp_random_seed + break + if random_seed: + log.debug("%s: setting random seed", name) + cloud.distro.set_random_seed(random_seed) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8cffb0ee..5642b529 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -52,7 +52,7 @@ class Distro(object): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" - random_seed_fn = None + random_seed_fn = '/dev/urandom' def __init__(self, name, cfg, paths): self._paths = paths @@ -171,11 +171,13 @@ class Distro(object): return distros def set_random_seed(self, seed): - if not self.random_seed_fn: + if not self.random_seed_fn or not os.path.exists(self.random_seed_fn): raise IOError("No random seed filename provided for %s" % (self.name)) + if not seed: + raise IOError("Unable to set empty random seed") # Ensure we only write 512 bytes worth - util.write_file(self.random_seed_fn, seed[0:512], mode=0600) + util.append_file(self.random_seed_fn, seed[0:512]) def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index db4afc76..8fe49cbe 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -44,7 +44,6 @@ class Distro(distros.Distro): network_conf_fn = "/etc/network/interfaces" tz_conf_fn = "/etc/timezone" tz_local_fn = "/etc/localtime" - random_seed_fn = "/var/lib/urandom/random-seed" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 96df9ae2..30195384 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -49,7 +49,6 @@ class Distro(distros.Distro): network_script_tpl = '/etc/sysconfig/network-scripts/ifcfg-%s' resolve_conf_fn = "/etc/resolv.conf" tz_local_fn = "/etc/localtime" - random_seed_fn = "/var/lib/random-seed" def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index af205d86..4f437244 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -59,13 +59,6 @@ class ConfigDriveHelper(object): write_files(files) except IOError: util.logexc(LOG, "Failed writing files") - random_seed = util.get_cfg_by_path(data, ('metadata', 'random_seed')) - if random_seed is not None: - LOG.debug("Writing random seed") - try: - self.distro.set_random_seed(random_seed) - except IOError: - util.logexc(LOG, "Failed writing random seed") class DataSourceConfigDrive(sources.DataSource): diff --git a/config/cloud.cfg b/config/cloud.cfg index b61b8a7d..cce1f376 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -24,6 +24,7 @@ preserve_hostname: false # The modules that run in the 'init' stage cloud_init_modules: - migrator + - seed_random - bootcmd - write-files - growpart -- cgit v1.2.3 From e058913486519c2a9e036aad95f6e029dbc89966 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 6 Sep 2013 23:46:27 -0700 Subject: Add jsonschema for namespaced and verifiable module configuration checking as well as make most of the module logic happen in the module itself instead of interacting with the distro object. --- Requires | 4 ++ cloudinit/config/cc_seed_random.py | 94 +++++++++++++++++++++++++++++++++----- cloudinit/distros/__init__.py | 10 ---- cloudinit/exceptions.py | 21 +++++++++ 4 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 cloudinit/exceptions.py diff --git a/Requires b/Requires index f19c9691..b00dd58e 100644 --- a/Requires +++ b/Requires @@ -34,3 +34,7 @@ boto # For patching pieces of cloud-config together jsonpatch + +# For validating that a config modules needed configuration specified +# in a correct format that the module can understand +jsonschema diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 5d9890d5..acacb8f7 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -16,21 +16,91 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import base64 +from StringIO import StringIO + +import jsonschema +from jsonschema import exceptions as js_exc + +from cloudinit import exceptions as exc from cloudinit.settings import PER_INSTANCE +from cloudinit import util frequency = PER_INSTANCE +schema = { + 'type': 'object', + 'properties': { + "random_seed": { + "type": "object", + "oneOf": [ + {"$ref": "#/definitions/random_seed"}, + ], + }, + }, + "required": ["random_seed"], + "additionalProperties": True, + "definitions": { + 'random_seed': { + 'type': 'object', + "properties" : { + 'data': { + 'type': "string", + }, + 'file': { + 'type': 'string', + }, + 'encoding': { + "enum": ["base64", 'gzip', 'b64', 'gz', ''], + }, + }, + "additionalProperties": True, + }, + }, +} + + +def validate(cfg): + """Method that can be used to ask if the given configuration will be + accepted as valid by this module, without having to actually activate this + module.""" + try: + jsonschema.validate(cfg, schema) + except js_exc.ValidationError as e: + raise exc.FormatValidationError("Invalid configuration: %s" % str(e)) + + +def _decode(data, encoding=None): + if not encoding: + return data + if not data: + return '' + if encoding.lower() in ['base64', 'b64']: + return base64.b64decode(data) + elif encoding.lower() in ['gzip', 'gz']: + return util.decomp_gzip(data, quiet=False) + else: + raise IOError("Unknown random_seed encoding: %s" % (encoding)) def handle(name, cfg, cloud, log, _args): - random_seed = None - # Prefer metadata over cfg for random_seed - for src in (cloud.datasource.metadata, cfg): - if not src: - continue - tmp_random_seed = src.get('random_seed') - if tmp_random_seed and isinstance(tmp_random_seed, (str, basestring)): - random_seed = tmp_random_seed - break - if random_seed: - log.debug("%s: setting random seed", name) - cloud.distro.set_random_seed(random_seed) + if not cfg or "random_seed" not in cfg: + log.debug(("Skipping module named %s, " + "no 'random_seed' configuration found"), name) + return + + validate(cfg) + my_cfg = cfg['random_seed'] + seed_path = my_cfg.get('file', '/dev/urandom') + seed_buf = StringIO() + seed_buf.write(_decode(my_cfg.get('data', ''), + encoding=my_cfg.get('encoding'))) + + metadata = cloud.datasource.metadata + if metadata and 'random_seed' in metadata: + seed_buf.write(metadata['random_seed']) + + seed_data = seed_buf.getvalue() + if len(seed_data): + log.debug("%s: adding %s bytes of random seed entrophy to %s", name, + len(seed_data), seed_path) + util.append_file(seed_path, seed_data) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5642b529..74e95797 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -52,7 +52,6 @@ class Distro(object): ci_sudoers_fn = "/etc/sudoers.d/90-cloud-init-users" hostname_conf_fn = "/etc/hostname" tz_zone_dir = "/usr/share/zoneinfo" - random_seed_fn = '/dev/urandom' def __init__(self, name, cfg, paths): self._paths = paths @@ -170,15 +169,6 @@ class Distro(object): distros.extend(OSFAMILIES[family]) return distros - def set_random_seed(self, seed): - if not self.random_seed_fn or not os.path.exists(self.random_seed_fn): - raise IOError("No random seed filename provided for %s" - % (self.name)) - if not seed: - raise IOError("Unable to set empty random seed") - # Ensure we only write 512 bytes worth - util.append_file(self.random_seed_fn, seed[0:512]) - def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname diff --git a/cloudinit/exceptions.py b/cloudinit/exceptions.py new file mode 100644 index 00000000..c09d15b1 --- /dev/null +++ b/cloudinit/exceptions.py @@ -0,0 +1,21 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Yahoo! Inc. +# +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class FormatValidationError(Exception): + pass -- cgit v1.2.3 From 2ee2d10a042c96160e4745431d1d0c25904b5d88 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 6 Sep 2013 23:54:51 -0700 Subject: Ensure validate checks key existence. --- cloudinit/config/cc_seed_random.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index acacb8f7..592d253f 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -63,6 +63,8 @@ def validate(cfg): """Method that can be used to ask if the given configuration will be accepted as valid by this module, without having to actually activate this module.""" + if not cfg or "random_seed" not in cfg: + return try: jsonschema.validate(cfg, schema) except js_exc.ValidationError as e: -- cgit v1.2.3 From 0575c837a6ca6c2a6d554177d9e0540fd70a7843 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 7 Sep 2013 00:07:26 -0700 Subject: Ensure udelta is valid It appears that udelta could have been left undefined or left defined as a string "N/A" and then put threw a float formatter previously. Fix that by ensure its set to a default and put strong checking to make sure it is a float before using float formatting. --- cloudinit/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 4a74ba57..5032cc47 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1791,15 +1791,19 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): ret = func(*args, **kwargs) finally: delta = time.time() - start + udelta = None if ustart is not None: try: udelta = float(uptime()) - ustart except ValueError: - udelta = "N/A" + pass tmsg = " took %0.3f seconds" % delta if get_uptime: - tmsg += "(%0.2f)" % udelta + if isinstance(udelta, (float)): + tmsg += " (%0.2f)" % udelta + else: + tmsg += " (N/A)" try: logfunc(msg + tmsg) except: -- cgit v1.2.3 From 0843952235072079bb66bd2fe9e96057df11c228 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 8 Sep 2013 09:31:03 -0400 Subject: tools/read-dependencies, read-version: cleanups, and use sed not grep There are just some cleanups here, and use of simply 'sed' rather than grep and cut. The motivation is to support running with non gnu 'grep' that doesn't have -P. --- tools/read-dependencies | 14 +++++++------- tools/read-version | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/read-dependencies b/tools/read-dependencies index cadb09a8..87db5d83 100755 --- a/tools/read-dependencies +++ b/tools/read-dependencies @@ -12,20 +12,20 @@ find_root() { [ $? -eq 0 -a -f "${topd}/setup.py" ] || return ROOT_DIR="$topd" } +fail() { echo "$0:" "$@" 1>&2; exit 1; } if ! find_root; then - echo "Unable to locate 'setup.py' file that should" \ - "exist in the cloud-init root directory." 1>&2 - exit 1; + fail "Unable to locate 'setup.py' file that should " \ + "exist in the cloud-init root directory." fi REQUIRES="$ROOT_DIR/Requires" if [ ! -e "$REQUIRES" ]; then - echo "Unable to find 'Requires' file located at $REQUIRES" - exit 1 + fail "Unable to find 'Requires' file located at '$REQUIRES'" fi -# Filter out comments and empty liens -DEPS=$(grep -Pv "^\s*#" "$REQUIRES" | grep -Pv '^\s*$') +# Filter out comments and empty lines +DEPS=$(sed -n -e 's,#.*,,' -e '/./p' "$REQUIRES") || + fail "failed to read deps from '${REQUIRES}'" echo "$DEPS" | sort -d -f diff --git a/tools/read-version b/tools/read-version index c76b24a9..c37228f8 100755 --- a/tools/read-version +++ b/tools/read-version @@ -12,20 +12,20 @@ find_root() { [ $? -eq 0 -a -f "${topd}/setup.py" ] || return ROOT_DIR="$topd" } +fail() { echo "$0:" "$@" 1>&2; exit 1; } if ! find_root; then - echo "Unable to locate 'setup.py' file that should" \ - "exist in the cloud-init root directory." 1>&2 - exit 1; + fail "Unable to locate 'setup.py' file that should " \ + "exist in the cloud-init root directory." fi CHNG_LOG="$ROOT_DIR/ChangeLog" -if [ ! -e "$CHNG_LOG" ] -then - echo "Unable to find 'ChangeLog' file located at $CHNG_LOG" - exit 1 +if [ ! -e "$CHNG_LOG" ]; then + fail "Unable to find 'ChangeLog' file located at '$CHNG_LOG'" fi -VERSION=$(grep -P "\d+.\d+.\d+:" "$CHNG_LOG" | cut -f1 -d ":" | head -n 1) +VERSION=$(sed -n '/^[0-9]\+[.][0-9]\+[.][0-9]\+:/ {s/://; p; :a;n; ba}' \ + "$CHNG_LOG") || + fail "failed to get version from '$CHNG_LOG'" echo "$VERSION" -- cgit v1.2.3 From e56659253c4284be4c78d373d3f0a1deab9bd201 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 8 Sep 2013 22:36:28 -0700 Subject: Add test + remove jsonschema (for now) --- Requires | 4 - cloudinit/config/cc_seed_random.py | 53 +------- .../test_handler/test_handler_seed_random.py | 150 +++++++++++++++++++++ 3 files changed, 153 insertions(+), 54 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_seed_random.py diff --git a/Requires b/Requires index b00dd58e..f19c9691 100644 --- a/Requires +++ b/Requires @@ -34,7 +34,3 @@ boto # For patching pieces of cloud-config together jsonpatch - -# For validating that a config modules needed configuration specified -# in a correct format that the module can understand -jsonschema diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 592d253f..22a31f29 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -19,64 +19,18 @@ import base64 from StringIO import StringIO -import jsonschema -from jsonschema import exceptions as js_exc - -from cloudinit import exceptions as exc from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE -schema = { - 'type': 'object', - 'properties': { - "random_seed": { - "type": "object", - "oneOf": [ - {"$ref": "#/definitions/random_seed"}, - ], - }, - }, - "required": ["random_seed"], - "additionalProperties": True, - "definitions": { - 'random_seed': { - 'type': 'object', - "properties" : { - 'data': { - 'type': "string", - }, - 'file': { - 'type': 'string', - }, - 'encoding': { - "enum": ["base64", 'gzip', 'b64', 'gz', ''], - }, - }, - "additionalProperties": True, - }, - }, -} - - -def validate(cfg): - """Method that can be used to ask if the given configuration will be - accepted as valid by this module, without having to actually activate this - module.""" - if not cfg or "random_seed" not in cfg: - return - try: - jsonschema.validate(cfg, schema) - except js_exc.ValidationError as e: - raise exc.FormatValidationError("Invalid configuration: %s" % str(e)) def _decode(data, encoding=None): - if not encoding: - return data if not data: return '' - if encoding.lower() in ['base64', 'b64']: + if not encoding or encoding.lower() in ['raw']: + return data + elif encoding.lower() in ['base64', 'b64']: return base64.b64decode(data) elif encoding.lower() in ['gzip', 'gz']: return util.decomp_gzip(data, quiet=False) @@ -90,7 +44,6 @@ def handle(name, cfg, cloud, log, _args): "no 'random_seed' configuration found"), name) return - validate(cfg) my_cfg = cfg['random_seed'] seed_path = my_cfg.get('file', '/dev/urandom') seed_buf = StringIO() diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py new file mode 100644 index 00000000..458b0028 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -0,0 +1,150 @@ + # Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Author: Juerg Haefliger +# +# Based on test_handler_set_hostname.py +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.config import cc_seed_random + +import base64 +import tempfile +import gzip + +from StringIO import StringIO + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from cloudinit.sources import DataSourceNone + +from tests.unittests import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestRandomSeed(t_help.TestCase): + def setUp(self): + super(TestRandomSeed, self).setUp() + self._seed_file = tempfile.mktemp() + + def tearDown(self): + util.del_file(self._seed_file) + + def _compress(self, text): + contents = StringIO() + gz_fh = gzip.GzipFile(mode='wb', fileobj=contents) + gz_fh.write(text) + gz_fh.close() + return contents.getvalue() + + def _get_cloud(self, distro, metadata=None): + paths = helpers.Paths({}) + cls = distros.fetch(distro) + ubuntu_distro = cls(distro, {}, paths) + ds = DataSourceNone.DataSourceNone({}, ubuntu_distro, paths) + if metadata: + ds.metadata = metadata + return cloud.Cloud(ds, paths, {}, ubuntu_distro, None) + + def test_append_random(self): + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': 'tiny-tim-was-here', + } + } + cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) + contents = util.load_file(self._seed_file) + self.assertEquals("tiny-tim-was-here", contents) + + def test_append_random_unknown_encoding(self): + data = self._compress("tiny-toe") + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': data, + 'encoding': 'special_encoding', + } + } + self.assertRaises(IOError, cc_seed_random.handle, 'test', cfg, + self._get_cloud('ubuntu'), LOG, []) + + def test_append_random_gzip(self): + data = self._compress("tiny-toe") + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': data, + 'encoding': 'gzip', + } + } + cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) + contents = util.load_file(self._seed_file) + self.assertEquals("tiny-toe", contents) + + def test_append_random_gz(self): + data = self._compress("big-toe") + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': data, + 'encoding': 'gz', + } + } + cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) + contents = util.load_file(self._seed_file) + self.assertEquals("big-toe", contents) + + def test_append_random_base64(self): + data = base64.b64encode('bubbles') + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': data, + 'encoding': 'base64', + } + } + cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) + contents = util.load_file(self._seed_file) + self.assertEquals("bubbles", contents) + + def test_append_random_b64(self): + data = base64.b64encode('kit-kat') + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': data, + 'encoding': 'b64', + } + } + cc_seed_random.handle('test', cfg, self._get_cloud('ubuntu'), LOG, []) + contents = util.load_file(self._seed_file) + self.assertEquals("kit-kat", contents) + + def test_append_random_metadata(self): + cfg = { + 'random_seed': { + 'file': self._seed_file, + 'data': 'tiny-tim-was-here', + } + } + c = self._get_cloud('ubuntu', {'random_seed': '-so-was-josh'}) + cc_seed_random.handle('test', cfg, c, LOG, []) + contents = util.load_file(self._seed_file) + self.assertEquals('tiny-tim-was-here-so-was-josh', contents) -- cgit v1.2.3 From 1bda6f6d525150966ae6d611222c659838ee8669 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 9 Sep 2013 16:30:56 -0400 Subject: remove cloudinit/exceptions.py --- cloudinit/exceptions.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 cloudinit/exceptions.py diff --git a/cloudinit/exceptions.py b/cloudinit/exceptions.py deleted file mode 100644 index c09d15b1..00000000 --- a/cloudinit/exceptions.py +++ /dev/null @@ -1,21 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Yahoo! Inc. -# -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -class FormatValidationError(Exception): - pass -- cgit v1.2.3 From 8e5b1fef27fa715e9ff8ef0130ada4869f8258e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 9 Sep 2013 17:09:43 -0400 Subject: add support for reading random_seed on azure also. --- cloudinit/sources/DataSourceAzure.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 66d7728b..7c50de4c 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -106,6 +106,14 @@ class DataSourceAzureNet(sources.DataSource): if found == ddir: LOG.debug("using files cached in %s", ddir) + rseedf = "/sys/firmware/acpi/tables/OEM0" + if os.path.isfile(rseedf): + try: + with open(rseedf, "rb") as fp: + self.metadata['random_seed'] = fp.read() + except: + LOG.warn("random seed '%s' existed but read failed", rseedf) + # now update ds_cfg to reflect contents pass in config usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) -- cgit v1.2.3 From c74d928cc681ee98d0a3893dbe1ee8ff5fe361de Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 9 Sep 2013 19:44:50 -0400 Subject: populate random_seed in azure data source Azure provides a random bit of data at '/sys/firmware/acpi/tables/OEM0'. The walinux calls this "Entropy in ACPI table provided by Hyper-V". --- cloudinit/sources/DataSourceAzure.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 7c50de4c..a77c3d9a 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -106,13 +106,10 @@ class DataSourceAzureNet(sources.DataSource): if found == ddir: LOG.debug("using files cached in %s", ddir) - rseedf = "/sys/firmware/acpi/tables/OEM0" - if os.path.isfile(rseedf): - try: - with open(rseedf, "rb") as fp: - self.metadata['random_seed'] = fp.read() - except: - LOG.warn("random seed '%s' existed but read failed", rseedf) + # azure / hyper-v provides random data here + seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True) + if seed: + self.metadata['random_seed'] = seed # now update ds_cfg to reflect contents pass in config usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) -- cgit v1.2.3