diff options
25 files changed, 555 insertions, 166 deletions
@@ -1,4 +1,13 @@ +0.7.1: + - sysvinit: fix missing dependency in cloud-init job for RHEL 5.6 + - config-drive: map hostname to local-hostname (LP: #1061964) + - landscape: install landscape-client package if not installed. + only take action if cloud-config is present (LP: #1066115) 0.7.0: + - add a 'exception_cb' argument to 'wait_for_url'. If provided, this + method will be called back with the exception received and the message. + - utilize the 'exception_cb' above to modify the oauth timestamp in + DataSourceMAAS requests if a 401 or 403 is received. (LP: #978127) - catch signals and exit rather than stack tracing - if logging fails, enable a fallback logger by patching the logging module - do not 'start networking' in cloud-init-nonet, but add diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 4e2e06bb..92d428b7 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -18,12 +18,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit import distros as ds + from cloudinit import util distros = ['ubuntu', 'debian'] -def handle(name, cfg, _cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: value = args[0] else: @@ -56,16 +61,20 @@ def handle(name, cfg, _cloud, log, args): shcmd = "" if mod_user: - user = util.get_cfg_option_str(cfg, "user", "ubuntu") - shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) - shcmd += " || X=$(($X+1)); " + (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) + (user, _user_config) = ds.extract_default(users) + if not user: + log.warn(("No default byobu user provided, " + "can not launch %s for the default user"), bl_inst) + else: + shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst) + shcmd += " || X=$(($X+1)); " if mod_sys: shcmd += "echo \"%s\" | debconf-set-selections" % dc_val shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive" shcmd += " || X=$(($X+1)); " - cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] - - log.debug("Setting byobu to %s", value) - - util.subp(cmd, capture=False) + if len(shcmd): + cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")] + log.debug("Setting byobu to %s", value) + util.subp(cmd, capture=False) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 7cfb8296..331559f4 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -59,6 +59,10 @@ def handle(_name, cfg, cloud, log, _args): raise RuntimeError(("'landscape' key existed in config," " but not a dictionary type," " is a %s instead"), util.obj_name(ls_cloudcfg)) + if not ls_cloudcfg: + return + + cloud.distro.install_packages(["landscape-client"]) merge_data = [ LSC_BUILTIN_CFG, @@ -79,8 +83,7 @@ def handle(_name, cfg, cloud, log, _args): util.write_file(lsc_client_fn, contents.getvalue()) log.debug("Wrote landscape config file to %s", lsc_client_fn) - if ls_cloudcfg: - util.write_file(LS_DEFAULT_FILE, "RUN=1\n") + util.write_file(LS_DEFAULT_FILE, "RUN=1\n") def merge_together(objs): diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index a017e6b6..26c558ad 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -20,6 +20,11 @@ import sys +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit import distros as ds + from cloudinit import ssh_util from cloudinit import util @@ -50,18 +55,10 @@ def handle(_name, cfg, cloud, log, args): expire = util.get_cfg_option_bool(chfg, 'expire', expire) if not plist and password: - user = cloud.distro.get_default_user() - - if 'users' in cfg: - - user_zero = cfg['users'][0] - - if isinstance(user_zero, dict) and 'name' in user_zero: - user = user_zero['name'] - + (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) + (user, _user_config) = ds.extract_default(users) if user: plist = "%s:%s" % (user, password) - else: log.warn("No default or defined user to change password for.") diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 0ded62ba..32e48c30 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -21,6 +21,11 @@ import glob import os +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit import distros as ds + from cloudinit import ssh_util from cloudinit import util @@ -102,16 +107,8 @@ def handle(_name, cfg, cloud, log, _args): " %s to file %s"), keytype, keyfile) try: - # TODO(utlemming): consolidate this stanza that occurs in: - # cc_ssh_import_id, cc_set_passwords, maybe cc_users_groups.py - user = cloud.distro.get_default_user() - - if 'users' in cfg: - user_zero = cfg['users'][0] - - if user_zero != "default": - user = user_zero - + (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) + (user, _user_config) = ds.extract_default(users) disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index f9bdf5fc..8c9a8806 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -21,7 +21,11 @@ import hashlib from prettytable import PrettyTable -from cloudinit import distros +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit import distros as ds + from cloudinit import ssh_util from cloudinit import util @@ -41,8 +45,10 @@ def _gen_fingerprint(b64_text, hash_meth='md5'): hasher = hashlib.new(hash_meth) hasher.update(base64.b64decode(b64_text)) return ":".join(_split_hash(hasher.hexdigest())) - except TypeError: + except (TypeError, ValueError): # Raised when b64 not really b64... + # or when the hash type is not really + # a known/supported hash type... return '?' @@ -92,8 +98,8 @@ def handle(name, cfg, cloud, log, _args): hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") extract_func = ssh_util.extract_authorized_keys - (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro) + (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) for (user_name, _cfg) in users.items(): (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths) - _pprint_key_entries(user_name, auth_key_fn, auth_key_entries, - hash_meth) + _pprint_key_entries(user_name, auth_key_fn, + auth_key_entries, hash_meth) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 08fb63c6..83af36e9 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -18,6 +18,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit import distros as ds + from cloudinit import util import pwd @@ -39,33 +44,27 @@ def handle(_name, cfg, cloud, log, args): return # import for cloudinit created users + (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) elist = [] - for user_cfg in cfg['users']: - user = None + for (user, user_cfg) in users.items(): import_ids = [] - - if isinstance(user_cfg, str) and user_cfg == "default": - user = cloud.distro.get_default_user() - if not user: - continue - + if user_cfg['default']: import_ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) - - elif isinstance(user_cfg, dict): - user = None - import_ids = [] - + else: try: - user = user_cfg['name'] import_ids = user_cfg['ssh_import_id'] - - if import_ids and isinstance(import_ids, str): - import_ids = str(import_ids).split(',') - except: - log.debug("user %s is not configured for ssh_import" % user) + log.debug("User %s is not configured for ssh_import_id", user) continue + try: + import_ids = util.uniq_merge(import_ids) + import_ids = [str(i) for i in import_ids] + except: + log.debug("User %s is not correctly configured for ssh_import_id", + user) + continue + if not len(import_ids): continue diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index aa6e0579..bf5b4581 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -16,7 +16,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from cloudinit import distros +# Ensure this is aliased to a name not 'distros' +# since the module attribute 'distros' +# is a list of distros that are supported, not a sub-module +from cloudinit import distros as ds from cloudinit.settings import PER_INSTANCE @@ -24,7 +27,7 @@ frequency = PER_INSTANCE def handle(name, cfg, cloud, _log, _args): - (users, groups) = distros.normalize_users_groups(cfg, cloud.distro) + (users, groups) = ds.normalize_users_groups(cfg, cloud.distro) for (name, members) in groups.items(): cloud.distro.create_group(name, members) for (user, config) in users.items(): diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index cdae1add..6ef63442 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -24,6 +24,7 @@ from StringIO import StringIO import abc +import itertools import os import re @@ -36,10 +37,7 @@ LOG = logging.getLogger(__name__) class Distro(object): - __metaclass__ = abc.ABCMeta - default_user = None - default_user_groups = None def __init__(self, name, cfg, paths): self._paths = paths @@ -175,20 +173,10 @@ class Distro(object): return False def get_default_user(self): - if not self.default_user: - return None - user_cfg = { - 'name': self.default_user, - 'plain_text_passwd': self.default_user, - 'home': "/home/%s" % (self.default_user), - 'shell': "/bin/bash", - 'lock_passwd': True, - 'gecos': "%s" % (self.default_user.title()), - 'sudo': "ALL=(ALL) NOPASSWD:ALL", - } - if self.default_user_groups: - user_cfg['groups'] = _uniq_merge_sorted(self.default_user_groups) - return user_cfg + return self.get_option('default_user') + + def get_default_user_groups(self): + return self.get_option('default_user_groups') def create_user(self, name, **kwargs): """ @@ -323,6 +311,7 @@ class Distro(object): else: with open(sudo_base, 'a') as f: f.write(sudoers_contents) + LOG.debug("added '#includedir %s' to %s" % (path, sudo_base)) except IOError as e: util.logexc(LOG, "Failed to write %s" % sudo_base, e) raise e @@ -431,39 +420,27 @@ def _get_arch_package_mirror_info(package_mirrors, arch): return default -def _uniq_merge_sorted(*lists): - return sorted(_uniq_merge(*lists)) - - -def _uniq_merge(*lists): - combined_list = [] - for a_list in lists: - if isinstance(a_list, (str, basestring)): - a_list = a_list.strip().split(",") - else: - a_list = [str(a) for a in a_list] - a_list = [a.strip() for a in a_list if a.strip()] - combined_list.extend(a_list) - uniq_list = [] - for a in combined_list: - if a in uniq_list: - continue - else: - uniq_list.append(a) - return uniq_list - - +# Normalizes a input group configuration +# which can be a comma seperated list of +# group names, or a list of group names +# or a python dictionary of group names +# to a list of members of that group. +# +# The output is a dictionary of group +# names => members of that group which +# is the standard form used in the rest +# of cloud-init def _normalize_groups(grp_cfg): if isinstance(grp_cfg, (str, basestring, list)): c_grp_cfg = {} - for i in _uniq_merge(grp_cfg): + for i in util.uniq_merge(grp_cfg): c_grp_cfg[i] = [] grp_cfg = c_grp_cfg groups = {} if isinstance(grp_cfg, (dict)): for (grp_name, grp_members) in grp_cfg.items(): - groups[grp_name] = _uniq_merge_sorted(grp_members) + groups[grp_name] = util.uniq_merge_sorted(grp_members) else: raise TypeError(("Group config must be list, dict " " or string types only and not %s") % @@ -471,6 +448,21 @@ def _normalize_groups(grp_cfg): return groups +# Normalizes a input group configuration +# which can be a comma seperated list of +# user names, or a list of string user names +# or a list of dictionaries with components +# that define the user config + 'name' (if +# a 'name' field does not exist then the +# default user is assumed to 'own' that +# configuration. +# +# The output is a dictionary of user +# names => user config which is the standard +# form used in the rest of cloud-init. Note +# the default user will have a special config +# entry 'default' which will be marked as true +# all other users will be marked as false. def _normalize_users(u_cfg, def_user_cfg=None): if isinstance(u_cfg, (dict)): ad_ucfg = [] @@ -486,12 +478,12 @@ def _normalize_users(u_cfg, def_user_cfg=None): " for key %s") % (util.obj_name(v), k)) u_cfg = ad_ucfg elif isinstance(u_cfg, (str, basestring)): - u_cfg = _uniq_merge_sorted(u_cfg) + u_cfg = util.uniq_merge_sorted(u_cfg) users = {} for user_config in u_cfg: if isinstance(user_config, (str, basestring, list)): - for u in _uniq_merge(user_config): + for u in util.uniq_merge(user_config): if u and u not in users: users[u] = {} elif isinstance(user_config, (dict)): @@ -524,22 +516,60 @@ def _normalize_users(u_cfg, def_user_cfg=None): # Fixup the default user into the real # default user name and replace it... + def_user = None if users and 'default' in users: def_config = users.pop('default') if def_user_cfg: + # Pickup what the default 'real name' is + # and any groups that are provided by the + # default config + def_user_cfg = def_user_cfg.copy() def_user = def_user_cfg.pop('name') def_groups = def_user_cfg.pop('groups', []) + # Pickup any config + groups for that user name + # that we may have previously extracted parsed_config = users.pop(def_user, {}) - users_groups = _uniq_merge_sorted(parsed_config.get('groups', []), - def_groups) + parsed_groups = parsed_config.get('groups', []) + # Now merge our extracted groups with + # anything the default config provided + users_groups = util.uniq_merge_sorted(parsed_groups, def_groups) parsed_config['groups'] = ",".join(users_groups) + # The real config for the default user is the + # combination of the default user config provided + # by the distro, the default user config provided + # by the above merging for the user 'default' and + # then the parsed config from the user's 'real name' + # which does not have to be 'default' (but could be) users[def_user] = util.mergemanydict([def_user_cfg, def_config, parsed_config]) + # Ensure that only the default user that we + # found (if any) is actually marked as being + # the default user + if users: + for (uname, uconfig) in users.items(): + if def_user and uname == def_user: + uconfig['default'] = True + else: + uconfig['default'] = False + return users +# Normalizes a set of user/users and group +# dictionary configuration into a useable +# format that the rest of cloud-init can +# understand using the default user +# provided by the input distrobution (if any) +# to allow for mapping of the 'default' user. +# +# Output is a dictionary of group names -> [member] (list) +# and a dictionary of user names -> user configuration (dict) +# +# If 'user' exists it will override +# the 'users'[0] entry (if a list) otherwise it will +# just become an entry in the returned dictionary (no override) def normalize_users_groups(cfg, distro): if not cfg: cfg = {} @@ -581,6 +611,33 @@ def normalize_users_groups(cfg, distro): return (users, groups) +# Given a user dictionary config it will +# extract the default user name and user config +# from that list and return that tuple or +# return (None, None) if no default user is +# found in the given input +def extract_default(users, default_name=None, default_config=None): + if not users: + users = {} + + def safe_find(entry): + config = entry[1] + if not config or 'default' not in config: + return False + else: + return config['default'] + + tmp_users = users.items() + tmp_users = dict(itertools.ifilter(safe_find, tmp_users)) + if not tmp_users: + return (default_name, default_config) + else: + name = tmp_users.keys()[0] + config = tmp_users[name] + config.pop('default', None) + return (name, config) + + def fetch(name): locs = importer.find_module(name, ['', __name__], diff --git a/cloudinit/distros/fedora.py b/cloudinit/distros/fedora.py index f65a820d..c777845d 100644 --- a/cloudinit/distros/fedora.py +++ b/cloudinit/distros/fedora.py @@ -28,4 +28,4 @@ LOG = logging.getLogger(__name__) class Distro(rhel.Distro): - default_user = 'ec2-user' + pass diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 4e697f82..c527f248 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -28,7 +28,4 @@ LOG = logging.getLogger(__name__) class Distro(debian.Distro): - - default_user = 'ubuntu' - default_user_groups = ("adm,audio,cdrom,dialout,floppy,video," - "plugdev,dip,netdev,sudo") + pass diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index b8154367..4af2e5ae 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -48,6 +48,7 @@ class DataSourceConfigDrive(sources.DataSource): self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None + self.ec2_metadata = None def __str__(self): mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode, @@ -55,6 +56,74 @@ class DataSourceConfigDrive(sources.DataSource): mstr += "[source=%s]" % (self.source) return mstr + def _ec2_name_to_device(self, name): + if not self.ec2_metadata: + return None + bdm = self.ec2_metadata.get('block-device-mapping', {}) + for (ent_name, device) in bdm.items(): + if name == ent_name: + return device + return None + + def _os_name_to_device(self, name): + device = None + try: + criteria = 'LABEL=%s' % (name) + if name in ['swap']: + criteria = 'TYPE=%s' % (name) + dev_entries = util.find_devs_with(criteria) + if dev_entries: + device = dev_entries[0] + except util.ProcessExecutionError: + pass + return device + + def _validate_device_name(self, device): + if not device: + return None + if not device.startswith("/"): + device = "/dev/%s" % device + if os.path.exists(device): + return device + # Durn, try adjusting the mapping + remapped = self._remap_device(os.path.basename(device)) + if remapped: + LOG.debug("Remapped device name %s => %s", device, remapped) + return remapped + return None + + def device_name_to_device(self, name): + # Translate a 'name' to a 'physical' device + if not name: + return None + # Try the ec2 mapping first + names = [name] + if name == 'root': + names.insert(0, 'ami') + if name == 'ami': + names.append('root') + device = None + LOG.debug("Using ec2 metadata lookup to find device %s", names) + for n in names: + device = self._ec2_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Try the openstack way second + if not device: + LOG.debug("Using os lookup to find device %s", names) + for n in names: + device = self._os_name_to_device(n) + device = self._validate_device_name(device) + if device: + break + # Ok give up... + if not device: + return None + else: + LOG.debug("Using cfg drive lookup mapped to device %s", device) + return device + def get_data(self): found = None md = {} @@ -85,6 +154,16 @@ class DataSourceConfigDrive(sources.DataSource): md = results['metadata'] md = util.mergedict(md, DEFAULT_METADATA) + # Perform some metadata 'fixups' + # + # OpenStack uses the 'hostname' key + # while most of cloud-init uses the metadata + # 'local-hostname' key instead so if it doesn't + # exist we need to make sure its copied over. + for (tgt, src) in [('local-hostname', 'hostname')]: + if tgt not in md and src in md: + md[tgt] = md[src] + user_dsmode = results.get('dsmode', None) if user_dsmode not in VALID_DSMODES + (None,): LOG.warn("user specified invalid mode: %s" % user_dsmode) @@ -133,6 +212,7 @@ class DataSourceConfigDrive(sources.DataSource): self.source = found self.metadata = md + self.ec2_metadata = results.get('ec2-metadata') self.userdata_raw = results.get('userdata') self.version = results['cfgdrive_ver'] @@ -217,7 +297,7 @@ def read_config_drive_dir_v2(source_dir, version="2012-08-10"): ('metadata', "openstack/%s/meta_data.json" % version, True, json.loads), ('userdata', "openstack/%s/user_data" % version, False, None), - ('ec2-metadata', "ec2/latest/metadata.json", False, json.loads), + ('ec2-metadata', "ec2/latest/meta-data.json", False, json.loads), ) results = {'userdata': None} diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index c7ad6d54..3686fa10 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -151,22 +151,6 @@ class DataSourceEc2(sources.DataSource): self.metadata_address = url2base.get(url) return bool(url) - def _remap_device(self, short_name): - # LP: #611137 - # the metadata service may believe that devices are named 'sda' - # when the kernel named them 'vda' or 'xvda' - # we want to return the correct value for what will actually - # exist in this instance - mappings = {"sd": ("vd", "xvd")} - for (nfrom, tlist) in mappings.iteritems(): - if not short_name.startswith(nfrom): - continue - for nto in tlist: - cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) - if os.path.exists(cand): - return cand - return None - def device_name_to_device(self, name): # Consult metadata service, that has # ephemeral0: sdb diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index c172150b..e187aec9 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -145,10 +145,10 @@ class DataSourceMAAS(sources.DataSource): def _except_cb(self, msg, exception): if not (isinstance(exception, urllib2.HTTPError) and - exception.code == 403): + (exception.code == 403 or exception.code == 401)): return if 'date' not in exception.headers: - LOG.warn("date field not in 403 headers") + LOG.warn("date field not in %d headers" % exception.code) return date = exception.headers['date'] @@ -336,7 +336,6 @@ if __name__ == "__main__": 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: - import yaml with open(args.config) as fp: cfg = util.load_yaml(fp.read()) if 'datasource' in cfg: diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 04083d0c..b22369a8 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -23,6 +23,7 @@ from email.mime.multipart import MIMEMultipart import abc +import os from cloudinit import importer from cloudinit import log as logging @@ -128,6 +129,22 @@ class DataSource(object): return keys + def _remap_device(self, short_name): + # LP: #611137 + # the metadata service may believe that devices are named 'sda' + # when the kernel named them 'vda' or 'xvda' + # we want to return the correct value for what will actually + # exist in this instance + mappings = {"sd": ("vd", "xvd")} + for (nfrom, tlist) in mappings.iteritems(): + if not short_name.startswith(nfrom): + continue + for nto in tlist: + cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) + if os.path.exists(cand): + return cand + return None + def device_name_to_device(self, _name): # translate a 'name' to a device # the primary function at this point is on ec2 diff --git a/cloudinit/util.py b/cloudinit/util.py index 46d490f7..f5a7ac12 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -249,6 +249,36 @@ def read_conf(fname): raise +# Merges X lists, and then keeps the +# unique ones, but orders by sort order +# instead of by the original order +def uniq_merge_sorted(*lists): + return sorted(uniq_merge(*lists)) + + +# Merges X lists and then iterates over those +# and only keeps the unique items (order preserving) +# and returns that merged and uniqued list as the +# final result. +# +# Note: if any entry is a string it will be +# split on commas and empty entries will be +# evicted and merged in accordingly. +def uniq_merge(*lists): + combined_list = [] + for a_list in lists: + if isinstance(a_list, (str, basestring)): + a_list = a_list.strip().split(",") + # Kickout the empty ones + a_list = [a for a in a_list if len(a)] + combined_list.extend(a_list) + uniq_list = [] + for i in combined_list: + if i not in uniq_list: + uniq_list.append(i) + return uniq_list + + def clean_filename(fn): for (k, v) in FN_REPLACEMENTS.iteritems(): fn = fn.replace(k, v) diff --git a/cloudinit/version.py b/cloudinit/version.py index 4599910c..12ff620a 100644 --- a/cloudinit/version.py +++ b/cloudinit/version.py @@ -20,7 +20,7 @@ from distutils import version as vr def version(): - return vr.StrictVersion("0.7.0") + return vr.StrictVersion("0.7.1") def version_string(): diff --git a/config/cloud.cfg b/config/cloud.cfg index b3411d11..f6c9065a 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -1,7 +1,9 @@ # The top level settings are used as module # and system configuration. -# Implement for Ubuntu only: create the default 'ubuntu' user +# A set of users which may be applied and/or used by various modules +# when a 'default' entry is found it will reference the 'default_user' +# from the distro configuration specified below users: - default @@ -71,6 +73,12 @@ cloud_final_modules: system_info: # This will affect which distro class gets used distro: ubuntu + # Default user name + that default users groups (if added/used) + default_user: + name: Ubuntu + lock_passwd: True + gecos: Ubuntu + groups: [adm, audio, cdrom, dialout, floppy, video, plugdev, dip, netdev] # Other config here will be given to the distro class and/or path classes paths: cloud_dir: /var/lib/cloud/ diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index 1a46c540..de5f321b 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -96,3 +96,14 @@ users: # foobar: ... # # users[0] (the first user in users) overrides the user directive. +# +# The 'default' user above references the distro's config: +# system_info: +# default_user: +# name: Ubuntu +# plain_text_passwd: 'ubuntu' +# home: /home/ubuntu +# shell: /bin/bash +# lock_passwd: True +# gecos: Ubuntu +# groups: [adm, audio, cdrom, dialout, floppy, video, plugdev, dip, netdev] diff --git a/packages/brpm b/packages/brpm index 1735f5bb..e6b03609 100755 --- a/packages/brpm +++ b/packages/brpm @@ -58,8 +58,7 @@ def get_log_header(version): a_rev = rev break if not a_rev: - return format_change_line(datetime.now(), - '??', version) + return None # Extract who made that tag as the header cmd = ['bzr', 'log', '-r%s' % (a_rev), '--timezone=utc'] @@ -107,6 +106,10 @@ def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn): subs['version'] = version subs['revno'] = revno subs['release'] = "bzr%s" % (revno) + if args.sub_release is not None: + subs['subrelease'] = "." + str(args.sub_release) + else: + subs['subrelease'] = '' subs['archive_name'] = arc_fn cmd = [util.abs_join(find_root(), 'tools', 'read-dependencies')] @@ -127,13 +130,23 @@ def generate_spec_contents(args, tmpl_fn, top_dir, arc_fn): # Format a nice changelog (as best as we can) changelog = util.load_file(util.abs_join(find_root(), 'ChangeLog')) changelog_lines = [] + missing_versions = 0 for line in changelog.splitlines(): if not line.strip(): continue if re.match(r"^\s*[\d][.][\d][.][\d]:\s*", line): line = line.strip(":") header = get_log_header(line) - changelog_lines.append(header) + if not header: + missing_versions += 1 + if missing_versions == 1: + # Must be using a new 'dev'/'trunk' release + changelog_lines.append(format_change_line(datetime.now(), '??')) + else: + sys.stderr.write(("Changelog version line %s " + "does not have a corresponding tag!\n") % (line)) + else: + changelog_lines.append(header) else: changelog_lines.append(line) subs['changelog'] = "\n".join(changelog_lines) @@ -166,6 +179,13 @@ def main(): " (default: %(default)s)"), default=False, action='store_true') + parser.add_argument('-s', "--sub-release", dest="sub_release", + metavar="RELEASE", + help=("a 'internal' release number to concat" + " with the bzr version number to form" + " the final version number"), + type=int, + default=None) parser.add_argument("-p", "--patch", dest="patches", help=("include the following patch when building"), default=[], diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 12f0b144..30bcd050 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -11,7 +11,7 @@ Name: cloud-init Version: ${version} -Release: ${release}%{?dist} +Release: ${release}${subrelease}%{?dist} Summary: Cloud instance init scripts Group: System Environment/Base diff --git a/sysvinit/cloud-init b/sysvinit/cloud-init index 4b44a615..f8ab5d5f 100755 --- a/sysvinit/cloud-init +++ b/sysvinit/cloud-init @@ -25,8 +25,8 @@ ### BEGIN INIT INFO # Provides: cloud-init -# Required-Start: $local_fs $network $named $remote_fs -# Should-Start: $time cloud-init-local +# Required-Start: $local_fs $network $named $remote_fs cloud-init-local +# Should-Start: $time # Required-Stop: # Should-Stop: # Default-Start: 3 5 diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 55573114..00379e03 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -2,10 +2,12 @@ from copy import copy import json import os import os.path -import shutil -import tempfile -from unittest import TestCase +import mocker +from mocker import MockerTestCase + +from cloudinit import helpers +from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit import util @@ -60,17 +62,140 @@ CFG_DRIVE_FILES_V2 = { 'openstack/latest/user_data': USER_DATA} -class TestConfigDriveDataSource(TestCase): +class TestConfigDriveDataSource(MockerTestCase): def setUp(self): super(TestConfigDriveDataSource, self).setUp() - self.tmp = tempfile.mkdtemp() + self.tmp = self.makeDir() - def tearDown(self): - try: - shutil.rmtree(self.tmp) - except OSError: - pass + def test_ec2_metadata(self): + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + found = ds.read_config_drive_dir(self.tmp) + self.assertTrue('ec2-metadata' in found) + ec2_md = found['ec2-metadata'] + self.assertEqual(EC2_META, ec2_md) + + def test_dev_os_remap(self): + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + found = ds.read_config_drive_dir(self.tmp) + cfg_ds.metadata = found['metadata'] + name_tests = { + 'ami': '/dev/vda1', + 'root': '/dev/vda1', + 'ephemeral0': '/dev/vda2', + 'swap': '/dev/vda3', + } + for name, dev_name in name_tests.items(): + my_mock = mocker.Mocker() + find_mock = my_mock.replace(util.find_devs_with, + spec=False, passthrough=False) + provided_name = dev_name[len('/dev/'):] + provided_name = "s" + provided_name[1:] + find_mock(mocker.ARGS) + my_mock.result([provided_name]) + exists_mock = my_mock.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + my_mock.result(False) + exists_mock(mocker.ARGS) + my_mock.result(True) + my_mock.replay() + device = cfg_ds.device_name_to_device(name) + my_mock.restore() + self.assertEquals(dev_name, device) + + def test_dev_os_map(self): + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + found = ds.read_config_drive_dir(self.tmp) + os_md = found['metadata'] + cfg_ds.metadata = os_md + name_tests = { + 'ami': '/dev/vda1', + 'root': '/dev/vda1', + 'ephemeral0': '/dev/vda2', + 'swap': '/dev/vda3', + } + for name, dev_name in name_tests.items(): + my_mock = mocker.Mocker() + find_mock = my_mock.replace(util.find_devs_with, + spec=False, passthrough=False) + find_mock(mocker.ARGS) + my_mock.result([dev_name]) + exists_mock = my_mock.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + my_mock.result(True) + my_mock.replay() + device = cfg_ds.device_name_to_device(name) + my_mock.restore() + self.assertEquals(dev_name, device) + + def test_dev_ec2_remap(self): + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + found = ds.read_config_drive_dir(self.tmp) + ec2_md = found['ec2-metadata'] + os_md = found['metadata'] + cfg_ds.ec2_metadata = ec2_md + cfg_ds.metadata = os_md + name_tests = { + 'ami': '/dev/vda1', + 'root': '/dev/vda1', + 'ephemeral0': '/dev/vda2', + 'swap': '/dev/vda3', + None: None, + 'bob': None, + 'root2k': None, + } + for name, dev_name in name_tests.items(): + my_mock = mocker.Mocker() + exists_mock = my_mock.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + my_mock.result(False) + exists_mock(mocker.ARGS) + my_mock.result(True) + my_mock.replay() + device = cfg_ds.device_name_to_device(name) + self.assertEquals(dev_name, device) + my_mock.restore() + + def test_dev_ec2_map(self): + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, + None, + helpers.Paths({})) + found = ds.read_config_drive_dir(self.tmp) + exists_mock = self.mocker.replace(os.path.exists, + spec=False, passthrough=False) + exists_mock(mocker.ARGS) + self.mocker.count(0, None) + self.mocker.result(True) + self.mocker.replay() + ec2_md = found['ec2-metadata'] + os_md = found['metadata'] + cfg_ds.ec2_metadata = ec2_md + cfg_ds.metadata = os_md + name_tests = { + 'ami': '/dev/sda1', + 'root': '/dev/sda1', + 'ephemeral0': '/dev/sda2', + 'swap': '/dev/sda3', + None: None, + 'bob': None, + 'root2k': None, + } + for name, dev_name in name_tests.items(): + device = cfg_ds.device_name_to_device(name) + self.assertEquals(dev_name, device) def test_dir_valid(self): """Verify a dir is read as such.""" diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index b319b673..8f0d8896 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -4,19 +4,27 @@ from cloudinit import distros from cloudinit import helpers from cloudinit import settings +bcfg = { + 'name': 'bob', + 'plain_text_passwd': 'ubuntu', + 'home': "/home/ubuntu", + 'shell': "/bin/bash", + 'lock_passwd': True, + 'gecos': "Ubuntu", + 'groups': ["foo"] +} + class TestUGNormalize(MockerTestCase): - def _make_distro(self, dtype, def_user=None, def_groups=None): + def _make_distro(self, dtype, def_user=None): cfg = dict(settings.CFG_BUILTIN) cfg['system_info']['distro'] = dtype paths = helpers.Paths(cfg['system_info']['paths']) distro_cls = distros.fetch(dtype) - distro = distro_cls(dtype, cfg['system_info'], paths) if def_user: - distro.default_user = def_user - if def_groups: - distro.default_user_groups = def_groups + cfg['system_info']['default_user'] = def_user.copy() + distro = distro_cls(dtype, cfg['system_info'], paths) return distro def _norm(self, cfg, distro): @@ -71,7 +79,7 @@ class TestUGNormalize(MockerTestCase): self.assertEquals({}, users) def test_users_simple_dict(self): - distro = self._make_distro('ubuntu', 'bob') + distro = self._make_distro('ubuntu', bcfg) ug_cfg = { 'users': { 'default': True, @@ -95,7 +103,7 @@ class TestUGNormalize(MockerTestCase): self.assertIn('bob', users) def test_users_simple_dict_no(self): - distro = self._make_distro('ubuntu', 'bob') + distro = self._make_distro('ubuntu', bcfg) ug_cfg = { 'users': { 'default': False, @@ -119,8 +127,8 @@ class TestUGNormalize(MockerTestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({}, users['joe']) - self.assertEquals({}, users['bob']) + self.assertEquals({'default': False}, users['joe']) + self.assertEquals({'default': False}, users['bob']) def test_users_simple(self): distro = self._make_distro('ubuntu') @@ -133,11 +141,11 @@ class TestUGNormalize(MockerTestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({}, users['joe']) - self.assertEquals({}, users['bob']) + self.assertEquals({'default': False}, users['joe']) + self.assertEquals({'default': False}, users['bob']) def test_users_old_user(self): - distro = self._make_distro('ubuntu', 'bob') + distro = self._make_distro('ubuntu', bcfg) ug_cfg = { 'user': 'zetta', 'users': 'default' @@ -179,14 +187,13 @@ class TestUGNormalize(MockerTestCase): } (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('zetta', users) - ug_cfg = { - } + ug_cfg = {} (users, groups) = self._norm(ug_cfg, distro) self.assertEquals({}, users) self.assertEquals({}, groups) def test_users_dict_default_additional(self): - distro = self._make_distro('ubuntu', 'bob') + distro = self._make_distro('ubuntu', bcfg) ug_cfg = { 'users': [ {'name': 'default', 'blah': True} @@ -198,9 +205,38 @@ class TestUGNormalize(MockerTestCase): users['bob']['groups']) self.assertEquals(True, users['bob']['blah']) + self.assertEquals(True, + users['bob']['default']) + + def test_users_dict_extract(self): + distro = self._make_distro('ubuntu', bcfg) + ug_cfg = { + 'users': [ + 'default', + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + self.assertIn('bob', users) + (name, config) = distros.extract_default(users) + self.assertEquals(name, 'bob') + expected_config = {} + def_config = None + try: + def_config = distro.get_default_user() + except NotImplementedError: + pass + if not def_config: + def_config = {} + expected_config.update(def_config) + + # Ignore these for now + expected_config.pop('name', None) + expected_config.pop('groups', None) + config.pop('groups', None) + self.assertEquals(config, expected_config) def test_users_dict_default(self): - distro = self._make_distro('ubuntu', 'bob') + distro = self._make_distro('ubuntu', bcfg) ug_cfg = { 'users': [ 'default', @@ -210,6 +246,8 @@ class TestUGNormalize(MockerTestCase): self.assertIn('bob', users) self.assertEquals(",".join(distro.get_default_user()['groups']), users['bob']['groups']) + self.assertEquals(True, + users['bob']['default']) def test_users_dict_trans(self): distro = self._make_distro('ubuntu') @@ -223,8 +261,8 @@ class TestUGNormalize(MockerTestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({'tr_me': True}, users['joe']) - self.assertEquals({}, users['bob']) + self.assertEquals({'tr_me': True, 'default': False}, users['joe']) + self.assertEquals({'default': False}, users['bob']) def test_users_dict(self): distro = self._make_distro('ubuntu') @@ -237,5 +275,5 @@ class TestUGNormalize(MockerTestCase): (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('joe', users) self.assertIn('bob', users) - self.assertEquals({}, users['joe']) - self.assertEquals({}, users['bob']) + self.assertEquals({'default': False}, users['joe']) + self.assertEquals({'default': False}, users['bob']) diff --git a/tools/make-dist-tarball b/tools/make-dist-tarball index 622283bd..7742caea 100755 --- a/tools/make-dist-tarball +++ b/tools/make-dist-tarball @@ -9,7 +9,7 @@ Usage: ${0##*/} version EOF } -topdir="../$PWD" +topdir="$PWD" tag=${1} [ -n "$tag" ] || { Usage 1>&2 ; exit 1; } |