From 91ccf1b55b5b79694449446b029dd7c4570517a5 Mon Sep 17 00:00:00 2001 From: Chris Cosby Date: Wed, 3 Dec 2014 01:13:52 -0500 Subject: Handle more possible ssh_pwauth values Update ssh_pwauth handler to accept all values mentioned in doc/examples/cloud-config.txt --- cloudinit/config/cc_set_passwords.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4ca85e21..fcfd3d1b 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -45,8 +45,6 @@ def handle(_name, cfg, cloud, log, args): password = util.get_cfg_option_str(cfg, "password", None) expire = True - pw_auth = "no" - change_pwauth = False plist = None if 'chpasswd' in cfg: @@ -104,11 +102,24 @@ def handle(_name, cfg, cloud, log, args): change_pwauth = False pw_auth = None if 'ssh_pwauth' in cfg: - change_pwauth = True if util.is_true(cfg['ssh_pwauth']): + change_pwauth = True pw_auth = 'yes' - if util.is_false(cfg['ssh_pwauth']): + elif util.is_false(cfg['ssh_pwauth']): + change_pwauth = True pw_auth = 'no' + elif str(cfg['ssh_pwauth']).lower() == 'unchanged': + log.debug('Leaving auth line unchanged') + change_pwauth = False + elif not str(cfg['ssh_pwauth']).strip(): + log.debug('Leaving auth line unchanged') + change_pwauth = False + elif not cfg['ssh_pwauth']: + log.debug('Leaving auth line unchanged') + change_pwauth = False + else: + util.logexc(log, 'Unrecognized value %r for ssh_pwauth' % cfg['ssh_pwauth']) + if change_pwauth: replaced_auth = False -- cgit v1.2.3 From 29ab69a1e550ba52c436a6c605aa691d1f20623c Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 13 Jan 2015 11:34:49 +0000 Subject: Fix lsblk output option. --- cloudinit/config/cc_disk_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 1660832b..6c5047a7 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -151,7 +151,7 @@ def enumerate_disk(device, nodeps=False): name: the device name, i.e. sda """ - lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL', + lsblk_cmd = [LSBLK_CMD, '--pairs', '--output', 'NAME,TYPE,FSTYPE,LABEL', device] if nodeps: -- cgit v1.2.3 From 1a56b32de0d2954c172e2de2c756e08471e47b6e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 13 Jan 2015 11:34:49 +0000 Subject: Implement check_partition_gpt_layout. This includes moving some shared logic in to check_partition_layout. --- cloudinit/config/cc_disk_setup.py | 42 ++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 6c5047a7..8334657e 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -393,6 +393,36 @@ def check_partition_mbr_layout(device, layout): break found_layout.append(type_label) + return found_layout + + +def check_partition_gpt_layout(device, layout): + prt_cmd = ['sgdisk', '-p', device] + try: + out, _err = util.subp(prt_cmd) + except Exception as e: + raise Exception("Error running partition command on %s\n%s" % ( + device, e)) + + out_lines = iter(out.splitlines()) + # Skip header + for line in out_lines: + if line.strip().startswith('Number'): + break + + return [line.strip().split()[-1] for line in out_lines] + + +def check_partition_layout(table_type, device, layout): + """ + See if the partition lay out matches. + + This is future a future proofing function. In order + to add support for other disk layout schemes, add a + function called check_partition_%s_layout + """ + found_layout = get_dyn_func( + "check_partition_%s_layout", table_type, device, layout) if isinstance(layout, bool): # if we are using auto partitioning, or "True" be happy @@ -417,18 +447,6 @@ def check_partition_mbr_layout(device, layout): return False -def check_partition_layout(table_type, device, layout): - """ - See if the partition lay out matches. - - This is future a future proofing function. In order - to add support for other disk layout schemes, add a - function called check_partition_%s_layout - """ - return get_dyn_func("check_partition_%s_layout", table_type, device, - layout) - - def get_partition_mbr_layout(size, layout): """ Calculate the layout of the partition table. Partition sizes -- cgit v1.2.3 From ed6219dbb5dc2fada4b9b86e7c9b94d2d35dcb7f Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 13 Jan 2015 11:34:49 +0000 Subject: Initial run at GPT disk handling. --- cloudinit/config/cc_disk_setup.py | 42 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 8334657e..dc607533 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -27,6 +27,7 @@ frequency = PER_INSTANCE # Define the commands to use UDEVADM_CMD = util.which('udevadm') SFDISK_CMD = util.which("sfdisk") +SGDISK_CMD = util.which("sgdisk") LSBLK_CMD = util.which("lsblk") BLKID_CMD = util.which("blkid") BLKDEV_CMD = util.which("blockdev") @@ -397,7 +398,7 @@ def check_partition_mbr_layout(device, layout): def check_partition_gpt_layout(device, layout): - prt_cmd = ['sgdisk', '-p', device] + prt_cmd = [SGDISK_CMD, '-p', device] try: out, _err = util.subp(prt_cmd) except Exception as e: @@ -499,6 +500,29 @@ def get_partition_mbr_layout(size, layout): return sfdisk_definition +def get_partition_gpt_layout(size, layout): + if isinstance(layout, bool): + return [(None, [0, 0])] + + partition_specs = [] + for partition in layout: + if isinstance(partition, list): + if len(partition) != 2: + raise Exception( + "Partition was incorrectly defined: %s" % partition) + percent, partition_type = partition + else: + percent = partition + partition_type = None + + part_size = int(float(size) * (float(percent) / 100)) + partition_specs.append((partition_type, [0, '+{}'.format(part_size)])) + + # The last partition should use up all remaining space + partition_specs[-1][-1][-1] = 0 + return partition_specs + + def purge_disk_ptable(device): # wipe the first and last megabyte of a disk (or file) # gpt stores partition table both at front and at end. @@ -574,6 +598,22 @@ def exec_mkpart_mbr(device, layout): read_parttbl(device) +def exec_mkpart_gpt(device, layout): + try: + util.subp([SGDISK_CMD, '-Z', device]) + for index, (partition_type, (start, end)) in enumerate(layout): + index += 1 + util.subp([SGDISK_CMD, + '-n', '{}:{}:{}'.format(index, start, end), device]) + if partition_type is not None: + util.subp( + [SGDISK_CMD, + '-t', '{}:{}'.format(index, partition_type), device]) + except Exception: + print "Failed to partition device %s" % (device,) + raise + + def exec_mkpart(table_type, device, layout): """ Fetches the function for creating the table type. -- cgit v1.2.3 From 54d1968f026cb0ee79913b599c2c90d9f07ef35d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 13 Jan 2015 11:34:49 +0000 Subject: Find disk size differently for GPT. MBR uses block sizes, which is what the current (apparently portable) code was producing. GPT uses sectors to determine partition size. --- cloudinit/config/cc_disk_setup.py | 42 +++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 17 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index dc607533..d8553167 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -316,22 +316,6 @@ def is_disk_used(device): return False -def get_hdd_size(device): - """ - Returns the hard disk size. - This works with any disk type, including GPT. - """ - - size_cmd = [SFDISK_CMD, '--show-size', device] - size = None - try: - size, _err = util.subp(size_cmd) - except Exception as e: - raise Exception("Failed to get %s size\n%s" % (device, e)) - - return int(size.strip()) - - def get_dyn_func(*args): """ Call the appropriate function. @@ -359,6 +343,30 @@ def get_dyn_func(*args): raise Exception("No such function %s to call!" % func_name) +def get_mbr_hdd_size(device): + size_cmd = [SFDISK_CMD, '--show-size', device] + size = None + try: + size, _err = util.subp(size_cmd) + except Exception as e: + raise Exception("Failed to get %s size\n%s" % (device, e)) + + return int(size.strip()) + + +def get_gpt_hdd_size(device): + out, _ = util.subp([SGDISK_CMD, '-p', device]) + return out.splitlines()[0].split()[2] + + +def get_hdd_size(table_type, device): + """ + Returns the hard disk size. + This works with any disk type, including GPT. + """ + return get_dyn_func("get_%s_hdd_size", table_type, device) + + def check_partition_mbr_layout(device, layout): """ Returns true if the partition layout matches the one on the disk @@ -676,7 +684,7 @@ def mkpart(device, definition): return LOG.debug("Checking for device size") - device_size = get_hdd_size(device) + device_size = get_hdd_size(table_type, device) LOG.debug("Calculating partition layout") part_definition = get_partition_layout(table_type, device_size, layout) -- cgit v1.2.3 From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 21 Jan 2015 17:56:53 -0500 Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because it seemed to be behind trunk. `tox -e py27` passes full test suite. Now to work on replacing mocker. --- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_debug.py | 7 +- cloudinit/config/cc_landscape.py | 2 +- cloudinit/config/cc_mcollective.py | 15 +-- cloudinit/config/cc_phone_home.py | 4 +- cloudinit/config/cc_puppet.py | 8 +- cloudinit/config/cc_resolv_conf.py | 4 +- cloudinit/config/cc_seed_random.py | 3 +- cloudinit/config/cc_ssh.py | 16 +-- cloudinit/config/cc_yum_add_repo.py | 7 +- cloudinit/distros/__init__.py | 55 ++++++----- cloudinit/distros/arch.py | 2 +- cloudinit/distros/freebsd.py | 12 ++- cloudinit/distros/net_util.py | 2 +- cloudinit/distros/parsers/hostname.py | 2 +- cloudinit/distros/parsers/hosts.py | 2 +- cloudinit/distros/parsers/resolv_conf.py | 2 +- cloudinit/distros/parsers/sys_conf.py | 5 +- cloudinit/distros/rhel.py | 2 +- cloudinit/distros/sles.py | 2 +- cloudinit/ec2_utils.py | 9 +- cloudinit/handlers/__init__.py | 2 +- cloudinit/handlers/boot_hook.py | 2 +- cloudinit/handlers/cloud_config.py | 2 +- cloudinit/handlers/shell_script.py | 2 +- cloudinit/handlers/upstart_job.py | 2 +- cloudinit/helpers.py | 13 +-- cloudinit/log.py | 7 +- cloudinit/mergers/__init__.py | 4 +- cloudinit/mergers/m_dict.py | 4 +- cloudinit/mergers/m_list.py | 6 +- cloudinit/mergers/m_str.py | 10 +- cloudinit/netinfo.py | 4 +- cloudinit/signal_handler.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 4 +- cloudinit/sources/DataSourceDigitalOcean.py | 9 +- cloudinit/sources/DataSourceEc2.py | 4 +- cloudinit/sources/DataSourceMAAS.py | 2 +- cloudinit/sources/DataSourceOVF.py | 6 +- cloudinit/sources/DataSourceSmartOS.py | 15 +-- cloudinit/sources/__init__.py | 10 +- cloudinit/sources/helpers/openstack.py | 10 +- cloudinit/ssh_util.py | 6 +- cloudinit/stages.py | 23 ++--- cloudinit/type_utils.py | 32 ++++-- cloudinit/url_helper.py | 22 +++-- cloudinit/user_data.py | 8 +- cloudinit/util.py | 109 +++++++++++++-------- packages/bddeb | 1 + packages/brpm | 2 + tests/unittests/test_data.py | 12 +-- tests/unittests/test_datasource/test_nocloud.py | 2 +- tests/unittests/test_datasource/test_openstack.py | 7 +- tests/unittests/test_distros/test_netconfig.py | 4 +- .../test_handler/test_handler_apt_configure.py | 10 +- .../unittests/test_handler/test_handler_locale.py | 6 +- .../test_handler/test_handler_seed_random.py | 2 +- .../test_handler/test_handler_set_hostname.py | 6 +- .../test_handler/test_handler_timezone.py | 6 +- .../test_handler/test_handler_yum_add_repo.py | 7 +- 60 files changed, 315 insertions(+), 233 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index f10b76a3..de72903f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -126,7 +126,7 @@ def mirror2lists_fileprefix(mirror): def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): - for (name, omirror) in old_mirrors.iteritems(): + for (name, omirror) in old_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: continue diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py index 8c489426..bdc32fe6 100644 --- a/cloudinit/config/cc_debug.py +++ b/cloudinit/config/cc_debug.py @@ -34,7 +34,8 @@ It can be configured with the following option structure:: """ import copy -from StringIO import StringIO + +from six import StringIO from cloudinit import type_utils from cloudinit import util @@ -77,7 +78,7 @@ def handle(name, cfg, cloud, log, args): dump_cfg = copy.deepcopy(cfg) for k in SKIP_KEYS: dump_cfg.pop(k, None) - all_keys = list(dump_cfg.keys()) + all_keys = list(dump_cfg) for k in all_keys: if k.startswith("_"): dump_cfg.pop(k, None) @@ -103,6 +104,6 @@ def handle(name, cfg, cloud, log, args): line = "ci-info: %s\n" % (line) content_to_file.append(line) if out_file: - util.write_file(out_file, "".join(content_to_file), 0644, "w") + util.write_file(out_file, "".join(content_to_file), 0o644, "w") else: util.multi_log("".join(content_to_file), console=True, stderr=False) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 8a709677..0b9d846e 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -20,7 +20,7 @@ import os -from StringIO import StringIO +from six import StringIO from configobj import ConfigObj diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index b670390d..425420ae 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -19,7 +19,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +import six +from six import StringIO # Used since this can maintain comments # and doesn't need a top level section @@ -51,17 +52,17 @@ def handle(name, cfg, cloud, log, _args): # original file in order to be able to mix the rest up mcollective_config = ConfigObj(SERVER_CFG) # See: http://tiny.cc/jh9agw - for (cfg_name, cfg) in mcollective_cfg['conf'].iteritems(): + for (cfg_name, cfg) in mcollective_cfg['conf'].items(): if cfg_name == 'public-cert': - util.write_file(PUBCERT_FILE, cfg, mode=0644) + util.write_file(PUBCERT_FILE, cfg, mode=0o644) mcollective_config['plugin.ssl_server_public'] = PUBCERT_FILE mcollective_config['securityprovider'] = 'ssl' elif cfg_name == 'private-cert': - util.write_file(PRICERT_FILE, cfg, mode=0600) + util.write_file(PRICERT_FILE, cfg, mode=0o600) mcollective_config['plugin.ssl_server_private'] = PRICERT_FILE mcollective_config['securityprovider'] = 'ssl' else: - if isinstance(cfg, (basestring, str)): + if isinstance(cfg, six.string_types): # Just set it in the 'main' section mcollective_config[cfg_name] = cfg elif isinstance(cfg, (dict)): @@ -69,7 +70,7 @@ def handle(name, cfg, cloud, log, _args): # if it is needed and then add/or create items as needed if cfg_name not in mcollective_config.sections: mcollective_config[cfg_name] = {} - for (o, v) in cfg.iteritems(): + for (o, v) in cfg.items(): mcollective_config[cfg_name][o] = v else: # Otherwise just try to convert it to a string @@ -81,7 +82,7 @@ def handle(name, cfg, cloud, log, _args): contents = StringIO() mcollective_config.write(contents) contents = contents.getvalue() - util.write_file(SERVER_CFG, contents, mode=0644) + util.write_file(SERVER_CFG, contents, mode=0o644) # Start mcollective util.subp(['service', 'mcollective', 'start'], capture=False) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 5bc68b83..18a7ddad 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -81,7 +81,7 @@ def handle(name, cfg, cloud, log, args): 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } - for (n, path) in pubkeys.iteritems(): + for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) except: @@ -99,7 +99,7 @@ def handle(name, cfg, cloud, log, args): # Get them read to be posted real_submit_keys = {} - for (k, v) in submit_keys.iteritems(): + for (k, v) in submit_keys.items(): if v is None: real_submit_keys[k] = 'N/A' else: diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 471a1a8a..6f1b3c57 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +from six import StringIO import os import socket @@ -81,13 +81,13 @@ def handle(name, cfg, cloud, log, _args): cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), filename=PUPPET_CONF_PATH) - for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): + for (cfg_name, cfg) in puppet_cfg['conf'].items(): # Cert configuration is a special case # Dump the puppet master ca certificate in the correct place if cfg_name == 'ca_cert': # Puppet ssl sub-directory isn't created yet # Create it with the proper permissions and ownership - util.ensure_dir(PUPPET_SSL_DIR, 0771) + util.ensure_dir(PUPPET_SSL_DIR, 0o771) util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') util.ensure_dir(PUPPET_SSL_CERT_DIR) util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') @@ -96,7 +96,7 @@ def handle(name, cfg, cloud, log, _args): else: # Iterate throug the config items, we'll use ConfigParser.set # to overwrite or create new items as needed - for (o, v) in cfg.iteritems(): + for (o, v) in cfg.items(): if o == 'certname': # Expand %f as the fqdn # TODO(harlowja) should this use the cloud fqdn?? diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index bbaa6c63..71d9e3a7 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -66,8 +66,8 @@ def generate_resolv_conf(template_fn, params, target_fname="/etc/resolv.conf"): false_flags = [] if 'options' in params: - for key, val in params['options'].iteritems(): - if type(val) == bool: + for key, val in params['options'].items(): + if isinstance(val, bool): if val: flags.append(key) else: diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 49a6b3e8..3b7235bf 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -21,7 +21,8 @@ import base64 import os -from StringIO import StringIO + +from six import StringIO from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 4c76581c..ab6940fa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -34,12 +34,12 @@ DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," "rather than the user \\\"root\\\".\';echo;sleep 10\"") KEY_2_FILE = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), + "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0o600), + "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0o644), + "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0o600), + "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0o644), + "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0o600), + "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0o644), } PRIV_2_PUB = { @@ -68,13 +68,13 @@ def handle(_name, cfg, cloud, log, _args): if "ssh_keys" in cfg: # if there are keys in cloud-config, use them - for (key, val) in cfg["ssh_keys"].iteritems(): + for (key, val) in cfg["ssh_keys"].items(): if key in KEY_2_FILE: tgt_fn = KEY_2_FILE[key][0] tgt_perms = KEY_2_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) - for (priv, pub) in PRIV_2_PUB.iteritems(): + for (priv, pub) in PRIV_2_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 0d836f28..3b821af9 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -18,9 +18,10 @@ import os -from cloudinit import util - import configobj +import six + +from cloudinit import util def _canonicalize_id(repo_id): @@ -37,7 +38,7 @@ def _format_repo_value(val): # Can handle 'lists' in certain cases # See: http://bit.ly/Qqrf1t return "\n ".join([_format_repo_value(v) for v in val]) - if not isinstance(val, (basestring, str)): + if not isinstance(val, six.string_types): return str(val) return val diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 49a0b652..4ebccdda 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -21,7 +21,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +import six +from six import StringIO import abc import itertools @@ -334,7 +335,7 @@ class Distro(object): redact_opts = ['passwd'] # Check the values and create the command - for key, val in kwargs.iteritems(): + for key, val in kwargs.items(): if key in adduser_opts and val and isinstance(val, str): adduser_cmd.extend([adduser_opts[key], val]) @@ -393,7 +394,7 @@ class Distro(object): if 'ssh_authorized_keys' in kwargs: # Try to handle this in a smart manner. keys = kwargs['ssh_authorized_keys'] - if isinstance(keys, (basestring, str)): + if isinstance(keys, six.string_types): keys = [keys] if isinstance(keys, dict): keys = list(keys.values()) @@ -491,7 +492,7 @@ class Distro(object): if isinstance(rules, (list, tuple)): for rule in rules: lines.append("%s %s" % (user, rule)) - elif isinstance(rules, (basestring, str)): + elif isinstance(rules, six.string_types): lines.append("%s %s" % (user, rules)) else: msg = "Can not create sudoers rule addition with type %r" @@ -561,10 +562,10 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, subst['ec2_region'] = "%s" % availability_zone[0:-1] results = {} - for (name, mirror) in mirror_info.get('failsafe', {}).iteritems(): + for (name, mirror) in mirror_info.get('failsafe', {}).items(): results[name] = mirror - for (name, searchlist) in mirror_info.get('search', {}).iteritems(): + for (name, searchlist) in mirror_info.get('search', {}).items(): mirrors = [] for tmpl in searchlist: try: @@ -604,30 +605,30 @@ def _get_arch_package_mirror_info(package_mirrors, arch): # is the standard form used in the rest # of cloud-init def _normalize_groups(grp_cfg): - if isinstance(grp_cfg, (str, basestring)): + if isinstance(grp_cfg, six.string_types): grp_cfg = grp_cfg.strip().split(",") - if isinstance(grp_cfg, (list)): + if isinstance(grp_cfg, list): c_grp_cfg = {} for i in grp_cfg: - if isinstance(i, (dict)): + if isinstance(i, dict): for k, v in i.items(): if k not in c_grp_cfg: - if isinstance(v, (list)): + if isinstance(v, list): c_grp_cfg[k] = list(v) - elif isinstance(v, (basestring, str)): + elif isinstance(v, six.string_types): c_grp_cfg[k] = [v] else: raise TypeError("Bad group member type %s" % type_utils.obj_name(v)) else: - if isinstance(v, (list)): + if isinstance(v, list): c_grp_cfg[k].extend(v) - elif isinstance(v, (basestring, str)): + elif isinstance(v, six.string_types): c_grp_cfg[k].append(v) else: raise TypeError("Bad group member type %s" % type_utils.obj_name(v)) - elif isinstance(i, (str, basestring)): + elif isinstance(i, six.string_types): if i not in c_grp_cfg: c_grp_cfg[i] = [] else: @@ -635,7 +636,7 @@ def _normalize_groups(grp_cfg): type_utils.obj_name(i)) grp_cfg = c_grp_cfg groups = {} - if isinstance(grp_cfg, (dict)): + if isinstance(grp_cfg, dict): for (grp_name, grp_members) in grp_cfg.items(): groups[grp_name] = util.uniq_merge_sorted(grp_members) else: @@ -661,29 +662,29 @@ def _normalize_groups(grp_cfg): # entry 'default' which will be marked as true # all other users will be marked as false. def _normalize_users(u_cfg, def_user_cfg=None): - if isinstance(u_cfg, (dict)): + if isinstance(u_cfg, dict): ad_ucfg = [] for (k, v) in u_cfg.items(): - if isinstance(v, (bool, int, basestring, str, float)): + if isinstance(v, (bool, int, float) + six.string_types): if util.is_true(v): ad_ucfg.append(str(k)) - elif isinstance(v, (dict)): + elif isinstance(v, dict): v['name'] = k ad_ucfg.append(v) else: raise TypeError(("Unmappable user value type %s" " for key %s") % (type_utils.obj_name(v), k)) u_cfg = ad_ucfg - elif isinstance(u_cfg, (str, basestring)): + elif isinstance(u_cfg, six.string_types): u_cfg = util.uniq_merge_sorted(u_cfg) users = {} for user_config in u_cfg: - if isinstance(user_config, (str, basestring, list)): + if isinstance(user_config, (list,) + six.string_types): for u in util.uniq_merge(user_config): if u and u not in users: users[u] = {} - elif isinstance(user_config, (dict)): + elif isinstance(user_config, dict): if 'name' in user_config: n = user_config.pop('name') prev_config = users.get(n) or {} @@ -784,11 +785,11 @@ def normalize_users_groups(cfg, distro): old_user = cfg['user'] # Translate it into the format that is more useful # going forward - if isinstance(old_user, (basestring, str)): + if isinstance(old_user, six.string_types): old_user = { 'name': old_user, } - if not isinstance(old_user, (dict)): + if not isinstance(old_user, dict): LOG.warn(("Format for 'user' key must be a string or " "dictionary and not %s"), type_utils.obj_name(old_user)) old_user = {} @@ -813,7 +814,7 @@ def normalize_users_groups(cfg, distro): default_user_config = util.mergemanydict([old_user, distro_user_config]) base_users = cfg.get('users', []) - if not isinstance(base_users, (list, dict, str, basestring)): + if not isinstance(base_users, (list, dict) + six.string_types): LOG.warn(("Format for 'users' key must be a comma separated string" " or a dictionary or a list and not %s"), type_utils.obj_name(base_users)) @@ -822,12 +823,12 @@ def normalize_users_groups(cfg, distro): if old_user: # Ensure that when user: is provided that this user # always gets added (as the default user) - if isinstance(base_users, (list)): + if isinstance(base_users, list): # Just add it on at the end... base_users.append({'name': 'default'}) - elif isinstance(base_users, (dict)): + elif isinstance(base_users, dict): base_users['default'] = dict(base_users).get('default', True) - elif isinstance(base_users, (str, basestring)): + elif isinstance(base_users, six.string_types): # Just append it on to be re-parsed later base_users += ",default" diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 68bf1aab..e540e0bc 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -66,7 +66,7 @@ class Distro(distros.Distro): settings, entries) dev_names = entries.keys() # Format for netctl - for (dev, info) in entries.iteritems(): + for (dev, info) in entries.items(): nameservers = [] net_fn = self.network_conf_dir + dev net_cfg = { diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index f1b4a256..4c484639 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -16,7 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +import six +from six import StringIO import re @@ -203,8 +204,9 @@ class Distro(distros.Distro): redact_opts = ['passwd'] - for key, val in kwargs.iteritems(): - if key in adduser_opts and val and isinstance(val, basestring): + for key, val in kwargs.items(): + if (key in adduser_opts and val + and isinstance(val, six.string_types)): adduser_cmd.extend([adduser_opts[key], val]) # Redact certain fields from the logs @@ -271,7 +273,7 @@ class Distro(distros.Distro): nameservers = [] searchdomains = [] dev_names = entries.keys() - for (device, info) in entries.iteritems(): + for (device, info) in entries.items(): # Skip the loopback interface. if device.startswith('lo'): continue @@ -323,7 +325,7 @@ class Distro(distros.Distro): resolvconf.add_search_domain(domain) except ValueError: util.logexc(LOG, "Failed to add search domain %s", domain) - util.write_file(self.resolv_conf_fn, str(resolvconf), 0644) + util.write_file(self.resolv_conf_fn, str(resolvconf), 0o644) return dev_names diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py index 8b28e2d1..cadfa6b6 100644 --- a/cloudinit/distros/net_util.py +++ b/cloudinit/distros/net_util.py @@ -103,7 +103,7 @@ def translate_network(settings): consume[cmd] = args # Check if anything left over to consume absorb = False - for (cmd, args) in consume.iteritems(): + for (cmd, args) in consume.items(): if cmd == 'iface': absorb = True if absorb: diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py index 617b3c36..84a1de42 100644 --- a/cloudinit/distros/parsers/hostname.py +++ b/cloudinit/distros/parsers/hostname.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +from six import StringIO from cloudinit.distros.parsers import chop_comment diff --git a/cloudinit/distros/parsers/hosts.py b/cloudinit/distros/parsers/hosts.py index 94c97051..3c5498ee 100644 --- a/cloudinit/distros/parsers/hosts.py +++ b/cloudinit/distros/parsers/hosts.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +from six import StringIO from cloudinit.distros.parsers import chop_comment diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 5733c25a..8aee03a4 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +from six import StringIO from cloudinit import util diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index 20ca1871..d795e12f 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -16,7 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO +import six +from six import StringIO import pipes import re @@ -69,7 +70,7 @@ class SysConf(configobj.ConfigObj): return out_contents.getvalue() def _quote(self, value, multiline=False): - if not isinstance(value, (str, basestring)): + if not isinstance(value, six.string_types): raise ValueError('Value "%s" is not a string' % (value)) if len(value) == 0: return '' diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index d9588632..7408989c 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -73,7 +73,7 @@ class Distro(distros.Distro): searchservers = [] dev_names = entries.keys() use_ipv6 = False - for (dev, info) in entries.iteritems(): + for (dev, info) in entries.items(): net_fn = self.network_script_tpl % (dev) net_cfg = { 'DEVICE': dev, diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index 43682a12..0c6d1203 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -62,7 +62,7 @@ class Distro(distros.Distro): nameservers = [] searchservers = [] dev_names = entries.keys() - for (dev, info) in entries.iteritems(): + for (dev, info) in entries.items(): net_fn = self.network_script_tpl % (dev) mode = info.get('auto') if mode and mode.lower() == 'true': diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index e69d06ff..e1ed4091 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -17,7 +17,6 @@ # along with this program. If not, see . import functools -import httplib import json from cloudinit import log as logging @@ -25,7 +24,7 @@ from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) -SKIP_USERDATA_CODES = frozenset([httplib.NOT_FOUND]) +SKIP_USERDATA_CODES = frozenset([url_helper.NOT_FOUND]) class MetadataLeafDecoder(object): @@ -123,7 +122,7 @@ class MetadataMaterializer(object): leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = url_helper.combine_url(base_url, resource) - leaf_blob = str(self._caller(leaf_url)) + leaf_blob = self._caller(leaf_url).contents leaf_contents[field] = self._leaf_decoder(field, leaf_blob) joined = {} joined.update(child_contents) @@ -160,7 +159,7 @@ def get_instance_userdata(api_version='latest', timeout=timeout, retries=retries, exception_cb=exception_cb) - user_data = str(response) + user_data = response.contents except url_helper.UrlError as e: if e.code not in SKIP_USERDATA_CODES: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) @@ -183,7 +182,7 @@ def get_instance_metadata(api_version='latest', try: response = caller(md_url) - materializer = MetadataMaterializer(str(response), + materializer = MetadataMaterializer(response.contents, md_url, caller, leaf_decoder=leaf_decoder) md = materializer.materialize() diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 059d7495..d67a70ea 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -147,7 +147,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): if not modfname.endswith(".py"): modfname = "%s.py" % (modfname) # TODO(harlowja): Check if path exists?? - util.write_file(modfname, payload, 0600) + util.write_file(modfname, payload, 0o600) handlers = pdata['handlers'] try: mod = fixup_handler(importer.import_module(modname)) diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 3a50cf87..a4ea47ac 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -50,7 +50,7 @@ class BootHookPartHandler(handlers.Handler): filepath = os.path.join(self.boothook_dir, filename) contents = util.strip_prefix_suffix(util.dos2unix(payload), prefix=BOOTHOOK_PREFIX) - util.write_file(filepath, contents.lstrip(), 0700) + util.write_file(filepath, contents.lstrip(), 0o700) return filepath def handle_part(self, data, ctype, filename, payload, frequency): diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index bf994e33..07b6d0e0 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -95,7 +95,7 @@ class CloudConfigPartHandler(handlers.Handler): lines.append(util.yaml_dumps(self.cloud_buf)) else: lines = [] - util.write_file(self.cloud_fn, "\n".join(lines), 0600) + util.write_file(self.cloud_fn, "\n".join(lines), 0o600) def _extract_mergers(self, payload, headers): merge_header_headers = '' diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 9755ab05..b5087693 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -52,4 +52,4 @@ class ShellScriptPartHandler(handlers.Handler): filename = util.clean_filename(filename) payload = util.dos2unix(payload) path = os.path.join(self.script_dir, filename) - util.write_file(path, payload, 0700) + util.write_file(path, payload, 0o700) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 50d193c4..c5bea711 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -65,7 +65,7 @@ class UpstartJobPartHandler(handlers.Handler): payload = util.dos2unix(payload) path = os.path.join(self.upstart_dir, filename) - util.write_file(path, payload, 0644) + util.write_file(path, payload, 0o644) if SUITABLE_UPSTART: util.subp(["initctl", "reload-configuration"], capture=False) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index e701126e..ed396b5a 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -23,10 +23,11 @@ from time import time import contextlib -import io import os -from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser) +import six +from six.moves.configparser import ( + NoSectionError, NoOptionError, RawConfigParser) from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, CFG_ENV_NAME) @@ -318,10 +319,10 @@ class ContentHandlers(object): return self.registered[content_type] def items(self): - return self.registered.items() + return list(self.registered.items()) - def iteritems(self): - return self.registered.iteritems() + # XXX This should really go away. + iteritems = items class Paths(object): @@ -449,7 +450,7 @@ class DefaultingConfigParser(RawConfigParser): def stringify(self, header=None): contents = '' - with io.BytesIO() as outputstream: + with six.StringIO() as outputstream: self.write(outputstream) outputstream.flush() contents = outputstream.getvalue() diff --git a/cloudinit/log.py b/cloudinit/log.py index 622c946c..3c79b9c9 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -28,7 +28,8 @@ import collections import os import sys -from StringIO import StringIO +import six +from six import StringIO # Logging levels for easy access CRITICAL = logging.CRITICAL @@ -72,13 +73,13 @@ def setupLogging(cfg=None): log_cfgs = [] log_cfg = cfg.get('logcfg') - if log_cfg and isinstance(log_cfg, (str, basestring)): + if log_cfg and isinstance(log_cfg, six.string_types): # If there is a 'logcfg' entry in the config, # respect it, it is the old keyname log_cfgs.append(str(log_cfg)) elif "log_cfgs" in cfg: for a_cfg in cfg['log_cfgs']: - if isinstance(a_cfg, (basestring, str)): + if isinstance(a_cfg, six.string_types): log_cfgs.append(a_cfg) elif isinstance(a_cfg, (collections.Iterable)): cfg_str = [str(c) for c in a_cfg] diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 03aa1ee1..e13f55ac 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -18,6 +18,8 @@ import re +import six + from cloudinit import importer from cloudinit import log as logging from cloudinit import type_utils @@ -95,7 +97,7 @@ def dict_extract_mergers(config): raw_mergers = config.pop('merge_type', None) if raw_mergers is None: return parsed_mergers - if isinstance(raw_mergers, (str, basestring)): + if isinstance(raw_mergers, six.string_types): return string_extract_mergers(raw_mergers) for m in raw_mergers: if isinstance(m, (dict)): diff --git a/cloudinit/mergers/m_dict.py b/cloudinit/mergers/m_dict.py index a16141fa..87cf1a72 100644 --- a/cloudinit/mergers/m_dict.py +++ b/cloudinit/mergers/m_dict.py @@ -16,6 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import six + DEF_MERGE_TYPE = 'no_replace' MERGE_TYPES = ('replace', DEF_MERGE_TYPE,) @@ -57,7 +59,7 @@ class Merger(object): return new_v if isinstance(new_v, (list, tuple)) and self._recurse_array: return self._merger.merge(old_v, new_v) - if isinstance(new_v, (basestring)) and self._recurse_str: + if isinstance(new_v, six.string_types) and self._recurse_str: return self._merger.merge(old_v, new_v) if isinstance(new_v, (dict)) and self._recurse_dict: return self._merger.merge(old_v, new_v) diff --git a/cloudinit/mergers/m_list.py b/cloudinit/mergers/m_list.py index 3b87b0fc..81e5c580 100644 --- a/cloudinit/mergers/m_list.py +++ b/cloudinit/mergers/m_list.py @@ -16,6 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import six + DEF_MERGE_TYPE = 'replace' MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE, 'no_replace') @@ -73,7 +75,7 @@ class Merger(object): return old_v if isinstance(new_v, (list, tuple)) and self._recurse_array: return self._merger.merge(old_v, new_v) - if isinstance(new_v, (str, basestring)) and self._recurse_str: + if isinstance(new_v, six.string_types) and self._recurse_str: return self._merger.merge(old_v, new_v) if isinstance(new_v, (dict)) and self._recurse_dict: return self._merger.merge(old_v, new_v) @@ -82,6 +84,6 @@ class Merger(object): # Ok now we are replacing same indexes merged_list.extend(value) common_len = min(len(merged_list), len(merge_with)) - for i in xrange(0, common_len): + for i in range(0, common_len): merged_list[i] = merge_same_index(merged_list[i], merge_with[i]) return merged_list diff --git a/cloudinit/mergers/m_str.py b/cloudinit/mergers/m_str.py index e22ce28a..b00c4bf3 100644 --- a/cloudinit/mergers/m_str.py +++ b/cloudinit/mergers/m_str.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import six + class Merger(object): def __init__(self, _merger, opts): @@ -34,11 +36,11 @@ class Merger(object): # perform the following action, if appending we will # merge them together, otherwise we will just return value. def _on_str(self, value, merge_with): - if not isinstance(value, (basestring)): + if not isinstance(value, six.string_types): return merge_with if not self._append: return merge_with - if isinstance(value, unicode): - return value + unicode(merge_with) + if isinstance(value, six.text_type): + return value + six.text_type(merge_with) else: - return value + str(merge_with) + return value + six.binary_type(merge_with) diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index fb40cc0d..e30d6fb5 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -87,7 +87,7 @@ def netdev_info(empty=""): devs[curdev][target] = toks[i][len(field) + 1:] if empty != "": - for (_devname, dev) in devs.iteritems(): + for (_devname, dev) in devs.items(): for field in dev: if dev[field] == "": dev[field] = empty @@ -181,7 +181,7 @@ def netdev_pformat(): else: fields = ['Device', 'Up', 'Address', 'Mask', 'Scope', 'Hw-Address'] tbl = PrettyTable(fields) - for (dev, d) in netdev.iteritems(): + for (dev, d) in netdev.items(): tbl.add_row([dev, d["up"], d["addr"], d["mask"], ".", d["hwaddr"]]) if d.get('addr6'): tbl.add_row([dev, d["up"], diff --git a/cloudinit/signal_handler.py b/cloudinit/signal_handler.py index 40b0c94c..0d95f506 100644 --- a/cloudinit/signal_handler.py +++ b/cloudinit/signal_handler.py @@ -22,7 +22,7 @@ import inspect import signal import sys -from StringIO import StringIO +from six import StringIO from cloudinit import log as logging from cloudinit import util diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 15244a0d..eb474079 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -216,11 +216,11 @@ def on_first_boot(data, distro=None): files = data.get('files', {}) if files: LOG.debug("Writing %s injected files", len(files)) - for (filename, content) in files.iteritems(): + for (filename, content) in files.items(): if not filename.startswith(os.sep): filename = os.sep + filename try: - util.write_file(filename, content, mode=0660) + util.write_file(filename, content, mode=0o660) except IOError: util.logexc(LOG, "Failed writing file: %s", filename) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 8f27ee89..b20ce2a1 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -18,7 +18,7 @@ from cloudinit import log as logging from cloudinit import util from cloudinit import sources from cloudinit import ec2_utils -from types import StringType + import functools @@ -72,10 +72,11 @@ class DataSourceDigitalOcean(sources.DataSource): return "\n".join(self.metadata['vendor-data']) def get_public_ssh_keys(self): - if type(self.metadata['public-keys']) is StringType: - return [self.metadata['public-keys']] + public_keys = self.metadata['public-keys'] + if isinstance(public_keys, list): + return public_keys else: - return self.metadata['public-keys'] + return [public_keys] @property def availability_zone(self): diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 1b20ecf3..798869b7 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -156,8 +156,8 @@ class DataSourceEc2(sources.DataSource): # 'ephemeral0': '/dev/sdb', # 'root': '/dev/sda1'} found = None - bdm_items = self.metadata['block-device-mapping'].iteritems() - for (entname, device) in bdm_items: + bdm = self.metadata['block-device-mapping'] + for (entname, device) in bdm.items(): if entname == name: found = device break diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index dfe90bc6..9a3e30c5 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -262,7 +262,7 @@ def check_seed_contents(content, seed): userdata = content.get('user-data', "") md = {} - for (key, val) in content.iteritems(): + for (key, val) in content.items(): if key == 'user-data': continue md[key] = val diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 7ba60735..58a4b2a2 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -66,7 +66,7 @@ class DataSourceOVF(sources.DataSource): np = {'iso': transport_iso9660, 'vmware-guestd': transport_vmware_guestd, } name = None - for (name, transfunc) in np.iteritems(): + for (name, transfunc) in np.items(): (contents, _dev, _fname) = transfunc() if contents: break @@ -138,7 +138,7 @@ def read_ovf_environment(contents): ud = "" cfg_props = ['password'] md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id'] - for (prop, val) in props.iteritems(): + for (prop, val) in props.items(): if prop == 'hostname': prop = "local-hostname" if prop in md_props: @@ -183,7 +183,7 @@ def transport_iso9660(require_iso=True): # Go through mounts to see if it was already mounted mounts = util.mounts() - for (dev, info) in mounts.iteritems(): + for (dev, info) in mounts.items(): fstype = info['fstype'] if fstype != "iso9660" and require_iso: continue diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 2733a2f6..7a975d78 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -30,12 +30,12 @@ # Comments with "@datadictionary" are snippets of the definition import base64 +import os +import serial + from cloudinit import log as logging from cloudinit import sources from cloudinit import util -import os -import os.path -import serial LOG = logging.getLogger(__name__) @@ -201,7 +201,7 @@ class DataSourceSmartOS(sources.DataSource): if b64_all is not None: self.b64_all = util.is_true(b64_all) - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): smartos_noun, strip = attribute md[ci_noun] = self.query(smartos_noun, strip=strip) @@ -218,11 +218,12 @@ class DataSourceSmartOS(sources.DataSource): user_script = os.path.join(data_d, 'user-script') u_script_l = "%s/user-script" % LEGACY_USER_D write_boot_content(md.get('user-script'), content_f=user_script, - link=u_script_l, shebang=True, mode=0700) + link=u_script_l, shebang=True, mode=0o700) operator_script = os.path.join(data_d, 'operator-script') write_boot_content(md.get('operator-script'), - content_f=operator_script, shebang=False, mode=0700) + content_f=operator_script, shebang=False, + mode=0o700) # @datadictionary: This key has no defined format, but its value # is written to the file /var/db/mdata-user-data on each boot prior @@ -381,7 +382,7 @@ def dmi_data(): def write_boot_content(content, content_f, link=None, shebang=False, - mode=0400): + mode=0o400): """ Write the content to content_f. Under the following rules: 1. If no content, remove the file diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7c7ef9ab..39eab51b 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -23,6 +23,8 @@ import abc import os +import six + from cloudinit import importer from cloudinit import log as logging from cloudinit import type_utils @@ -130,7 +132,7 @@ class DataSource(object): # we want to return the correct value for what will actually # exist in this instance mappings = {"sd": ("vd", "xvd", "vtb")} - for (nfrom, tlist) in mappings.iteritems(): + for (nfrom, tlist) in mappings.items(): if not short_name.startswith(nfrom): continue for nto in tlist: @@ -218,18 +220,18 @@ def normalize_pubkey_data(pubkey_data): if not pubkey_data: return keys - if isinstance(pubkey_data, (basestring, str)): + if isinstance(pubkey_data, six.string_types): return str(pubkey_data).splitlines() if isinstance(pubkey_data, (list, set)): return list(pubkey_data) if isinstance(pubkey_data, (dict)): - for (_keyname, klist) in pubkey_data.iteritems(): + for (_keyname, klist) in pubkey_data.items(): # lp:506332 uec metadata service responds with # data that makes boto populate a string for 'klist' rather # than a list. - if isinstance(klist, (str, basestring)): + if isinstance(klist, six.string_types): klist = [klist] if isinstance(klist, (list, set)): for pkey in klist: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b7e19314..88c7a198 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -24,6 +24,8 @@ import copy import functools import os +import six + from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources @@ -205,7 +207,7 @@ class BaseReader(object): """ load_json_anytype = functools.partial( - util.load_json, root_types=(dict, basestring, list)) + util.load_json, root_types=(dict, list) + six.string_types) def datafiles(version): files = {} @@ -234,7 +236,7 @@ class BaseReader(object): 'version': 2, } data = datafiles(self._find_working_version()) - for (name, (path, required, translator)) in data.iteritems(): + for (name, (path, required, translator)) in data.items(): path = self._path_join(self.base_path, path) data = None found = False @@ -364,7 +366,7 @@ class ConfigDriveReader(BaseReader): raise NonReadable("%s: no files found" % (self.base_path)) md = {} - for (name, (key, translator, default)) in FILES_V1.iteritems(): + for (name, (key, translator, default)) in FILES_V1.items(): if name in found: path = found[name] try: @@ -478,7 +480,7 @@ def convert_vendordata_json(data, recurse=True): """ if not data: return None - if isinstance(data, (str, unicode, basestring)): + if isinstance(data, six.string_types): return data if isinstance(data, list): return copy.deepcopy(data) diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 14d0cb0f..9b2f5ed5 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -239,7 +239,7 @@ def setup_user_keys(keys, username, options=None): # Make sure the users .ssh dir is setup accordingly (ssh_dir, pwent) = users_ssh_info(username) if not os.path.isdir(ssh_dir): - util.ensure_dir(ssh_dir, mode=0700) + util.ensure_dir(ssh_dir, mode=0o700) util.chownbyid(ssh_dir, pwent.pw_uid, pwent.pw_gid) # Turn the 'update' keys given into actual entries @@ -252,8 +252,8 @@ def setup_user_keys(keys, username, options=None): (auth_key_fn, auth_key_entries) = extract_authorized_keys(username) with util.SeLinuxGuard(ssh_dir, recursive=True): content = update_authorized_keys(auth_key_entries, key_entries) - util.ensure_dir(os.path.dirname(auth_key_fn), mode=0700) - util.write_file(auth_key_fn, content, mode=0600) + util.ensure_dir(os.path.dirname(auth_key_fn), mode=0o700) + util.write_file(auth_key_fn, content, mode=0o600) util.chownbyid(auth_key_fn, pwent.pw_uid, pwent.pw_gid) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 67f467f7..f4f4591d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -20,12 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cPickle as pickle - import copy import os import sys +import six +from six.moves import cPickle as pickle + from cloudinit.settings import (PER_INSTANCE, FREQUENCIES, CLOUD_CONFIG) from cloudinit import handlers @@ -202,7 +203,7 @@ class Init(object): util.logexc(LOG, "Failed pickling datasource %s", self.datasource) return False try: - util.write_file(pickled_fn, pk_contents, mode=0400) + util.write_file(pickled_fn, pk_contents, mode=0o400) except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False @@ -324,15 +325,15 @@ class Init(object): def _store_userdata(self): raw_ud = "%s" % (self.datasource.get_userdata_raw()) - util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0600) + util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0o600) processed_ud = "%s" % (self.datasource.get_userdata()) - util.write_file(self._get_ipath('userdata'), processed_ud, 0600) + util.write_file(self._get_ipath('userdata'), processed_ud, 0o600) def _store_vendordata(self): raw_vd = "%s" % (self.datasource.get_vendordata_raw()) - util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0600) + util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0o600) processed_vd = "%s" % (self.datasource.get_vendordata()) - util.write_file(self._get_ipath('vendordata'), processed_vd, 0600) + util.write_file(self._get_ipath('vendordata'), processed_vd, 0o600) def _default_handlers(self, opts=None): if opts is None: @@ -384,7 +385,7 @@ class Init(object): if not path or not os.path.isdir(path): return potential_handlers = util.find_modules(path) - for (fname, mod_name) in potential_handlers.iteritems(): + for (fname, mod_name) in potential_handlers.items(): try: mod_locs, looked_locs = importer.find_module( mod_name, [''], ['list_types', 'handle_part']) @@ -422,7 +423,7 @@ class Init(object): def init_handlers(): # Init the handlers first - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in c_handlers.initialized: # Avoid initing the same module twice (if said module # is registered to more than one content-type). @@ -449,7 +450,7 @@ class Init(object): def finalize_handlers(): # Give callbacks opportunity to finalize - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod not in c_handlers.initialized: # Said module was never inited in the first place, so lets # not attempt to finalize those that never got called. @@ -574,7 +575,7 @@ class Modules(object): for item in cfg_mods: if not item: continue - if isinstance(item, (str, basestring)): + if isinstance(item, six.string_types): module_list.append({ 'mod': item.strip(), }) diff --git a/cloudinit/type_utils.py b/cloudinit/type_utils.py index cc3d9495..b93efd6a 100644 --- a/cloudinit/type_utils.py +++ b/cloudinit/type_utils.py @@ -22,11 +22,31 @@ import types +import six + + +if six.PY3: + _NAME_TYPES = ( + types.ModuleType, + types.FunctionType, + types.LambdaType, + type, + ) +else: + _NAME_TYPES = ( + types.TypeType, + types.ModuleType, + types.FunctionType, + types.LambdaType, + types.ClassType, + ) + def obj_name(obj): - if isinstance(obj, (types.TypeType, - types.ModuleType, - types.FunctionType, - types.LambdaType)): - return str(obj.__name__) - return obj_name(obj.__class__) + if isinstance(obj, _NAME_TYPES): + return six.text_type(obj.__name__) + else: + if not hasattr(obj, '__class__'): + return repr(obj) + else: + return obj_name(obj.__class__) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 3074dd08..62001dff 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -20,21 +20,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httplib import time -import urllib + +import six import requests from requests import exceptions -from urlparse import (urlparse, urlunparse) +from six.moves.urllib.parse import ( + urlparse, urlunparse, + quote as urlquote) from cloudinit import log as logging from cloudinit import version LOG = logging.getLogger(__name__) -NOT_FOUND = httplib.NOT_FOUND +if six.PY2: + import httplib + NOT_FOUND = httplib.NOT_FOUND +else: + import http.client + NOT_FOUND = http.client.NOT_FOUND + # Check if requests has ssl support (added in requests >= 0.8.8) SSL_ENABLED = False @@ -70,7 +78,7 @@ def combine_url(base, *add_ons): path = url_parsed[2] if path and not path.endswith("/"): path += "/" - path += urllib.quote(str(add_on), safe="/:") + path += urlquote(str(add_on), safe="/:") url_parsed[2] = path return urlunparse(url_parsed) @@ -111,7 +119,7 @@ class UrlResponse(object): @property def contents(self): - return self._response.content + return self._response.text @property def url(self): @@ -135,7 +143,7 @@ class UrlResponse(object): return self._response.status_code def __str__(self): - return self.contents + return self._response.text class UrlError(IOError): diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index de6487d8..9111bd39 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -29,6 +29,8 @@ from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText +import six + from cloudinit import handlers from cloudinit import log as logging from cloudinit import util @@ -235,7 +237,7 @@ class UserDataProcessor(object): resp = util.read_file_or_url(include_url, ssl_details=self.ssl_details) if include_once_on and resp.ok(): - util.write_file(include_once_fn, str(resp), mode=0600) + util.write_file(include_once_fn, str(resp), mode=0o600) if resp.ok(): content = str(resp) else: @@ -256,7 +258,7 @@ class UserDataProcessor(object): # filename and type not be present # or # scalar(payload) - if isinstance(ent, (str, basestring)): + if isinstance(ent, six.string_types): ent = {'content': ent} if not isinstance(ent, (dict)): # TODO(harlowja) raise? @@ -337,7 +339,7 @@ def convert_string(raw_data, headers=None): data = util.decomp_gzip(raw_data) if "mime-version:" in data[0:4096].lower(): msg = email.message_from_string(data) - for (key, val) in headers.iteritems(): + for (key, val) in headers.items(): _replace_header(msg, key, val) else: mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE) diff --git a/cloudinit/util.py b/cloudinit/util.py index 9efc704a..434ba7fb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -20,8 +20,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO - import contextlib import copy as obj_copy import ctypes @@ -45,8 +43,10 @@ import subprocess import sys import tempfile import time -import urlparse +from six.moves.urllib import parse as urlparse + +import six import yaml from cloudinit import importer @@ -69,8 +69,26 @@ FN_REPLACEMENTS = { } FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) +TRUE_STRINGS = ('true', '1', 'on', 'yes') +FALSE_STRINGS = ('off', '0', 'no', 'false') + + # Helper utils to see if running in a container -CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] +CONTAINER_TESTS = ('running-in-container', 'lxc-is-container') + + +def decode_binary(blob, encoding='utf-8'): + # Converts a binary type into a text type using given encoding. + if isinstance(blob, six.text_type): + return blob + return blob.decode(encoding) + + +def encode_text(text, encoding='utf-8'): + # Converts a text string into a binary type using given encoding. + if isinstance(text, six.binary_type): + return text + return text.encode(encoding) class ProcessExecutionError(IOError): @@ -95,7 +113,7 @@ class ProcessExecutionError(IOError): else: self.description = description - if not isinstance(exit_code, (long, int)): + if not isinstance(exit_code, six.integer_types): self.exit_code = '-' else: self.exit_code = exit_code @@ -151,7 +169,8 @@ class SeLinuxGuard(object): path = os.path.realpath(self.path) # path should be a string, not unicode - path = str(path) + if six.PY2: + path = str(path) try: stats = os.lstat(path) self.selinux.matchpathcon(path, stats[stat.ST_MODE]) @@ -209,10 +228,10 @@ def fork_cb(child_cb, *args, **kwargs): def is_true(val, addons=None): if isinstance(val, (bool)): return val is True - check_set = ['true', '1', 'on', 'yes'] + check_set = TRUE_STRINGS if addons: - check_set = check_set + addons - if str(val).lower().strip() in check_set: + check_set = list(check_set) + addons + if six.text_type(val).lower().strip() in check_set: return True return False @@ -220,10 +239,10 @@ def is_true(val, addons=None): def is_false(val, addons=None): if isinstance(val, (bool)): return val is False - check_set = ['off', '0', 'no', 'false'] + check_set = FALSE_STRINGS if addons: - check_set = check_set + addons - if str(val).lower().strip() in check_set: + check_set = list(check_set) + addons + if six.text_type(val).lower().strip() in check_set: return True return False @@ -273,7 +292,7 @@ def uniq_merge_sorted(*lists): def uniq_merge(*lists): combined_list = [] for a_list in lists: - if isinstance(a_list, (str, basestring)): + if isinstance(a_list, six.string_types): a_list = a_list.strip().split(",") # Kickout the empty ones a_list = [a for a in a_list if len(a)] @@ -282,7 +301,7 @@ def uniq_merge(*lists): def clean_filename(fn): - for (k, v) in FN_REPLACEMENTS.iteritems(): + for (k, v) in FN_REPLACEMENTS.items(): fn = fn.replace(k, v) removals = [] for k in fn: @@ -296,14 +315,14 @@ def clean_filename(fn): def decomp_gzip(data, quiet=True): try: - buf = StringIO(str(data)) + buf = six.BytesIO(encode_text(data)) with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: - return gh.read() + return decode_binary(gh.read()) except Exception as e: if quiet: return data else: - raise DecompressionError(str(e)) + raise DecompressionError(six.text_type(e)) def extract_usergroup(ug_pair): @@ -362,7 +381,7 @@ def multi_log(text, console=True, stderr=True, def load_json(text, root_types=(dict,)): - decoded = json.loads(text) + decoded = json.loads(decode_binary(text)) if not isinstance(decoded, tuple(root_types)): expected_types = ", ".join([str(t) for t in root_types]) raise TypeError("(%s) root types expected, got %s instead" @@ -394,7 +413,7 @@ def get_cfg_option_str(yobj, key, default=None): if key not in yobj: return default val = yobj[key] - if not isinstance(val, (str, basestring)): + if not isinstance(val, six.string_types): val = str(val) return val @@ -433,7 +452,7 @@ def get_cfg_option_list(yobj, key, default=None): if isinstance(val, (list)): cval = [v for v in val] return cval - if not isinstance(val, (basestring)): + if not isinstance(val, six.string_types): val = str(val) return [val] @@ -708,10 +727,10 @@ def read_file_or_url(url, timeout=5, retries=10, def load_yaml(blob, default=None, allowed=(dict,)): loaded = default + blob = decode_binary(blob) try: - blob = str(blob) - LOG.debug(("Attempting to load yaml from string " - "of length %s with allowed root types %s"), + LOG.debug("Attempting to load yaml from string " + "of length %s with allowed root types %s", len(blob), allowed) converted = safeyaml.load(blob) if not isinstance(converted, allowed): @@ -746,14 +765,12 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None if md_resp.ok(): - md_str = str(md_resp) - md = load_yaml(md_str, default={}) + md = load_yaml(md_resp.contents, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None if ud_resp.ok(): - ud_str = str(ud_resp) - ud = ud_str + ud = ud_resp.contents return (md, ud) @@ -784,7 +801,7 @@ def read_conf_with_confd(cfgfile): if "conf_d" in cfg: confd = cfg['conf_d'] if confd: - if not isinstance(confd, (str, basestring)): + if not isinstance(confd, six.string_types): raise TypeError(("Config file %s contains 'conf_d' " "with non-string type %s") % (cfgfile, type_utils.obj_name(confd))) @@ -921,8 +938,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), return (None, None, None) resp = read_file_or_url(url) - if resp.contents.startswith(starts) and resp.ok(): - return (key, url, str(resp)) + if resp.ok() and resp.contents.startswith(starts): + return (key, url, resp.contents) return (key, url, None) @@ -1076,9 +1093,9 @@ def uniq_list(in_list): return out_list -def load_file(fname, read_cb=None, quiet=False): +def load_file(fname, read_cb=None, quiet=False, decode=True): LOG.debug("Reading from %s (quiet=%s)", fname, quiet) - ofh = StringIO() + ofh = six.BytesIO() try: with open(fname, 'rb') as ifh: pipe_in_out(ifh, ofh, chunk_cb=read_cb) @@ -1089,7 +1106,10 @@ def load_file(fname, read_cb=None, quiet=False): raise contents = ofh.getvalue() LOG.debug("Read %s bytes from %s", len(contents), fname) - return contents + if decode: + return decode_binary(contents) + else: + return contents def get_cmdline(): @@ -1219,7 +1239,7 @@ def logexc(log, msg, *args): def hash_blob(blob, routine, mlen=None): hasher = hashlib.new(routine) - hasher.update(blob) + hasher.update(encode_text(blob)) digest = hasher.hexdigest() # Don't get to long now if mlen is not None: @@ -1280,8 +1300,7 @@ def yaml_dumps(obj, explicit_start=True, explicit_end=True): indent=4, explicit_start=explicit_start, explicit_end=explicit_end, - default_flow_style=False, - allow_unicode=True) + default_flow_style=False) def ensure_dir(path, mode=None): @@ -1515,11 +1534,17 @@ def write_file(filename, content, mode=0o644, omode="wb"): @param filename: The full path of the file to write. @param content: The content to write to the file. @param mode: The filesystem mode to set on the file. - @param omode: The open mode used when opening the file (r, rb, a, etc.) + @param omode: The open mode used when opening the file (w, wb, a, etc.) """ ensure_dir(os.path.dirname(filename)) - LOG.debug("Writing to %s - %s: [%s] %s bytes", - filename, omode, mode, len(content)) + if 'b' in omode.lower(): + content = encode_text(content) + write_type = 'bytes' + else: + content = decode_binary(content) + write_type = 'characters' + LOG.debug("Writing to %s - %s: [%s] %s %s", + filename, omode, mode, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) @@ -1608,10 +1633,10 @@ def shellify(cmdlist, add_header=True): if isinstance(args, list): fixed = [] for f in args: - fixed.append("'%s'" % (str(f).replace("'", escaped))) + fixed.append("'%s'" % (six.text_type(f).replace("'", escaped))) content = "%s%s\n" % (content, ' '.join(fixed)) cmds_made += 1 - elif isinstance(args, (str, basestring)): + elif isinstance(args, six.string_types): content = "%s%s\n" % (content, args) cmds_made += 1 else: @@ -1722,7 +1747,7 @@ def expand_package_list(version_fmt, pkgs): pkglist = [] for pkg in pkgs: - if isinstance(pkg, basestring): + if isinstance(pkg, six.string_types): pkglist.append(pkg) continue diff --git a/packages/bddeb b/packages/bddeb index 9d264f92..83ca68bb 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -38,6 +38,7 @@ PKG_MP = { 'pyserial': 'python-serial', 'pyyaml': 'python-yaml', 'requests': 'python-requests', + 'six': 'python-six', } DEBUILD_ARGS = ["-S", "-d"] diff --git a/packages/brpm b/packages/brpm index 9657b1dd..72bfca08 100755 --- a/packages/brpm +++ b/packages/brpm @@ -45,6 +45,7 @@ PKG_MP = { 'pyserial': 'pyserial', 'pyyaml': 'PyYAML', 'requests': 'python-requests', + 'six': 'python-six', }, 'suse': { 'argparse': 'python-argparse', @@ -56,6 +57,7 @@ PKG_MP = { 'pyserial': 'python-pyserial', 'pyyaml': 'python-yaml', 'requests': 'python-requests', + 'six': 'python-six', } } diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 03296e62..a35afc27 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -1,11 +1,11 @@ """Tests for handling of userdata within cloud init.""" -import StringIO - import gzip import logging import os +from six import BytesIO, StringIO + from email.mime.application import MIMEApplication from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart @@ -53,7 +53,7 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): self.patchUtils(root) def capture_log(self, lvl=logging.DEBUG): - log_file = StringIO.StringIO() + log_file = StringIO() self._log_handler = logging.StreamHandler(log_file) self._log_handler.setLevel(lvl) self._log = log.getLogger() @@ -351,9 +351,9 @@ p: 1 """Tests that individual message gzip encoding works.""" def gzip_part(text): - contents = StringIO.StringIO() - f = gzip.GzipFile(fileobj=contents, mode='w') - f.write(str(text)) + contents = BytesIO() + f = gzip.GzipFile(fileobj=contents, mode='wb') + f.write(util.encode_text(text)) f.flush() f.close() return MIMEApplication(contents.getvalue(), 'gzip') diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index e9235951..ae9e6c22 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -85,7 +85,7 @@ class TestNoCloudDataSource(MockerTestCase): data = { 'fs_label': None, - 'meta-data': {'instance-id': 'IID'}, + 'meta-data': yaml.safe_dump({'instance-id': 'IID'}), 'user-data': "USER_DATA_RAW", } diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 49894e51..81ef1546 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -20,12 +20,11 @@ import copy import json import re -from StringIO import StringIO - -from urlparse import urlparse - from .. import helpers as test_helpers +from six import StringIO +from six.moves.urllib.parse import urlparse + from cloudinit import helpers from cloudinit import settings from cloudinit.sources import DataSourceOpenStack as ds diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 33a1d6e1..6e1a0b69 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -4,6 +4,8 @@ import mocker import os +from six import StringIO + from cloudinit import distros from cloudinit import helpers from cloudinit import settings @@ -11,8 +13,6 @@ from cloudinit import util from cloudinit.distros.parsers.sys_conf import SysConf -from StringIO import StringIO - BASE_NET_CFG = ''' auto lo diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 203dd2aa..f5832365 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -16,12 +16,12 @@ class TestAptProxyConfig(MockerTestCase): self.cfile = os.path.join(self.tmp, "config.cfg") def _search_apt_config(self, contents, ptype, value): - print( + ## print( + ## r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value), + ## contents, "flags=re.IGNORECASE") + return re.search( r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value), - contents, "flags=re.IGNORECASE") - return(re.search( - r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value), - contents, flags=re.IGNORECASE)) + contents, flags=re.IGNORECASE) def test_apt_proxy_written(self): cfg = {'apt_proxy': 'myproxy'} diff --git a/tests/unittests/test_handler/test_handler_locale.py b/tests/unittests/test_handler/test_handler_locale.py index eb251636..690ef86f 100644 --- a/tests/unittests/test_handler/test_handler_locale.py +++ b/tests/unittests/test_handler/test_handler_locale.py @@ -29,7 +29,7 @@ from .. import helpers as t_help from configobj import ConfigObj -from StringIO import StringIO +from six import BytesIO import logging @@ -59,6 +59,6 @@ class TestLocale(t_help.FilesystemMockingTestCase): cc = self._get_cloud('sles') cc_locale.handle('cc_locale', cfg, cc, LOG, []) - contents = util.load_file('/etc/sysconfig/language') - n_cfg = ConfigObj(StringIO(contents)) + contents = util.load_file('/etc/sysconfig/language', decode=False) + n_cfg = ConfigObj(BytesIO(contents)) self.assertEquals({'RC_LANG': cfg['locale']}, dict(n_cfg)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 40481f16..579377fb 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -22,7 +22,7 @@ import base64 import gzip import tempfile -from StringIO import StringIO +from six import StringIO from cloudinit import cloud from cloudinit import distros diff --git a/tests/unittests/test_handler/test_handler_set_hostname.py b/tests/unittests/test_handler/test_handler_set_hostname.py index e1530e30..a9f7829b 100644 --- a/tests/unittests/test_handler/test_handler_set_hostname.py +++ b/tests/unittests/test_handler/test_handler_set_hostname.py @@ -9,7 +9,7 @@ from .. import helpers as t_help import logging -from StringIO import StringIO +from six import BytesIO from configobj import ConfigObj @@ -38,8 +38,8 @@ class TestHostname(t_help.FilesystemMockingTestCase): cc_set_hostname.handle('cc_set_hostname', cfg, cc, LOG, []) if not distro.uses_systemd(): - contents = util.load_file("/etc/sysconfig/network") - n_cfg = ConfigObj(StringIO(contents)) + contents = util.load_file("/etc/sysconfig/network", decode=False) + n_cfg = ConfigObj(BytesIO(contents)) self.assertEquals({'HOSTNAME': 'blah.blah.blah.yahoo.com'}, dict(n_cfg)) diff --git a/tests/unittests/test_handler/test_handler_timezone.py b/tests/unittests/test_handler/test_handler_timezone.py index 874db340..10ea2040 100644 --- a/tests/unittests/test_handler/test_handler_timezone.py +++ b/tests/unittests/test_handler/test_handler_timezone.py @@ -29,7 +29,7 @@ from .. import helpers as t_help from configobj import ConfigObj -from StringIO import StringIO +from six import BytesIO import logging @@ -67,8 +67,8 @@ class TestTimezone(t_help.FilesystemMockingTestCase): cc_timezone.handle('cc_timezone', cfg, cc, LOG, []) - contents = util.load_file('/etc/sysconfig/clock') - n_cfg = ConfigObj(StringIO(contents)) + contents = util.load_file('/etc/sysconfig/clock', decode=False) + n_cfg = ConfigObj(BytesIO(contents)) self.assertEquals({'TIMEZONE': cfg['timezone']}, dict(n_cfg)) contents = util.load_file('/etc/localtime') diff --git a/tests/unittests/test_handler/test_handler_yum_add_repo.py b/tests/unittests/test_handler/test_handler_yum_add_repo.py index 435c9787..81806ad1 100644 --- a/tests/unittests/test_handler/test_handler_yum_add_repo.py +++ b/tests/unittests/test_handler/test_handler_yum_add_repo.py @@ -6,7 +6,7 @@ from .. import helpers import logging -from StringIO import StringIO +from six import BytesIO import configobj @@ -52,8 +52,9 @@ class TestConfig(helpers.FilesystemMockingTestCase): } self.patchUtils(self.tmp) cc_yum_add_repo.handle('yum_add_repo', cfg, None, LOG, []) - contents = util.load_file("/etc/yum.repos.d/epel_testing.repo") - contents = configobj.ConfigObj(StringIO(contents)) + contents = util.load_file("/etc/yum.repos.d/epel_testing.repo", + decode=False) + contents = configobj.ConfigObj(BytesIO(contents)) expected = { 'epel_testing': { 'name': 'Extra Packages for Enterprise Linux 5 - Testing', -- cgit v1.2.3 From 3b798b5d5c3caa5d0e8e534855e29010ca932aaa Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Jan 2015 21:21:04 -0500 Subject: Low hanging Python 3 fruit. --- cloudinit/config/cc_ca_certs.py | 4 ++-- cloudinit/config/cc_chef.py | 6 ++++-- cloudinit/distros/__init__.py | 12 ++++++++++-- cloudinit/distros/debian.py | 2 +- cloudinit/distros/rhel_util.py | 4 ++-- cloudinit/distros/sles.py | 2 +- cloudinit/sources/DataSourceAltCloud.py | 12 ++++++------ cloudinit/sources/DataSourceAzure.py | 4 ++-- cloudinit/sources/DataSourceMAAS.py | 10 ++++++---- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/templater.py | 2 +- cloudinit/util.py | 7 +++++-- templates/resolv.conf.tmpl | 2 +- tests/unittests/helpers.py | 4 ++-- tests/unittests/test_datasource/test_configdrive.py | 2 +- tests/unittests/test_datasource/test_digitalocean.py | 7 +++---- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 2 +- tests/unittests/test_datasource/test_smartos.py | 4 +++- .../unittests/test_handler/test_handler_apt_configure.py | 2 +- tests/unittests/test_merging.py | 16 +++++++++------- tools/ccfg-merge-debug | 4 ++-- 22 files changed, 65 insertions(+), 47 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 4f2a46a1..8248b020 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -44,7 +44,7 @@ def add_ca_certs(certs): if certs: # First ensure they are strings... cert_file_contents = "\n".join([str(c) for c in certs]) - util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644) + util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644) # Append cert filename to CA_CERT_CONFIG file. # We have to strip the content because blank lines in the file @@ -63,7 +63,7 @@ def remove_default_ca_certs(): """ util.delete_dir_contents(CA_CERT_PATH) util.delete_dir_contents(CA_CERT_SYSTEM_PATH) - util.write_file(CA_CERT_CONFIG, "", mode=0644) + util.write_file(CA_CERT_CONFIG, "", mode=0o644) debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no" util.subp(('debconf-set-selections', '-'), debconf_sel) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fc837363..584199e5 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -76,6 +76,8 @@ from cloudinit import templater from cloudinit import url_helper from cloudinit import util +import six + RUBY_VERSION_DEFAULT = "1.8" CHEF_DIRS = tuple([ @@ -261,7 +263,7 @@ def run_chef(chef_cfg, log): cmd_args = chef_cfg['exec_arguments'] if isinstance(cmd_args, (list, tuple)): cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): + elif isinstance(cmd_args, six.string_types): cmd.append(cmd_args) else: log.warn("Unknown type %s provided for chef" @@ -300,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) + util.write_file(tmpf, str(content), mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 4ebccdda..6b96d58c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -25,7 +25,6 @@ import six from six import StringIO import abc -import itertools import os import re @@ -37,6 +36,15 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +try: + # Python 3 + from six import filter +except ImportError: + # Python 2 + from itertools import ifilter as filter + + + OSFAMILIES = { 'debian': ['debian', 'ubuntu'], 'redhat': ['fedora', 'rhel'], @@ -853,7 +861,7 @@ def extract_default(users, default_name=None, default_config=None): return config['default'] tmp_users = users.items() - tmp_users = dict(itertools.ifilter(safe_find, tmp_users)) + tmp_users = dict(filter(safe_find, tmp_users)) if not tmp_users: return (default_name, default_config) else: diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index b09eb094..6d3a82bf 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -97,7 +97,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 063d536e..903d7793 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -50,7 +50,7 @@ def update_sysconfig_file(fn, adjustments, allow_empty=False): ] if not exists: lines.insert(0, util.make_header()) - util.write_file(fn, "\n".join(lines) + "\n", 0644) + util.write_file(fn, "\n".join(lines) + "\n", 0o644) # Helper function to read a RHEL/SUSE /etc/sysconfig/* file @@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers): r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0644) + util.write_file(fn, str(r_conf), 0o644) diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py index 0c6d1203..620c974c 100644 --- a/cloudinit/distros/sles.py +++ b/cloudinit/distros/sles.py @@ -113,7 +113,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, str(conf), 0o644) def _read_system_hostname(self): host_fn = self.hostname_conf_fn diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 1e913a6e..69053d0b 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -124,11 +124,11 @@ class DataSourceAltCloud(sources.DataSource): cmd = CMD_DMI_SYSTEM try: (cmd_out, _err) = util.subp(cmd) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: LOG.debug(('Failed command: %s\n%s') % \ (' '.join(cmd), _err.message)) return 'UNKNOWN' - except OSError, _err: + except OSError as _err: LOG.debug(('Failed command: %s\n%s') % \ (' '.join(cmd), _err.message)) return 'UNKNOWN' @@ -211,11 +211,11 @@ class DataSourceAltCloud(sources.DataSource): cmd = CMD_PROBE_FLOPPY (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False - except OSError, _err: + except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False @@ -228,11 +228,11 @@ class DataSourceAltCloud(sources.DataSource): cmd.append('--exit-if-exists=' + floppy_dev) (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) - except ProcessExecutionError, _err: + except ProcessExecutionError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False - except OSError, _err: + except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err.message) return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 09bc196d..29ae2c22 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -151,7 +151,7 @@ class DataSourceAzureNet(sources.DataSource): # walinux agent writes files world readable, but expects # the directory to be protected. - write_files(ddir, files, dirmode=0700) + write_files(ddir, files, dirmode=0o700) # handle the hostname 'publishing' try: @@ -390,7 +390,7 @@ def write_files(datadir, files, dirmode=None): util.ensure_dir(datadir, dirmode) for (name, content) in files.items(): util.write_file(filename=os.path.join(datadir, name), - content=content, mode=0600) + content=content, mode=0o600) def invoke_agent(cmd): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 9a3e30c5..8f9c81de 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function + from email.utils import parsedate import errno import oauth.oauth as oauth @@ -361,7 +363,7 @@ if __name__ == "__main__": return (urllib2.urlopen(req).read()) def printurl(url, headers_cb): - print "== %s ==\n%s\n" % (url, geturl(url, headers_cb)) + print("== %s ==\n%s\n" % (url, geturl(url, headers_cb))) def crawl(url, headers_cb=None): if url.endswith("/"): @@ -386,9 +388,9 @@ if __name__ == "__main__": version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) - print "=== userdata ===" - print userdata - print "=== metadata ===" + print("=== userdata ===") + print(userdata) + print("=== metadata ===") pprint.pprint(metadata) elif args.subcmd == "get": diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index e2469f6e..f9dac29e 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -280,7 +280,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None, # allvars expands to all existing variables by using '${!x*}' notation # where x is lower or upper case letters or '_' - allvars = ["${!%s*}" % x for x in string.letters + "_"] + allvars = ["${!%s*}" % x for x in string.ascii_letters + "_"] keylist_in = keylist if keylist is None: diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 4cd3f13d..a9231482 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -137,7 +137,7 @@ def render_from_file(fn, params): return renderer(content, params) -def render_to_file(fn, outfn, params, mode=0644): +def render_to_file(fn, outfn, params, mode=0o644): contents = render_from_file(fn, params) util.write_file(outfn, contents, mode=mode) diff --git a/cloudinit/util.py b/cloudinit/util.py index 434ba7fb..94fd5c70 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -142,6 +142,9 @@ class ProcessExecutionError(IOError): 'reason': self.reason, } IOError.__init__(self, message) + # For backward compatibility with Python 2. + if not hasattr(self, 'message'): + self.message = message class SeLinuxGuard(object): @@ -260,7 +263,7 @@ def translate_bool(val, addons=None): def rand_str(strlen=32, select_from=None): if not select_from: - select_from = string.letters + string.digits + select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) @@ -1127,7 +1130,7 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None): bytes_piped = 0 while True: data = in_fh.read(chunk_size) - if data == '': + if len(data) == 0: break else: out_fh.write(data) diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl index 1300156c..bfae80db 100644 --- a/templates/resolv.conf.tmpl +++ b/templates/resolv.conf.tmpl @@ -24,7 +24,7 @@ sortlist {% for sort in sortlist %}{{sort}} {% endfor %} {% if options or flags %} options {% for flag in flags %}{{flag}} {% endfor %} -{% for key, value in options.iteritems() -%} +{% for key, value in options.items() -%} {{key}}:{{value}} {% endfor %} {% endif %} diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 38a2176d..70b8116f 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -65,7 +65,7 @@ if PY26: def assertDictContainsSubset(self, expected, actual, msg=None): missing = [] mismatched = [] - for k, v in expected.iteritems(): + for k, v in expected.items(): if k not in actual: missing.append(k) elif actual[k] != v: @@ -243,7 +243,7 @@ class HttprettyTestCase(TestCase): def populate_dir(path, files): if not os.path.exists(path): os.makedirs(path) - for (name, content) in files.iteritems(): + for (name, content) in files.items(): with open(os.path.join(path, name), "w") as fp: fp.write(content) fp.close() diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 800c5fd8..258c68e2 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -338,7 +338,7 @@ def populate_ds_from_read_config(cfg_ds, source, results): def populate_dir(seed_dir, files): - for (name, content) in files.iteritems(): + for (name, content) in files.items(): path = os.path.join(seed_dir, name) dirname = os.path.dirname(path) if not os.path.isdir(dirname): diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py index d1270fc2..98f9cfac 100644 --- a/tests/unittests/test_datasource/test_digitalocean.py +++ b/tests/unittests/test_datasource/test_digitalocean.py @@ -18,8 +18,7 @@ import httpretty import re -from types import ListType -from urlparse import urlparse +from six.moves.urllib_parse import urlparse from cloudinit import settings from cloudinit import helpers @@ -110,7 +109,7 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual([DO_META.get('public-keys')], self.ds.get_public_ssh_keys()) - self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + self.assertIsInstance(self.ds.get_public_ssh_keys(), list) @httpretty.activate def test_multiple_ssh_keys(self): @@ -124,4 +123,4 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase): self.assertEqual(DO_META.get('public-keys').splitlines(), self.ds.get_public_ssh_keys()) - self.assertIs(type(self.ds.get_public_ssh_keys()), ListType) + self.assertIsInstance(self.ds.get_public_ssh_keys(), list) diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 06050bb1..aa60eb33 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -19,7 +19,7 @@ import httpretty import re from base64 import b64encode, b64decode -from urlparse import urlparse +from six.moves.urllib_parse import urlparse from cloudinit import settings from cloudinit import helpers diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index ddf77265..b79237f0 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -294,7 +294,7 @@ class TestParseShellConfig(unittest.TestCase): def populate_context_dir(path, variables): data = "# Context variables generated by OpenNebula\n" - for (k, v) in variables.iteritems(): + for (k, v) in variables.items(): data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''"))) populate_dir(path, {'context.sh': data}) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 35d7ef5e..01b9b73e 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,6 +22,8 @@ # return responses. # +from __future__ import print_function + import base64 from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS @@ -369,7 +371,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] if re.match(r'.*\/mdata-user-data$', name_f): found_new = True - print name_f + print(name_f) self.assertEquals(permissions, '400') self.assertFalse(found_new) diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 2c3dad72..d72fa8c7 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -62,7 +62,7 @@ class TestAptProxyConfig(unittest.TestCase): contents = str(util.read_file_or_url(self.pfile)) - for ptype, pval in values.iteritems(): + for ptype, pval in values.items(): self.assertTrue(self._search_apt_config(contents, ptype, pval)) def test_proxy_deleted(self): diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 07b610f7..976d8283 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -11,11 +11,13 @@ import glob import os import random import re +import six import string SOURCE_PAT = "source*.*yaml" EXPECTED_PAT = "expected%s.yaml" -TYPES = [long, int, dict, str, list, tuple, None] +TYPES = [dict, str, list, tuple, None] +TYPES.extend(six.integer_types) def _old_mergedict(src, cand): @@ -25,7 +27,7 @@ def _old_mergedict(src, cand): Nested dictionaries are merged recursively. """ if isinstance(src, dict) and isinstance(cand, dict): - for (k, v) in cand.iteritems(): + for (k, v) in cand.items(): if k not in src: src[k] = v else: @@ -42,8 +44,8 @@ def _old_mergemanydict(*args): def _random_str(rand): base = '' - for _i in xrange(rand.randint(1, 2 ** 8)): - base += rand.choice(string.letters + string.digits) + for _i in range(rand.randint(1, 2 ** 8)): + base += rand.choice(string.ascii_letters + string.digits) return base @@ -64,7 +66,7 @@ def _make_dict(current_depth, max_depth, rand): if t in [dict, list, tuple]: if t in [dict]: amount = rand.randint(0, 5) - keys = [_random_str(rand) for _i in xrange(0, amount)] + keys = [_random_str(rand) for _i in range(0, amount)] base = {} for k in keys: try: @@ -74,14 +76,14 @@ def _make_dict(current_depth, max_depth, rand): elif t in [list, tuple]: base = [] amount = rand.randint(0, 5) - for _i in xrange(0, amount): + for _i in range(0, amount): try: base.append(_make_dict(current_depth + 1, max_depth, rand)) except _NoMoreException: pass if t in [tuple]: base = tuple(base) - elif t in [long, int]: + elif t in six.integer_types: base = rand.randint(0, 2 ** 8) elif t in [str]: base = _random_str(rand) diff --git a/tools/ccfg-merge-debug b/tools/ccfg-merge-debug index 85227da7..1f08e0cb 100755 --- a/tools/ccfg-merge-debug +++ b/tools/ccfg-merge-debug @@ -51,7 +51,7 @@ def main(): c_handlers.register(ccph) called = [] - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in called: continue handlers.call_begin(mod, data, frequency) @@ -76,7 +76,7 @@ def main(): # Give callbacks opportunity to finalize called = [] - for (_ctype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.items(): if mod in called: continue handlers.call_end(mod, data, frequency) -- cgit v1.2.3 From 926a3df79a10ede61967c60f48ff0670a36e689a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 12:41:04 -0500 Subject: More Python 3 test fixes. --- cloudinit/config/cc_write_files.py | 5 +++-- tests/unittests/test_datasource/test_opennebula.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index a73d6f4e..4b03ea91 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -18,6 +18,7 @@ import base64 import os +import six from cloudinit.settings import PER_INSTANCE from cloudinit import util @@ -25,7 +26,7 @@ from cloudinit import util frequency = PER_INSTANCE DEFAULT_OWNER = "root:root" -DEFAULT_PERMS = 0644 +DEFAULT_PERMS = 0o644 UNKNOWN_ENC = 'text/plain' @@ -79,7 +80,7 @@ def write_files(name, files, log): def decode_perms(perm, default, log): try: - if isinstance(perm, (int, long, float)): + if isinstance(perm, six.integer_types + (float,)): # Just 'downcast' it (if a float) return int(perm) else: diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index 1a8d2122..31c6232f 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -33,7 +33,7 @@ TEST_VARS = { } INVALID_CONTEXT = ';' -USER_DATA = '#cloud-config\napt_upgrade: true' +USER_DATA = b'#cloud-config\napt_upgrade: true' SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -- cgit v1.2.3 From 18b35de06432869a9d859e2978e7e9567eba66a2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 14:48:23 -0500 Subject: Another handling of b64decode. Also, restore Python 2 compatibility. --- cloudinit/config/cc_seed_random.py | 8 +++++++- cloudinit/sources/DataSourceOpenNebula.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 3b7235bf..981e1b08 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -38,7 +38,13 @@ def _decode(data, encoding=None): if not encoding or encoding.lower() in ['raw']: return data elif encoding.lower() in ['base64', 'b64']: - return base64.b64decode(data) + # Try to give us a native string in both Python 2 and 3, and remember + # that b64decode() returns bytes in Python 3. + decoded = base64.b64decode(data) + try: + return decoded.decode('utf-8') + except UnicodeDecodeError: + return decoded elif encoding.lower() in ['gzip', 'gz']: return util.decomp_gzip(data, quiet=False) else: diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 691b39f8..6da569ec 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -25,6 +25,7 @@ # along with this program. If not, see . import base64 +import codecs import os import pwd import re @@ -34,6 +35,8 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import util +import six + LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" @@ -43,6 +46,12 @@ CONTEXT_DISK_FILES = ["context.sh"] VALID_DSMODES = ("local", "net", "disabled") +def utf8_open(path): + if six.PY3: + return open(path, 'r', encoding='utf-8') + return codecs.open(path, 'r', encoding='utf-8') + + class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -380,7 +389,7 @@ def read_context_disk_dir(source_dir, asuser=None): "does not exist", asuser) try: path = os.path.join(source_dir, 'context.sh') - with open(path, 'r', encoding='utf-8') as f: + with utf8_open(path) as f: content = f.read().strip() context = parse_shell_config(content, asuser=asuser) -- cgit v1.2.3 From 1e76dad45e3bce4dac5a638dda970fc02a044dbb Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 14:24:22 -0500 Subject: Respond to review: - Remove str() wrappers to second argument to write_files() where it is no longer necessary. Also: Fixed a couple of other octal literals which clearly weren't being tested. --- cloudinit/config/cc_chef.py | 2 +- cloudinit/config/cc_puppet.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 2 +- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/distros/arch.py | 2 +- cloudinit/distros/gentoo.py | 2 +- cloudinit/distros/rhel_util.py | 2 +- cloudinit/user_data.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 584199e5..e18c5405 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -302,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0o700) + util.write_file(tmpf, content, mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 6f1b3c57..4501598e 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -91,7 +91,7 @@ def handle(name, cfg, cloud, log, _args): util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') util.ensure_dir(PUPPET_SSL_CERT_DIR) util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') - util.write_file(PUPPET_SSL_CERT_PATH, str(cfg)) + util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: # Iterate throug the config items, we'll use ConfigParser.set diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 7d2ec10a..1f769c0a 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -82,7 +82,7 @@ def handle(name, _cfg, cloud, log, _args): resp = uhelp.readurl(url) # Ensure its a valid http response (and something gotten) if resp.ok() and resp.contents: - util.write_file(fname, str(resp), mode=0700) + util.write_file(fname, resp, mode=0o700) wrote_fns.append(fname) except Exception as e: captured_excps.append(e) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 598c3a3e..66dc3363 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -33,6 +33,6 @@ def handle(name, cfg, cloud, log, _args): cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(out_fn, content, 0700) + util.write_file(out_fn, content, 0o700) except: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 53013dcb..f5786a31 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, _args): # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') - with util.umask(077): + with util.umask(0o77): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') pem_name = os.path.join(pki_dir, 'minion.pem') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index e540e0bc..45fcf26f 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -129,7 +129,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, conf, 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 09dd0d73..9e80583c 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -108,7 +108,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, conf, 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 903d7793..84aad623 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers): r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0o644) + util.write_file(fn, r_conf, 0o644) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 3f860f3b..bf5642a5 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -248,7 +248,7 @@ class UserDataProcessor(object): resp = util.read_file_or_url(include_url, ssl_details=self.ssl_details) if include_once_on and resp.ok(): - util.write_file(include_once_fn, str(resp), mode=0o600) + util.write_file(include_once_fn, resp, mode=0o600) if resp.ok(): content = str(resp) else: -- cgit v1.2.3 From 6e742d20e9ed56498925c7c850cd5da65d063b4b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 15:03:52 -0500 Subject: Respond to review: - Refactor both the base64 encoding and decoding into utility functions. Also: - Mechanically fix some other broken untested code. --- cloudinit/config/cc_seed_random.py | 8 +------ cloudinit/config/cc_ssh_authkey_fingerprints.py | 2 +- cloudinit/sources/DataSourceOpenNebula.py | 7 +----- cloudinit/sources/DataSourceSmartOS.py | 11 +-------- cloudinit/util.py | 20 ++++++++++++++++ tests/unittests/test_datasource/test_azure.py | 28 ++++++++-------------- tests/unittests/test_datasource/test_opennebula.py | 11 ++------- tests/unittests/test_datasource/test_smartos.py | 14 ++++------- .../test_handler/test_handler_seed_random.py | 12 ++-------- 9 files changed, 42 insertions(+), 71 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 981e1b08..bb64b0f5 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -38,13 +38,7 @@ def _decode(data, encoding=None): if not encoding or encoding.lower() in ['raw']: return data elif encoding.lower() in ['base64', 'b64']: - # Try to give us a native string in both Python 2 and 3, and remember - # that b64decode() returns bytes in Python 3. - decoded = base64.b64decode(data) - try: - return decoded.decode('utf-8') - except UnicodeDecodeError: - return decoded + return util.b64d(data) elif encoding.lower() in ['gzip', 'gz']: return util.decomp_gzip(data, quiet=False) else: diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py index 51580633..6ce831bc 100644 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -32,7 +32,7 @@ from cloudinit import util def _split_hash(bin_hash): split_up = [] - for i in xrange(0, len(bin_hash), 2): + for i in range(0, len(bin_hash), 2): split_up.append(bin_hash[i:i + 2]) return split_up diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index a0275cda..61709c1b 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -426,12 +426,7 @@ def read_context_disk_dir(source_dir, asuser=None): context.get('USER_DATA_ENCODING')) if encoding == "base64": try: - userdata = base64.b64decode(results['userdata']) - # In Python 3 we still expect a str, but b64decode will return - # bytes. Convert to str. - if isinstance(userdata, bytes): - userdata = userdata.decode('utf-8') - results['userdata'] = userdata + results['userdata'] = util.b64d(results['userdata']) except TypeError: LOG.warn("Failed base64 decoding of userdata") diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f59ad3d6..9d48beab 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -351,16 +351,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None, if b64: try: - # Generally, we want native strings in the values. Python 3's - # b64decode will return bytes though, so decode them to utf-8 if - # possible. If that fails, return the bytes. - decoded = base64.b64decode(resp) - try: - if isinstance(decoded, bytes): - return decoded.decode('utf-8') - except UnicodeDecodeError: - pass - return decoded + return util.b64d(resp) # Bogus input produces different errors in Python 2 and 3; catch both. except (TypeError, binascii.Error): LOG.warn("Failed base64 decoding key '%s'", noun) diff --git a/cloudinit/util.py b/cloudinit/util.py index 766f8e32..8916cc11 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -44,6 +44,7 @@ import sys import tempfile import time +from base64 import b64decode, b64encode from six.moves.urllib import parse as urlparse import six @@ -90,6 +91,25 @@ def encode_text(text, encoding='utf-8'): return text return text.encode(encoding) + +def b64d(source): + # Base64 decode some data, accepting bytes or unicode/str, and returning + # str/unicode if the result is utf-8 compatible, otherwise returning bytes. + decoded = b64decode(source) + if isinstance(decoded, bytes): + try: + return decoded.decode('utf-8') + except UnicodeDecodeError: + return decoded + +def b64e(source): + # Base64 encode some data, accepting bytes or unicode/str, and returning + # str/unicode if the result is utf-8 compatible, otherwise returning bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return b64encode(source).decode('utf-8') + + # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 97a53bee..965bce4b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -1,5 +1,5 @@ from cloudinit import helpers -from cloudinit.util import load_file +from cloudinit.util import b64e, load_file from cloudinit.sources import DataSourceAzure from ..helpers import TestCase, populate_dir @@ -12,7 +12,6 @@ try: except ImportError: from contextlib2 import ExitStack -import base64 import crypt import os import stat @@ -22,13 +21,6 @@ import tempfile import unittest -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return base64.b64encode(source).decode('us-ascii') - - def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): if data is None: data = {'HostName': 'FOOHOST'} @@ -58,7 +50,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): content += "<%s%s>%s\n" % (key, attrs, val, key) if userdata: - content += "%s\n" % (b64(userdata)) + content += "%s\n" % (b64e(userdata)) if pubkeys: content += "\n" @@ -189,7 +181,7 @@ class TestAzureDataSource(TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64(yaml.dump(cfg)), + 'dscfg': {'text': b64e(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} @@ -241,7 +233,7 @@ class TestAzureDataSource(TestCase): def test_userdata_found(self): mydata = "FOOBAR" - odata = {'UserData': b64(mydata)} + odata = {'UserData': b64e(mydata)} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -289,7 +281,7 @@ class TestAzureDataSource(TestCase): 'command': 'my-bounce-command', 'hostname_command': 'my-hostname-command'}} odata = {'HostName': "xhost", - 'dscfg': {'text': b64(yaml.dump(cfg)), + 'dscfg': {'text': b64e(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -304,7 +296,7 @@ class TestAzureDataSource(TestCase): # config specifying set_hostname off should not bounce cfg = {'set_hostname': False} odata = {'HostName': "xhost", - 'dscfg': {'text': b64(yaml.dump(cfg)), + 'dscfg': {'text': b64e(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -333,7 +325,7 @@ class TestAzureDataSource(TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64(yaml.dump(dscfg)), + 'dscfg': {'text': b64e(yaml.dump(dscfg)), 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} @@ -370,7 +362,7 @@ class TestAzureDataSource(TestCase): def test_existing_ovf_same(self): # waagent/SharedConfig left alone if found ovf-env.xml same as cached - odata = {'UserData': b64("SOMEUSERDATA")} + odata = {'UserData': b64e("SOMEUSERDATA")} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} populate_dir(self.waagent_d, @@ -394,9 +386,9 @@ class TestAzureDataSource(TestCase): # 'get_data' should remove SharedConfig.xml in /var/lib/waagent # if ovf-env.xml differs. cached_ovfenv = construct_valid_ovf_env( - {'userdata': b64("FOO_USERDATA")}) + {'userdata': b64e("FOO_USERDATA")}) new_ovfenv = construct_valid_ovf_env( - {'userdata': b64("NEW_USERDATA")}) + {'userdata': b64e("NEW_USERDATA")}) populate_dir(self.waagent_d, {'ovf-env.xml': cached_ovfenv, diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index e5a4bd18..27adf21b 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -3,19 +3,12 @@ from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util from ..helpers import TestCase, populate_dir -from base64 import b64encode import os import pwd import shutil import tempfile import unittest -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return b64encode(source).decode('us-ascii') - TEST_VARS = { 'VAR1': 'single', @@ -186,7 +179,7 @@ class TestOpenNebulaDataSource(TestCase): self.assertEqual(USER_DATA, results['userdata']) def test_user_data_encoding_required_for_decode(self): - b64userdata = b64(USER_DATA) + b64userdata = util.b64e(USER_DATA) for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) populate_context_dir(my_d, {k: b64userdata}) @@ -198,7 +191,7 @@ class TestOpenNebulaDataSource(TestCase): def test_user_data_base64_encoding(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_context_dir(my_d, {k: b64(USER_DATA), + populate_context_dir(my_d, {k: util.b64e(USER_DATA), 'USERDATA_ENCODING': 'base64'}) results = ds.read_context_disk_dir(my_d) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index b5ebf94d..8b62b1b1 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,9 +24,9 @@ from __future__ import print_function -import base64 from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS +from cloudinit.util import b64e from .. import helpers import os import os.path @@ -36,12 +36,6 @@ import tempfile import stat import uuid -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return base64.b64encode(source).decode('us-ascii') - MOCK_RETURNS = { 'hostname': 'test-host', @@ -239,7 +233,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_all'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64(my_returns[k]) + my_returns[k] = b64e(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -260,7 +254,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns['b64-cloud-init:user-data'] = "true" my_returns['b64-hostname'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64(my_returns[k]) + my_returns[k] = b64e(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -276,7 +270,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_keys'] = 'hostname,ignored' for k in ('hostname',): - my_returns[k] = b64(my_returns[k]) + my_returns[k] = b64e(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index d3f18fa0..0bcdcb31 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -18,7 +18,6 @@ from cloudinit.config import cc_seed_random -import base64 import gzip import tempfile @@ -38,13 +37,6 @@ import logging LOG = logging.getLogger(__name__) -def b64(source): - # In Python 3, b64encode only accepts bytes and returns bytes. - if not isinstance(source, bytes): - source = source.encode('utf-8') - return base64.b64encode(source).decode('us-ascii') - - class TestRandomSeed(t_help.TestCase): def setUp(self): super(TestRandomSeed, self).setUp() @@ -141,7 +133,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("big-toe", contents) def test_append_random_base64(self): - data = b64('bubbles') + data = util.b64e('bubbles') cfg = { 'random_seed': { 'file': self._seed_file, @@ -154,7 +146,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("bubbles", contents) def test_append_random_b64(self): - data = b64('kit-kat') + data = util.b64e('kit-kat') cfg = { 'random_seed': { 'file': self._seed_file, -- cgit v1.2.3 From c8e75ddf808f4ac017f609bafc101648b7568935 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 17:17:48 +0000 Subject: fix use of 'letters' and translate --- cloudinit/config/cc_set_passwords.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 4ca85e21..8b705d90 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -28,11 +28,11 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -from string import letters, digits +from string import ascii_letters, digits # We are removing certain 'painful' letters/numbers -PW_SET = (letters.translate(None, 'loLOI') + - digits.translate(None, '01')) +PW_SET = (''.join([x for x in ascii_letters + digits + if x not in 'loLOI01'])) def handle(_name, cfg, cloud, log, args): -- cgit v1.2.3 From 888db3e6bb9076973d2f6a73e0c4f691caa89603 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 17:25:01 +0000 Subject: fix parse_qs usage --- cloudinit/config/cc_rightscale_userdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 1f769c0a..24880d13 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -41,7 +41,7 @@ from cloudinit.settings import PER_INSTANCE from cloudinit import url_helper as uhelp from cloudinit import util -from urlparse import parse_qs +from six.moves.urllib_parse import parse_qs frequency = PER_INSTANCE -- cgit v1.2.3 From b8eb55f9acdf92a58d3c72b0c5e5437c4f0272c1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 21:33:11 +0000 Subject: use encode_text --- bin/cloud-init | 2 +- cloudinit/config/cc_bootcmd.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/bin/cloud-init b/bin/cloud-init index d67b2b6d..6c83c2e7 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -428,7 +428,7 @@ def atomic_write_json(path, data): try: tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), delete=False) - tf.write((json.dumps(data, indent=1) + "\n").encode()) + tf.write(util.encode_text(json.dumps(data, indent=1) + "\n")) tf.close() os.rename(tf.name, path) except Exception as e: diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 3ac22967..a295cc4e 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -36,7 +36,7 @@ def handle(name, cfg, cloud, log, _args): with util.ExtendedTemporaryFile(suffix=".sh") as tmpf: try: content = util.shellify(cfg["bootcmd"]) - tmpf.write(content) + tmpf.write(util.encode_text(content)) tmpf.flush() except: util.logexc(log, "Failed to shellify bootcmd") -- cgit v1.2.3 From f62b86bd45c8df78ada32ab4040a639c9d096202 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 11 Feb 2015 01:09:34 +0000 Subject: fix random_seed module --- cloudinit/config/cc_seed_random.py | 16 ++++++++-------- cloudinit/sources/DataSourceAzure.py | 3 ++- cloudinit/util.py | 16 +++++++++------- 3 files changed, 19 insertions(+), 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index bb64b0f5..3288a853 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -22,7 +22,7 @@ import base64 import os -from six import StringIO +from six import BytesIO from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging @@ -34,13 +34,13 @@ LOG = logging.getLogger(__name__) def _decode(data, encoding=None): if not data: - return '' + return b'' if not encoding or encoding.lower() in ['raw']: - return data + return util.encode_text(data) elif encoding.lower() in ['base64', 'b64']: - return util.b64d(data) + return base64.b64decode(data) elif encoding.lower() in ['gzip', 'gz']: - return util.decomp_gzip(data, quiet=False) + return util.decomp_gzip(data, quiet=False, decode=None) else: raise IOError("Unknown random_seed encoding: %s" % (encoding)) @@ -65,9 +65,9 @@ def handle_random_seed_command(command, required, env=None): def handle(name, cfg, cloud, log, _args): mycfg = cfg.get('random_seed', {}) seed_path = mycfg.get('file', '/dev/urandom') - seed_data = mycfg.get('data', '') + seed_data = mycfg.get('data', b'') - seed_buf = StringIO() + seed_buf = BytesIO() if seed_data: seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding'))) @@ -75,7 +75,7 @@ def handle(name, cfg, cloud, log, _args): # openstack meta_data.json metadata = cloud.datasource.metadata if metadata and 'random_seed' in metadata: - seed_buf.write(metadata['random_seed']) + seed_buf.write(util.encode_text(metadata['random_seed'])) seed_data = seed_buf.getvalue() if len(seed_data): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 29ae2c22..c599d50f 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -124,7 +124,8 @@ class DataSourceAzureNet(sources.DataSource): LOG.debug("using files cached in %s", ddir) # azure / hyper-v provides random data here - seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True) + seed = util.load_file("/sys/firmware/acpi/tables/OEM0", + quiet=True, decode=False) if seed: self.metadata['random_seed'] = seed diff --git a/cloudinit/util.py b/cloudinit/util.py index 3a921afe..c998154a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -96,11 +96,10 @@ def b64d(source): # Base64 decode some data, accepting bytes or unicode/str, and returning # str/unicode if the result is utf-8 compatible, otherwise returning bytes. decoded = b64decode(source) - if isinstance(decoded, bytes): - try: - return decoded.decode('utf-8') - except UnicodeDecodeError: - return decoded + try: + return decoded.decode('utf-8') + except UnicodeDecodeError: + return decoded def b64e(source): # Base64 encode some data, accepting bytes or unicode/str, and returning @@ -354,11 +353,14 @@ def clean_filename(fn): return fn -def decomp_gzip(data, quiet=True): +def decomp_gzip(data, quiet=True, decode=True): try: buf = six.BytesIO(encode_text(data)) with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh: - return decode_binary(gh.read()) + if decode: + return decode_binary(gh.read()) + else: + return gh.read() except Exception as e: if quiet: return data -- cgit v1.2.3 From f67d459da3d81f3b4c4c4171eaf5940dbc73ea25 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 10 Feb 2015 20:50:45 -0500 Subject: pep8 --- cloudinit/config/cc_set_passwords.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 8b705d90..0c315361 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -31,7 +31,7 @@ from cloudinit import util from string import ascii_letters, digits # We are removing certain 'painful' letters/numbers -PW_SET = (''.join([x for x in ascii_letters + digits +PW_SET = (''.join([x for x in ascii_letters + digits if x not in 'loLOI01'])) -- cgit v1.2.3 From ceb229043cec98d79aa8e72c6eb5e79f796a96d7 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 11 Feb 2015 12:57:50 -0500 Subject: provide default final message in jinja to avoid WARN in log --- cloudinit/config/cc_final_message.py | 9 ++++++--- doc/examples/cloud-config.txt | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index b24294e4..ad957e12 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -26,9 +26,12 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -# Cheetah formated default message -FINAL_MESSAGE_DEF = ("Cloud-init v. ${version} finished at ${timestamp}." - " Datasource ${datasource}. Up ${uptime} seconds") +# Jinja formated default message +FINAL_MESSAGE_DEF = ( + "## template: jinja\n" + "Cloud-init v. {{version}} finished at {{timestamp}}." + " Datasource {{datasource}}. Up {{uptime}} seconds" +) def handle(_name, cfg, cloud, log, args): diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index ed4eb7fc..1c59c2cf 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -484,7 +484,9 @@ resize_rootfs: True # final_message # default: cloud-init boot finished at $TIMESTAMP. Up $UPTIME seconds # this message is written by cloud-final when the system is finished -# its first boot +# its first boot. +# This message is rendered as if it were a template. If you +# want jinja, you have to start the line with '## template:jinja\n' final_message: "The system is finally up, after $UPTIME seconds" # configure where output will go -- cgit v1.2.3 From 9c224b8bbe5e133fca00d04d070337ffed23bbd9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 13 Feb 2015 16:04:03 -0500 Subject: fix usage of python2 'print' --- cloudinit/config/cc_disk_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d8553167..f899210b 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -618,7 +618,7 @@ def exec_mkpart_gpt(device, layout): [SGDISK_CMD, '-t', '{}:{}'.format(index, partition_type), device]) except Exception: - print "Failed to partition device %s" % (device,) + LOG.warn("Failed to partition device %s" % device) raise -- cgit v1.2.3 From 30868baba637b45b654bd8a23719624a25b2a00b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 09:47:17 -0500 Subject: run emit_upstart only if upstart was init system --- cloudinit/config/cc_emit_upstart.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 6d376184..e1b9a4c2 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -21,11 +21,32 @@ import os from cloudinit.settings import PER_ALWAYS +from cloudinit import log as logging from cloudinit import util frequency = PER_ALWAYS distros = ['ubuntu', 'debian'] +LOG = logging.getLogger(__name__) + + +def is_upstart_system(): + if not os.path.isfile("/sbin/initctl"): + LOG.debug(("Skipping module named %s," + " no /sbin/initctl located"), name) + return False + + myenv = os.environ.copy() + if 'UPSTART_SESSION' in myenv: + del myenv['UPSTART_SESSION'] + check_cmd = ['initctl', 'version'] + try: + (out, err) = util.subp(check_cmd, env=myenv) + return 'upstart' in out + except util.ProcessExecutionError as e: + LOG.debug("'%s' returned '%s', not using upstart", + ' '.join(check_cmd), e.exit_code) + return False def handle(name, _cfg, cloud, log, args): @@ -34,10 +55,11 @@ def handle(name, _cfg, cloud, log, args): # Default to the 'cloud-config' # event for backwards compat. event_names = ['cloud-config'] - if not os.path.isfile("/sbin/initctl"): - log.debug(("Skipping module named %s," - " no /sbin/initctl located"), name) + + if not is_upstart_system(): + log.debug("not upstart system, '%s' disabled") return + cfgpath = cloud.paths.get_ipath_cur("cloud_config") for n in event_names: cmd = ['initctl', 'emit', str(n), 'CLOUD_CFG=%s' % cfgpath] -- cgit v1.2.3 From df975abae42664bbd5fd56436eb2947e2a6f46f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 10:10:11 -0500 Subject: add snappy module --- cloudinit/config/cc_snappy.py | 133 ++++++++++++++++++++++++++++++++++++++++++ config/cloud.cfg | 1 + 2 files changed, 134 insertions(+) create mode 100644 cloudinit/config/cc_snappy.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py new file mode 100644 index 00000000..1588443f --- /dev/null +++ b/cloudinit/config/cc_snappy.py @@ -0,0 +1,133 @@ +# vi: ts=4 expandtab +# + +from cloudinit import log as logging +from cloudinit import templater +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +import glob +import os + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" + +CI_SNAPPY_CFG = { + 'env_file_path': SNAPPY_ENV_PATH, + 'packages': [], + 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'ssh_enabled': False +} + +""" +snappy: + ssh_enabled: True + packages: + - etcd + - {'name': 'pkg1', 'config': "wark"} +""" + + +def flatten(data, fill=None, tok="_", prefix='', recurse=True): + if fill is None: + fill = {} + for key, val in data.items(): + key = key.replace("-", "_") + if isinstance(val, dict) and recurse: + flatten(val, fill, tok=tok, prefix=prefix + key + tok, + recurse=recurse) + elif isinstance(key, str): + fill[prefix + key] = val + return fill + + +def render2env(data, tok="_", prefix=''): + flat = flatten(data, tok=tok, prefix=prefix) + ret = ["%s='%s'" % (key, val) for key, val in flat.items()] + return '\n'.join(ret) + '\n' + + +def install_package(pkg_name, config=None): + cmd = ["snappy", "install"] + if config: + if os.path.isfile(config): + cmd.append("--config-file=" + config) + else: + cmd.append("--config=" + config) + cmd.append(pkg_name) + util.subp(cmd) + + +def install_packages(package_dir, packages): + local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) + LOG.debug("installing local packages %s" % local_pkgs) + if local_pkgs: + for pkg in local_pkgs: + cfg = pkg.replace(".click", ".config") + if not os.path.isfile(cfg): + cfg = None + install_package(pkg, config=cfg) + + LOG.debug("installing click packages") + if packages: + for pkg in packages: + if not pkg: + continue + if isinstance(pkg, str): + name = pkg + config = None + elif pkg: + name = pkg.get('name', pkg) + config = pkg.get('config') + install_package(pkg_name=name, config=config) + + +def disable_enable_ssh(enabled): + LOG.debug("setting enablement of ssh to: %s", enabled) + # do something here that would enable or disable + not_to_be_run = "/etc/ssh/sshd_not_to_be_run" + if enabled: + util.del_file(not_to_be_run) + # this is an indempotent operation + util.subp(["systemctl", "start", "ssh"]) + else: + # this is an indempotent operation + util.subp(["systemctl", "stop", "ssh"]) + util.write_file(not_to_be_run, "cloud-init\n") + + +def handle(name, cfg, cloud, log, args): + mycfg = cfg.get('snappy', {'ssh_enabled': False}) + + if not mycfg: + LOG.debug("%s: no top level found", name) + return + + # take out of 'cfg' the cfg keys that cloud-init uses, so + # mycfg has only content external to cloud-init. + ci_cfg = CI_SNAPPY_CFG.copy() + for i in ci_cfg: + if i in mycfg: + ci_cfg[i] = mycfg[i] + del mycfg[i] + + # render the flattened environment variable style file to a path + # this was useful for systemd config environment files. given: + # snappy: + # foo: + # bar: wark + # cfg1: + # key1: value + # you get the following in env_file_path. + # foo_bar=wark + # foo_cfg1_key1=value + contents = render2env(mycfg) + header = '# for internal use only, not a guaranteed interface\n' + util.write_file(ci_cfg['env_file_path'], header + render2env(mycfg)) + + install_packages(ci_cfg['packages_dir'], + ci_cfg['packages']) + + disable_enable_ssh(ci_cfg.get('ssh_enabled', False)) diff --git a/config/cloud.cfg b/config/cloud.cfg index 200050d3..e96e1781 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -48,6 +48,7 @@ cloud_config_modules: - ssh-import-id - locale - set-passwords + - snappy - grub-dpkg - apt-pipelining - apt-configure -- cgit v1.2.3 From 2db45b26d2e5412aa55b33ec924afce7c0dbd12c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 14:49:44 -0500 Subject: locale: make able to be turned off --- cloudinit/config/cc_locale.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 6feaae9d..bbe5fcae 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -27,9 +27,9 @@ def handle(name, cfg, cloud, log, args): else: locale = util.get_cfg_option_str(cfg, "locale", cloud.get_locale()) - if not locale: - log.debug(("Skipping module named %s, " - "no 'locale' configuration found"), name) + if util.is_false(locale): + log.debug("Skipping module named %s, disabled by config: %s", + name, locale) return log.debug("Setting locale to %s", locale) -- cgit v1.2.3 From 0ce85f53c15bce21e65a419fe71127c6d94064aa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 14:49:59 -0500 Subject: grub-dpkg: allow to be disabled --- cloudinit/config/cc_grub_dpkg.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index e3219e81..456597af 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -25,15 +25,20 @@ from cloudinit import util distros = ['ubuntu', 'debian'] -def handle(_name, cfg, _cloud, log, _args): - idevs = None - idevs_empty = None +def handle(name, cfg, _cloud, log, _args): - if "grub-dpkg" in cfg: - idevs = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices", None) - idevs_empty = util.get_cfg_option_str(cfg["grub-dpkg"], - "grub-pc/install_devices_empty", None) + mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {})) + if not mycfg: + mycfg = {} + + enabled = mycfg.get('enabled', True) + if util.is_false(enabled): + log.debug("%s disabled by config grub_dpkg/enabled=%s", name, enabled) + return + + idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) + idevs_empty = util.get_cfg_option_str(mycfg, + "grub-pc/install_devices_empty", None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or (os.path.exists("/dev/xvda1") -- cgit v1.2.3 From e7cce1a06429813b8d2acc87e6609671d39a3254 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 4 Mar 2015 15:51:46 -0500 Subject: apt_configure: allow disabling --- cloudinit/config/cc_apt_configure.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index de72903f..2c51d116 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -51,6 +51,10 @@ EXPORT_GPG_KEYID = """ def handle(name, cfg, cloud, log, _args): + if util.is_false(cfg.get('apt_configure_enabled', True)): + log.debug("Skipping module named %s, disabled by config.", name) + return + release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) if not mirrors or "primary" not in mirrors: -- cgit v1.2.3 From 46da1b83fba8d1e70dc58dbbf18697216b1eb1e3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Mar 2015 11:18:45 -0500 Subject: fixes bug: https://launchpad.net/bugs/1428495 snappy: disable by default this does 2 things actually a.) disables snappy by default, and adds checks to filesystem to enable it b.) removes the 'render2env' that was mostly spike code. --- cloudinit/config/cc_snappy.py | 74 ++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 47 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 1588443f..32fbc9f6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -14,15 +14,16 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" -CI_SNAPPY_CFG = { - 'env_file_path': SNAPPY_ENV_PATH, +BUILTIN_CFG = { 'packages': [], 'packages_dir': '/writable/user-data/cloud-init/click_packages', - 'ssh_enabled': False + 'ssh_enabled': False, + 'system_snappy': "auto" } """ snappy: + system_snappy: auto ssh_enabled: True packages: - etcd @@ -30,25 +31,6 @@ snappy: """ -def flatten(data, fill=None, tok="_", prefix='', recurse=True): - if fill is None: - fill = {} - for key, val in data.items(): - key = key.replace("-", "_") - if isinstance(val, dict) and recurse: - flatten(val, fill, tok=tok, prefix=prefix + key + tok, - recurse=recurse) - elif isinstance(key, str): - fill[prefix + key] = val - return fill - - -def render2env(data, tok="_", prefix=''): - flat = flatten(data, tok=tok, prefix=prefix) - ret = ["%s='%s'" % (key, val) for key, val in flat.items()] - return '\n'.join(ret) + '\n' - - def install_package(pkg_name, config=None): cmd = ["snappy", "install"] if config: @@ -98,34 +80,32 @@ def disable_enable_ssh(enabled): util.write_file(not_to_be_run, "cloud-init\n") -def handle(name, cfg, cloud, log, args): - mycfg = cfg.get('snappy', {'ssh_enabled': False}) +def system_is_snappy(): + # channel.ini is configparser loadable. + # snappy will move to using /etc/system-image/config.d/*.ini + # this is certainly not a perfect test, but good enough for now. + content = util.load_file("/etc/system-image/channel.ini") + if 'ubuntu-core' in content.lower(): + return True + if os.path.isdir("/etc/system-image/config.d/"): + return True + return False + - if not mycfg: - LOG.debug("%s: no top level found", name) +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snappy') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([BUILTIN_CFG, cfgin]) + + sys_snappy = mycfg.get("system_snappy", "auto") + if util.is_false(sys_snappy): + LOG.debug("%s: System is not snappy. disabling", name) return - # take out of 'cfg' the cfg keys that cloud-init uses, so - # mycfg has only content external to cloud-init. - ci_cfg = CI_SNAPPY_CFG.copy() - for i in ci_cfg: - if i in mycfg: - ci_cfg[i] = mycfg[i] - del mycfg[i] - - # render the flattened environment variable style file to a path - # this was useful for systemd config environment files. given: - # snappy: - # foo: - # bar: wark - # cfg1: - # key1: value - # you get the following in env_file_path. - # foo_bar=wark - # foo_cfg1_key1=value - contents = render2env(mycfg) - header = '# for internal use only, not a guaranteed interface\n' - util.write_file(ci_cfg['env_file_path'], header + render2env(mycfg)) + if sys_snappy.lower() == "auto" and not(system_is_snappy()): + LOG.debug("%s: 'auto' mode, and system not snappy", name) + return install_packages(ci_cfg['packages_dir'], ci_cfg['packages']) -- cgit v1.2.3 From d05f1b00e2498343c03ba2de543990fffde8a02f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Mar 2015 12:26:26 -0500 Subject: do not raise exception on non-existant channel.ini file --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 32fbc9f6..8d73dca3 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -84,7 +84,7 @@ def system_is_snappy(): # channel.ini is configparser loadable. # snappy will move to using /etc/system-image/config.d/*.ini # this is certainly not a perfect test, but good enough for now. - content = util.load_file("/etc/system-image/channel.ini") + content = util.load_file("/etc/system-image/channel.ini", quiet=True) if 'ubuntu-core' in content.lower(): return True if os.path.isdir("/etc/system-image/config.d/"): -- cgit v1.2.3 From c501a37e94b9601740fd7b3dcbcc4cce9136d7f4 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 5 Mar 2015 13:16:28 -0500 Subject: fixes from testing --- cloudinit/config/cc_snappy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 8d73dca3..133336d4 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -96,9 +96,9 @@ def handle(name, cfg, cloud, log, args): cfgin = cfg.get('snappy') if not cfgin: cfgin = {} - mycfg = util.mergemanydict([BUILTIN_CFG, cfgin]) + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - sys_snappy = mycfg.get("system_snappy", "auto") + sys_snappy = str(mycfg.get("system_snappy", "auto")) if util.is_false(sys_snappy): LOG.debug("%s: System is not snappy. disabling", name) return @@ -107,7 +107,7 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return - install_packages(ci_cfg['packages_dir'], - ci_cfg['packages']) + install_packages(mycfg['packages_dir'], + mycfg['packages']) - disable_enable_ssh(ci_cfg.get('ssh_enabled', False)) + disable_enable_ssh(mycfg.get('ssh_enabled', False)) -- cgit v1.2.3 From 516af9ba927dd9b4dcc3461f8a8bb6883c61c036 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 16 Mar 2015 13:20:26 -0400 Subject: emit_upstart: fix use of undeclared variable --- cloudinit/config/cc_emit_upstart.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index e1b9a4c2..86ae97ab 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -32,8 +32,7 @@ LOG = logging.getLogger(__name__) def is_upstart_system(): if not os.path.isfile("/sbin/initctl"): - LOG.debug(("Skipping module named %s," - " no /sbin/initctl located"), name) + LOG.debug("no /sbin/initctl located") return False myenv = os.environ.copy() -- cgit v1.2.3 From 0b9e0444f5092e647a3fa55887d96ffaf3d23c06 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Wed, 18 Mar 2015 13:33:12 +0000 Subject: Update is_disk_used for changed enumerate_disk output. Fixes Launchpad bug #1311463. --- cloudinit/config/cc_disk_setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index f899210b..e2ce6db4 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -304,8 +304,7 @@ def is_disk_used(device): # If the child count is higher 1, then there are child nodes # such as partition or device mapper nodes - use_count = [x for x in enumerate_disk(device)] - if len(use_count.splitlines()) > 1: + if len(list(enumerate_disk(device))) > 1: return True # If we see a file system, then its used -- cgit v1.2.3 From a373e1097f6be460914e6cbbc897c6aa8e4aaefe Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 20:39:25 -0400 Subject: commit work in progress. tests pass. --- cloudinit/config/cc_snappy.py | 159 +++++++++++++++----- .../unittests/test_handler/test_handler_snappy.py | 163 +++++++++++++++++++++ 2 files changed, 285 insertions(+), 37 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_snappy.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 133336d4..bef8c170 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -7,18 +7,21 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE import glob +import six +import tempfile import os LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" +SNAPPY_CMD = "snappy" BUILTIN_CFG = { 'packages': [], 'packages_dir': '/writable/user-data/cloud-init/click_packages', 'ssh_enabled': False, - 'system_snappy': "auto" + 'system_snappy': "auto", + 'configs': {}, } """ @@ -27,43 +30,111 @@ snappy: ssh_enabled: True packages: - etcd - - {'name': 'pkg1', 'config': "wark"} + - pkg2 + configs: + pkgname: config-blob + pkgname2: config-blob """ -def install_package(pkg_name, config=None): - cmd = ["snappy", "install"] - if config: - if os.path.isfile(config): - cmd.append("--config-file=" + config) +def get_fs_package_ops(fspath): + if not fspath: + return [] + ops = [] + for snapfile in glob.glob(os.path.sep.join([fspath, '*.snap'])): + cfg = snapfile.rpartition(".")[0] + ".config" + name = os.path.basename(snapfile).rpartition(".")[0] + if not os.path.isfile(cfg): + cfg = None + ops.append(makeop('install', name, config=None, + path=snapfile, cfgfile=cfg)) + return ops + + +def makeop(op, name, config=None, path=None, cfgfile=None): + return({'op': op, 'name': name, 'config': config, 'path': path, + 'cfgfile': cfgfile}) + + +def get_package_ops(packages, configs, installed=None, fspath=None): + # get the install an config operations that should be done + if installed is None: + installed = read_installed_packages() + + if not packages: + packages = [] + if not configs: + configs = {} + + ops = [] + ops += get_fs_package_ops(fspath) + + for name in packages: + ops.append(makeop('install', name, configs.get('name'))) + + to_install = [f['name'] for f in ops] + + for name in configs: + if name in installed and name not in to_install: + ops.append(makeop('config', name, config=configs[name])) + + # prefer config entries to filepath entries + for op in ops: + name = op['name'] + if name in configs and op['op'] == 'install' and 'cfgfile' in op: + LOG.debug("preferring configs[%s] over '%s'", name, op['cfgfile']) + op['cfgfile'] = None + op['config'] = configs[op['name']] + + return ops + + +def render_snap_op(op, name, path=None, cfgfile=None, config=None): + if op not in ('install', 'config'): + raise ValueError("cannot render op '%s'" % op) + + try: + cfg_tmpf = None + if config is not None: + if isinstance(config, six.binary_type): + cfg_bytes = config + elif isinstance(config, six.text_type): + cfg_bytes = config_data.encode() + else: + cfg_bytes = yaml.safe_dump(config).encode() + + (fd, cfg_tmpf) = tempfile.mkstemp() + os.write(fd, config_data) + os.close(fd) + cfgfile = cfg_tmpf + + cmd = [SNAPPY_CMD, op] + if op == 'install' and cfgfile: + cmd.append('--config=' + cfgfile) + elif op == 'config': + cmd.append(cfgfile) + + util.subp(cmd) + + finally: + if tmpfile: + os.unlink(tmpfile) + + +def read_installed_packages(): + return [p[0] for p in read_pkg_data()] + + +def read_pkg_data(): + out, err = util.subp([SNAPPY_CMD, "list"]) + for line in out.splitlines()[1:]: + toks = line.split(sep=None, maxsplit=3) + if len(toks) == 3: + (name, date, version) = toks + dev = None else: - cmd.append("--config=" + config) - cmd.append(pkg_name) - util.subp(cmd) - - -def install_packages(package_dir, packages): - local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) - LOG.debug("installing local packages %s" % local_pkgs) - if local_pkgs: - for pkg in local_pkgs: - cfg = pkg.replace(".click", ".config") - if not os.path.isfile(cfg): - cfg = None - install_package(pkg, config=cfg) - - LOG.debug("installing click packages") - if packages: - for pkg in packages: - if not pkg: - continue - if isinstance(pkg, str): - name = pkg - config = None - elif pkg: - name = pkg.get('name', pkg) - config = pkg.get('config') - install_package(pkg_name=name, config=config) + (name, date, version, dev) = toks + pkgs.append((name, date, version, dev,)) def disable_enable_ssh(enabled): @@ -107,7 +178,21 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return - install_packages(mycfg['packages_dir'], - mycfg['packages']) + pkg_ops = get_package_ops(packages=mycfg['packages'], + configs=mycfg['configs'], + fspath=mycfg['packages_dir']) + + fails = [] + for pkg_op in pkg_ops: + try: + render_snap_op(op=pkg_op['op'], name=pkg_op['name'], + cfgfile=pkg_op['cfgfile'], config=pkg_op['config']) + except Exception as e: + fails.append((pkg_op, e,)) + LOG.warn("'%s' failed for '%s': %s", + pkg_op['op'], pkg_op['name'], e) disable_enable_ssh(mycfg.get('ssh_enabled', False)) + + if fails: + raise Exception("failed to install/configure snaps") diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py new file mode 100644 index 00000000..6b6d3584 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -0,0 +1,163 @@ +from cloudinit.config.cc_snappy import (makeop, get_package_ops) +from cloudinit import util +from .. import helpers as t_help + +import os +import tempfile + +class TestInstallPackages(t_help.TestCase): + def setUp(self): + super(TestInstallPackages, self).setUp() + self.unapply = [] + + # by default 'which' has nothing in its path + self.apply_patches([(util, 'subp', self._subp)]) + self.subp_called = [] + self.snapcmds = [] + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def _subp(self, *args, **kwargs): + # supports subp calling with cmd as args or kwargs + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + snap_cmds = [] + args = kwargs['args'] + if args[0:2] == ['snappy', 'config']: + if args[3] == "-": + config = kwargs.get('data', '') + else: + with open(args[3], "rb") as fp: + config = fp.read() + snap_cmds.append(('config', args[2], config,)) + elif args[0:2] == ['snappy', 'install']: + # basically parse the snappy command and add + # to snap_installs a tuple (pkg, config) + config = None + pkg = None + for arg in args[2:]: + if arg.startswith("--config="): + cfgfile = arg.partition("=")[2] + if cfgfile == "-": + config = kwargs.get('data', '') + elif cfgfile: + with open(cfgfile, "rb") as fp: + config = fp.read() + elif not pkg and not arg.startswith("-"): + pkg = os.path.basename(arg) + self.snap_installs.append(('install', pkg, config,)) + + def test_package_ops_1(self): + ret = get_package_ops( + packages=['pkg1', 'pkg2', 'pkg3'], + configs={'pkg2': b'mycfg2'}, installed=[]) + self.assertEqual(ret, + [makeop('install', 'pkg1', None, None), + makeop('install', 'pkg2', b'mycfg2', None), + makeop('install', 'pkg3', None, None)]) + + def test_package_ops_config_only(self): + ret = get_package_ops( + packages=None, + configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) + self.assertEqual(ret, + [makeop('config', 'pkg2', b'mycfg2')]) + + def test_package_ops_install_and_config(self): + ret = get_package_ops( + packages=['pkg3', 'pkg2'], + configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, + installed=['xinstalled']) + self.assertEqual(ret, + [makeop('install', 'pkg3'), + makeop('install', 'pkg2', b'mycfg2'), + makeop('config', 'xinstalled', b'xcfg')]) + + def test_package_ops_with_file(self): + t_help.populate_dir(self.tmp, + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", + "snapf2.snap": b"foo2", "foo.bar": "ignored"}) + ret = get_package_ops( + packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) + self.assertEqual(ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config"), + makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), + makeop('install', 'pkg1')]) + + +def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): + if cfgfile: + cfgfile = os.path.sep.join([tmpd, cfgfile]) + if path: + path = os.path.sep.join([tmpd, path]) + return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) + +# def test_local_snaps_no_config(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": b"foo", "snap2.snap": b"foo", "foosnap.txt": b"foo"}) +# cc_snappy.install_packages(self.tmp, None) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", None), ("snap2.snap", None)]) +# +# def test_local_snaps_mixed_config(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": b"foo", "snap2.snap": b"snap2", +# "snap1.config": b"snap1config"}) +# cc_snappy.install_packages(self.tmp, None) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", b"snap1config"), ("snap2.snap", None)]) +# +# def test_local_snaps_all_config(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": "foo", "snap1.config": b"snap1config", +# "snap2.snap": "snap2", "snap2.config": b"snap2config"}) +# cc_snappy.install_packages(self.tmp, None) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", b"snap1config"), ("snap2.snap", b"snap2config")]) +# +# def test_local_snaps_and_packages(self): +# t_help.populate_dir(self.tmp, +# {"snap1.snap": "foo", "snap1.config": b"snap1config"}) +# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) +# self.assertEqual(self.snap_installs, +# [("snap1.snap", b"snap1config"), ("snap-in-store", None)]) +# +# def test_packages_no_config(self): +# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) +# self.assertEqual(self.snap_installs, +# [("snap-in-store", None)]) +# +# def test_packages_mixed_config(self): +# cc_snappy.install_packages(self.tmp, +# ["snap-in-store", +# {'name': 'snap2-in-store', 'config': b"foo"}]) +# self.assertEqual(self.snap_installs, +# [("snap-in-store", None), ("snap2-in-store", b"foo")]) +# +# def test_packages_all_config(self): +# cc_snappy.install_packages(self.tmp, +# [{'name': 'snap1-in-store', 'config': b"boo"}, +# {'name': 'snap2-in-store', 'config': b"wark"}]) +# self.assertEqual(self.snap_installs, +# [("snap1-in-store", b"boo"), ("snap2-in-store", b"wark")]) +# +# + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret + -- cgit v1.2.3 From bd7165dd67338f742f999fb2c53ec5f67fc66477 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 21:14:17 -0400 Subject: start of snap_op tests --- cloudinit/config/cc_snappy.py | 10 ++++-- .../unittests/test_handler/test_handler_snappy.py | 40 +++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bef8c170..cf441c92 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -114,11 +114,17 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): elif op == 'config': cmd.append(cfgfile) + if op == 'install': + if path: + cmd.append(path) + else: + cmd.append(name) + util.subp(cmd) finally: - if tmpfile: - os.unlink(tmpfile) + if cfg_tmpf: + os.unlink(cfg_tmpf) def read_installed_packages(): diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 6b6d3584..7dc77970 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -1,4 +1,5 @@ -from cloudinit.config.cc_snappy import (makeop, get_package_ops) +from cloudinit.config.cc_snappy import ( + makeop, get_package_ops, render_snap_op) from cloudinit import util from .. import helpers as t_help @@ -36,7 +37,7 @@ class TestInstallPackages(t_help.TestCase): else: with open(args[3], "rb") as fp: config = fp.read() - snap_cmds.append(('config', args[2], config,)) + self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: # basically parse the snappy command and add # to snap_installs a tuple (pkg, config) @@ -51,8 +52,8 @@ class TestInstallPackages(t_help.TestCase): with open(cfgfile, "rb") as fp: config = fp.read() elif not pkg and not arg.startswith("-"): - pkg = os.path.basename(arg) - self.snap_installs.append(('install', pkg, config,)) + pkg = arg + self.snapcmds.append(['install', pkg, config]) def test_package_ops_1(self): ret = get_package_ops( @@ -92,6 +93,37 @@ class TestInstallPackages(t_help.TestCase): makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), makeop('install', 'pkg1')]) + #def render_snap_op(op, name, path=None, cfgfile=None, config=None): + def test_render_op_localsnap(self): + t_help.populate_dir(self.tmp, {"snapf1.snap": b"foo1"}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap') + render_snap_op(**op) + self.assertEqual(self.snapcmds, + [['install', op['path'], None]]) + + def test_render_op_localsnap_localconfig(self): + t_help.populate_dir(self.tmp, + {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap', cfgfile='snapf1.config') + render_snap_op(**op) + self.assertEqual(self.snapcmds, + [['install', op['path'], b'snapf1cfg']]) + + def test_render_op_localsnap_config(self): + pass + + def test_render_op_snap(self): + pass + + def test_render_op_snap_config(self): + pass + + def test_render_op_config(self): + pass + + def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: -- cgit v1.2.3 From df43c6bd3726c9a34b9f8ff4bbf75957aa751011 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 21:55:26 -0400 Subject: pep8, and some more tests --- cloudinit/config/cc_snappy.py | 13 +- .../unittests/test_handler/test_handler_snappy.py | 131 ++++++++------------- 2 files changed, 57 insertions(+), 87 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index cf441c92..c926ae0a 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -99,26 +99,25 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): if isinstance(config, six.binary_type): cfg_bytes = config elif isinstance(config, six.text_type): - cfg_bytes = config_data.encode() + cfg_bytes = config.encode() else: cfg_bytes = yaml.safe_dump(config).encode() (fd, cfg_tmpf) = tempfile.mkstemp() - os.write(fd, config_data) + os.write(fd, cfg_bytes) os.close(fd) cfgfile = cfg_tmpf cmd = [SNAPPY_CMD, op] - if op == 'install' and cfgfile: - cmd.append('--config=' + cfgfile) - elif op == 'config': - cmd.append(cfgfile) - if op == 'install': + if cfgfile: + cmd.append('--config=' + cfgfile) if path: cmd.append(path) else: cmd.append(name) + elif op == 'config': + cmd += [name, cfgfile] util.subp(cmd) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 7dc77970..8759a07d 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -4,8 +4,10 @@ from cloudinit import util from .. import helpers as t_help import os +import shutil import tempfile + class TestInstallPackages(t_help.TestCase): def setUp(self): super(TestInstallPackages, self).setUp() @@ -15,15 +17,19 @@ class TestInstallPackages(t_help.TestCase): self.apply_patches([(util, 'subp', self._subp)]) self.subp_called = [] self.snapcmds = [] - self.tmp = tempfile.mkdtemp() + self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages") def tearDown(self): apply_patches([i for i in reversed(self.unapply)]) + shutil.rmtree(self.tmp) def apply_patches(self, patches): ret = apply_patches(patches) self.unapply += ret + def populate_tmp(self, files): + return t_help.populate_dir(self.tmp, files) + def _subp(self, *args, **kwargs): # supports subp calling with cmd as args or kwargs if 'args' not in kwargs: @@ -31,6 +37,8 @@ class TestInstallPackages(t_help.TestCase): self.subp_called.append(kwargs) snap_cmds = [] args = kwargs['args'] + # here we basically parse the snappy command invoked + # and append to snapcmds a list of (mode, pkg, config) if args[0:2] == ['snappy', 'config']: if args[3] == "-": config = kwargs.get('data', '') @@ -39,8 +47,6 @@ class TestInstallPackages(t_help.TestCase): config = fp.read() self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: - # basically parse the snappy command and add - # to snap_installs a tuple (pkg, config) config = None pkg = None for arg in args[2:]: @@ -59,72 +65,88 @@ class TestInstallPackages(t_help.TestCase): ret = get_package_ops( packages=['pkg1', 'pkg2', 'pkg3'], configs={'pkg2': b'mycfg2'}, installed=[]) - self.assertEqual(ret, - [makeop('install', 'pkg1', None, None), - makeop('install', 'pkg2', b'mycfg2', None), - makeop('install', 'pkg3', None, None)]) + self.assertEqual( + ret, [makeop('install', 'pkg1', None, None), + makeop('install', 'pkg2', b'mycfg2', None), + makeop('install', 'pkg3', None, None)]) def test_package_ops_config_only(self): ret = get_package_ops( packages=None, configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) - self.assertEqual(ret, - [makeop('config', 'pkg2', b'mycfg2')]) + self.assertEqual( + ret, [makeop('config', 'pkg2', b'mycfg2')]) def test_package_ops_install_and_config(self): ret = get_package_ops( packages=['pkg3', 'pkg2'], configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, installed=['xinstalled']) - self.assertEqual(ret, - [makeop('install', 'pkg3'), - makeop('install', 'pkg2', b'mycfg2'), - makeop('config', 'xinstalled', b'xcfg')]) + self.assertEqual( + ret, [makeop('install', 'pkg3'), + makeop('install', 'pkg2', b'mycfg2'), + makeop('config', 'xinstalled', b'xcfg')]) def test_package_ops_with_file(self): - t_help.populate_dir(self.tmp, + self.populate_tmp( {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", "snapf2.snap": b"foo2", "foo.bar": "ignored"}) ret = get_package_ops( packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) - self.assertEqual(ret, + self.assertEqual( + ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", cfgfile="snapf1.config"), makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), makeop('install', 'pkg1')]) - #def render_snap_op(op, name, path=None, cfgfile=None, config=None): + def test_package_ops_config_overrides_file(self): + # config data overrides local file .config + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"}) + ret = get_package_ops( + packages=[], configs={'snapf1': 'snapf1cfg-config'}, + installed=[], fspath=self.tmp) + self.assertEqual( + ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', + path="snapf1.snap", config="snapf1cfg-config")]) + def test_render_op_localsnap(self): - t_help.populate_dir(self.tmp, {"snapf1.snap": b"foo1"}) + self.populate_tmp({"snapf1.snap": b"foo1"}) op = makeop_tmpd(self.tmp, 'install', 'snapf1', path='snapf1.snap') render_snap_op(**op) - self.assertEqual(self.snapcmds, - [['install', op['path'], None]]) + self.assertEqual( + self.snapcmds, [['install', op['path'], None]]) def test_render_op_localsnap_localconfig(self): - t_help.populate_dir(self.tmp, + self.populate_tmp( {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) op = makeop_tmpd(self.tmp, 'install', 'snapf1', path='snapf1.snap', cfgfile='snapf1.config') render_snap_op(**op) - self.assertEqual(self.snapcmds, - [['install', op['path'], b'snapf1cfg']]) - - def test_render_op_localsnap_config(self): - pass + self.assertEqual( + self.snapcmds, [['install', op['path'], b'snapf1cfg']]) def test_render_op_snap(self): - pass + op = makeop('install', 'snapf1') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', 'snapf1', None]]) def test_render_op_snap_config(self): - pass + op = makeop('install', 'snapf1', config=b'myconfig') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', 'snapf1', b'myconfig']]) def test_render_op_config(self): - pass + op = makeop('config', 'snapf1', config=b'myconfig') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', b'myconfig']]) - def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: cfgfile = os.path.sep.join([tmpd, cfgfile]) @@ -132,56 +154,6 @@ def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): path = os.path.sep.join([tmpd, path]) return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) -# def test_local_snaps_no_config(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": b"foo", "snap2.snap": b"foo", "foosnap.txt": b"foo"}) -# cc_snappy.install_packages(self.tmp, None) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", None), ("snap2.snap", None)]) -# -# def test_local_snaps_mixed_config(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": b"foo", "snap2.snap": b"snap2", -# "snap1.config": b"snap1config"}) -# cc_snappy.install_packages(self.tmp, None) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", b"snap1config"), ("snap2.snap", None)]) -# -# def test_local_snaps_all_config(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": "foo", "snap1.config": b"snap1config", -# "snap2.snap": "snap2", "snap2.config": b"snap2config"}) -# cc_snappy.install_packages(self.tmp, None) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", b"snap1config"), ("snap2.snap", b"snap2config")]) -# -# def test_local_snaps_and_packages(self): -# t_help.populate_dir(self.tmp, -# {"snap1.snap": "foo", "snap1.config": b"snap1config"}) -# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) -# self.assertEqual(self.snap_installs, -# [("snap1.snap", b"snap1config"), ("snap-in-store", None)]) -# -# def test_packages_no_config(self): -# cc_snappy.install_packages(self.tmp, ["snap-in-store"]) -# self.assertEqual(self.snap_installs, -# [("snap-in-store", None)]) -# -# def test_packages_mixed_config(self): -# cc_snappy.install_packages(self.tmp, -# ["snap-in-store", -# {'name': 'snap2-in-store', 'config': b"foo"}]) -# self.assertEqual(self.snap_installs, -# [("snap-in-store", None), ("snap2-in-store", b"foo")]) -# -# def test_packages_all_config(self): -# cc_snappy.install_packages(self.tmp, -# [{'name': 'snap1-in-store', 'config': b"boo"}, -# {'name': 'snap2-in-store', 'config': b"wark"}]) -# self.assertEqual(self.snap_installs, -# [("snap1-in-store", b"boo"), ("snap2-in-store", b"wark")]) -# -# def apply_patches(patches): ret = [] @@ -192,4 +164,3 @@ def apply_patches(patches): setattr(ref, name, replace) ret.append((ref, name, orig)) return ret - -- cgit v1.2.3 From 973c8b05358fe6ad1ce7adb25cb743ef4d38d792 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 21:55:45 -0400 Subject: pep8 --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index c926ae0a..d1447fe5 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,7 +47,7 @@ def get_fs_package_ops(fspath): if not os.path.isfile(cfg): cfg = None ops.append(makeop('install', name, config=None, - path=snapfile, cfgfile=cfg)) + path=snapfile, cfgfile=cfg)) return ops -- cgit v1.2.3 From 4c341a87d4b0804565e74e6335a0293dab6c0c7b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:10:01 -0400 Subject: add tests for data types --- cloudinit/config/cc_snappy.py | 2 +- .../unittests/test_handler/test_handler_snappy.py | 35 +++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index d1447fe5..bd928e54 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -101,7 +101,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): elif isinstance(config, six.text_type): cfg_bytes = config.encode() else: - cfg_bytes = yaml.safe_dump(config).encode() + cfg_bytes = util.yaml_dumps(config) (fd, cfg_tmpf) = tempfile.mkstemp() os.write(fd, cfg_bytes) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 8759a07d..81d891d2 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -140,12 +140,45 @@ class TestInstallPackages(t_help.TestCase): self.assertEqual( self.snapcmds, [['install', 'snapf1', b'myconfig']]) - def test_render_op_config(self): + def test_render_op_config_bytes(self): op = makeop('config', 'snapf1', config=b'myconfig') render_snap_op(**op) self.assertEqual( self.snapcmds, [['config', 'snapf1', b'myconfig']]) + def test_render_op_config_string(self): + mycfg = 'myconfig: foo\nhisconfig: bar\n' + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', mycfg.encode()]]) + + def test_render_op_config_dict(self): + # config entry for package can be a dict, not a string blob + mycfg = {'foo': 'bar'} + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + # snapcmds is a list of 3-entry lists. data_found will be the + # blob of data in the file in 'snappy install --config=' + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, util.load_yaml(data_found)) + + def test_render_op_config_list(self): + # config entry for package can be a list, not a string blob + mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(list,))) + + def test_render_op_config_int(self): + # config entry for package can be a list, not a string blob + mycfg = 1 + op = makeop('config', 'snapf1', config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(int,))) + def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: -- cgit v1.2.3 From 5e012b1e5f51f82e503a760c8c9c0e2c66aedfee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:16:13 -0400 Subject: prefer snappy-go to snappy --- cloudinit/config/cc_snappy.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bd928e54..dbdc402c 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -168,6 +168,14 @@ def system_is_snappy(): return False +def set_snappy_command(): + if util.which("snappy-go"): + SNAPPY_COMMAND = "snappy-go" + else: + SNAPPY_COMMAND = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_COMMAND) + + def handle(name, cfg, cloud, log, args): cfgin = cfg.get('snappy') if not cfgin: @@ -187,11 +195,12 @@ def handle(name, cfg, cloud, log, args): configs=mycfg['configs'], fspath=mycfg['packages_dir']) + set_snappy_command() + fails = [] for pkg_op in pkg_ops: try: - render_snap_op(op=pkg_op['op'], name=pkg_op['name'], - cfgfile=pkg_op['cfgfile'], config=pkg_op['config']) + render_snap_op(**pkg_op) except Exception as e: fails.append((pkg_op, e,)) LOG.warn("'%s' failed for '%s': %s", -- cgit v1.2.3 From d0eacf97c72b2613a3f1ce179e284d5aa98744dc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:18:16 -0400 Subject: encode needed for yaml_dumps --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index dbdc402c..adb25bc2 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -101,7 +101,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): elif isinstance(config, six.text_type): cfg_bytes = config.encode() else: - cfg_bytes = util.yaml_dumps(config) + cfg_bytes = util.yaml_dumps(config).encode() (fd, cfg_tmpf) = tempfile.mkstemp() os.write(fd, cfg_bytes) -- cgit v1.2.3 From ef4a19658f354f1cb52b59c093d38d8448e26a70 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:42:26 -0400 Subject: rad_pkg_data: return data, fix undefined variable --- cloudinit/config/cc_snappy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index adb25bc2..61c70f03 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -132,6 +132,7 @@ def read_installed_packages(): def read_pkg_data(): out, err = util.subp([SNAPPY_CMD, "list"]) + pkg_data = [] for line in out.splitlines()[1:]: toks = line.split(sep=None, maxsplit=3) if len(toks) == 3: @@ -139,7 +140,8 @@ def read_pkg_data(): dev = None else: (name, date, version, dev) = toks - pkgs.append((name, date, version, dev,)) + pkg_data.append((name, date, version, dev,)) + return pkg_data def disable_enable_ssh(enabled): -- cgit v1.2.3 From f32a0c32081a4c38b9738bd65a2efc35f26ee983 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:51:59 -0400 Subject: improve doc, change 'click_packages' path to be 'snaps' --- cloudinit/config/cc_snappy.py | 45 ++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 61c70f03..09a8d239 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -1,5 +1,36 @@ # vi: ts=4 expandtab # +""" +snappy modules allows configuration of snappy. +Example config: + #cloud-config + snappy: + system_snappy: auto + ssh_enabled: False + packages: [etcd, pkg2] + configs: + pkgname: pkgname-config-blob + pkg2: config-blob + packages_dir: '/writable/user-data/cloud-init/snaps' + + - ssh_enabled: + This defaults to 'False'. Set to a non-false value to enable ssh service + - snap installation and config + The above would install 'etcd', and then install 'pkg2' with a + '--config=' argument where 'file' as 'config-blob' inside it. + If 'pkgname' is installed already, then 'snappy config pkgname ' + will be called where 'file' has 'pkgname-config-blob' as its content. + + If 'packages_dir' has files in it that end in '.snap', then they are + installed. Given 3 files: + /foo.snap + /foo.config + /bar.snap + cloud-init will invoke: + snappy install "--config=/foo.config" \ + /foo.snap + snappy install /bar.snap +""" from cloudinit import log as logging from cloudinit import templater @@ -18,24 +49,12 @@ SNAPPY_CMD = "snappy" BUILTIN_CFG = { 'packages': [], - 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'packages_dir': '/writable/user-data/cloud-init/snaps', 'ssh_enabled': False, 'system_snappy': "auto", 'configs': {}, } -""" -snappy: - system_snappy: auto - ssh_enabled: True - packages: - - etcd - - pkg2 - configs: - pkgname: config-blob - pkgname2: config-blob -""" - def get_fs_package_ops(fspath): if not fspath: -- cgit v1.2.3 From feab0f913b2c3e98cf5200ea2dd7c19aed347395 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 22:56:43 -0400 Subject: mention ubuntu-core --- cloudinit/config/cc_snappy.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 09a8d239..de6fae4b 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -30,6 +30,10 @@ Example config: snappy install "--config=/foo.config" \ /foo.snap snappy install /bar.snap + + Note, that if provided a 'configs' entry for 'ubuntu-core', then + cloud-init will invoke: snappy config ubuntu-core + Allowing you to configure ubuntu-core in this way. """ from cloudinit import log as logging -- cgit v1.2.3 From b9cdcb6a8ef499c6e3be178fb5f59d369eb3b169 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 Mar 2015 23:22:12 -0400 Subject: fix scope so that SNAPPY_CMD is affected by set_snappy_command --- cloudinit/config/cc_snappy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index de6fae4b..e664234a 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -194,11 +194,12 @@ def system_is_snappy(): def set_snappy_command(): + global SNAPPY_CMD if util.which("snappy-go"): - SNAPPY_COMMAND = "snappy-go" + SNAPPY_CMD = "snappy-go" else: - SNAPPY_COMMAND = "snappy" - LOG.debug("snappy command is '%s'", SNAPPY_COMMAND) + SNAPPY_CMD = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_CMD) def handle(name, cfg, cloud, log, args): -- cgit v1.2.3 From b7b8004bc58f1243d023092e67f3b78743086ff2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 11:11:05 -0400 Subject: change 'configs' to 'config', and namespace input to 'snappy config' the input to 'snappy config ' is expected to have config: : content: So here we pad that input correctly. Note, that a .config file on disk is not modified. Also, we change 'configs' to just be 'config', to be possibly compatible with the a future 'snappy config /' that dumped: config: pkg1: data1 pkg2: data2 --- cloudinit/config/cc_snappy.py | 29 +++++----- .../unittests/test_handler/test_handler_snappy.py | 61 ++++++++++++++++------ 2 files changed, 61 insertions(+), 29 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index e664234a..a3af98a6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -8,9 +8,11 @@ Example config: system_snappy: auto ssh_enabled: False packages: [etcd, pkg2] - configs: - pkgname: pkgname-config-blob - pkg2: config-blob + config: + pkgname: + key2: value2 + pkg2: + key1: value1 packages_dir: '/writable/user-data/cloud-init/snaps' - ssh_enabled: @@ -31,7 +33,7 @@ Example config: /foo.snap snappy install /bar.snap - Note, that if provided a 'configs' entry for 'ubuntu-core', then + Note, that if provided a 'config' entry for 'ubuntu-core', then cloud-init will invoke: snappy config ubuntu-core Allowing you to configure ubuntu-core in this way. """ @@ -56,7 +58,7 @@ BUILTIN_CFG = { 'packages_dir': '/writable/user-data/cloud-init/snaps', 'ssh_enabled': False, 'system_snappy': "auto", - 'configs': {}, + 'config': {}, } @@ -119,15 +121,14 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): try: cfg_tmpf = None if config is not None: - if isinstance(config, six.binary_type): - cfg_bytes = config - elif isinstance(config, six.text_type): - cfg_bytes = config.encode() - else: - cfg_bytes = util.yaml_dumps(config).encode() - + # input to 'snappy config packagename' must have nested data. odd. + # config: + # packagename: + # config + # Note, however, we do not touch config files on disk. + nested_cfg = {'config': {name: config}} (fd, cfg_tmpf) = tempfile.mkstemp() - os.write(fd, cfg_bytes) + os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) cfgfile = cfg_tmpf @@ -218,7 +219,7 @@ def handle(name, cfg, cloud, log, args): return pkg_ops = get_package_ops(packages=mycfg['packages'], - configs=mycfg['configs'], + configs=mycfg['config'], fspath=mycfg['packages_dir']) set_snappy_command() diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 81d891d2..f56a22f7 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -6,6 +6,9 @@ from .. import helpers as t_help import os import shutil import tempfile +import yaml + +ALLOWED = (dict, list, int, str) class TestInstallPackages(t_help.TestCase): @@ -44,7 +47,7 @@ class TestInstallPackages(t_help.TestCase): config = kwargs.get('data', '') else: with open(args[3], "rb") as fp: - config = fp.read() + config = yaml.safe_load(fp.read()) self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: config = None @@ -56,7 +59,7 @@ class TestInstallPackages(t_help.TestCase): config = kwargs.get('data', '') elif cfgfile: with open(cfgfile, "rb") as fp: - config = fp.read() + config = yaml.safe_load(fp.read()) elif not pkg and not arg.startswith("-"): pkg = arg self.snapcmds.append(['install', pkg, config]) @@ -126,7 +129,7 @@ class TestInstallPackages(t_help.TestCase): path='snapf1.snap', cfgfile='snapf1.config') render_snap_op(**op) self.assertEqual( - self.snapcmds, [['install', op['path'], b'snapf1cfg']]) + self.snapcmds, [['install', op['path'], 'snapf1cfg']]) def test_render_op_snap(self): op = makeop('install', 'snapf1') @@ -135,49 +138,77 @@ class TestInstallPackages(t_help.TestCase): self.snapcmds, [['install', 'snapf1', None]]) def test_render_op_snap_config(self): - op = makeop('install', 'snapf1', config=b'myconfig') + mycfg = {'key1': 'value1'} + name = "snapf1" + op = makeop('install', name, config=mycfg) render_snap_op(**op) self.assertEqual( - self.snapcmds, [['install', 'snapf1', b'myconfig']]) + self.snapcmds, [['install', name, {'config': {name: mycfg}}]]) def test_render_op_config_bytes(self): - op = makeop('config', 'snapf1', config=b'myconfig') + name = "snapf1" + mycfg = b'myconfig' + op = makeop('config', name, config=mycfg) render_snap_op(**op) self.assertEqual( - self.snapcmds, [['config', 'snapf1', b'myconfig']]) + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) def test_render_op_config_string(self): + name = 'snapf1' mycfg = 'myconfig: foo\nhisconfig: bar\n' - op = makeop('config', 'snapf1', config=mycfg) + op = makeop('config', name, config=mycfg) render_snap_op(**op) self.assertEqual( - self.snapcmds, [['config', 'snapf1', mycfg.encode()]]) + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) def test_render_op_config_dict(self): # config entry for package can be a dict, not a string blob mycfg = {'foo': 'bar'} - op = makeop('config', 'snapf1', config=mycfg) + name = 'snapf1' + op = makeop('config', name, config=mycfg) render_snap_op(**op) # snapcmds is a list of 3-entry lists. data_found will be the # blob of data in the file in 'snappy install --config=' data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, util.load_yaml(data_found)) + self.assertEqual(mycfg, data_found['config'][name]) def test_render_op_config_list(self): # config entry for package can be a list, not a string blob mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] - op = makeop('config', 'snapf1', config=mycfg) + name = "snapf1" + op = makeop('config', name, config=mycfg) render_snap_op(**op) data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(list,))) + self.assertEqual(mycfg, data_found['config'][name]) def test_render_op_config_int(self): # config entry for package can be a list, not a string blob mycfg = 1 - op = makeop('config', 'snapf1', config=mycfg) + name = 'snapf1' + op = makeop('config', name, config=mycfg) render_snap_op(**op) data_found = self.snapcmds[0][2] - self.assertEqual(mycfg, util.load_yaml(data_found, allowed=(int,))) + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_does_not_pad_cfgfile(self): + # package_ops with cfgfile should not modify --file= content. + mydata = "foo1: bar1\nk: [l1, l2, l3]\n" + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()}) + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config")]) + + # now the op was ok, but test that render didn't mess it up. + render_snap_op(**ret[0]) + data_found = self.snapcmds[0][2] + # the data found gets loaded in the snapcmd interpretation + # so this comparison is a bit lossy, but input to snappy config + # is expected to be yaml loadable, so it should be OK. + self.assertEqual(yaml.safe_load(mydata), data_found) def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): -- cgit v1.2.3 From 6c48673245225c5530c7cc08f5ab82794c708f71 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 11:33:58 -0400 Subject: set snappy command earlier --- cloudinit/config/cc_snappy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index a3af98a6..f237feef 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -218,12 +218,12 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return + set_snappy_command() + pkg_ops = get_package_ops(packages=mycfg['packages'], configs=mycfg['config'], fspath=mycfg['packages_dir']) - set_snappy_command() - fails = [] for pkg_op in pkg_ops: try: -- cgit v1.2.3 From b4989280d7285f214c1016efa36a20ad57821d6b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 14:19:56 -0400 Subject: address namespacing --- cloudinit/config/cc_snappy.py | 52 +++++++++++++++++----- .../unittests/test_handler/test_handler_snappy.py | 46 +++++++++++++++++++ 2 files changed, 88 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index f237feef..f8f67e1f 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -7,7 +7,7 @@ Example config: snappy: system_snappy: auto ssh_enabled: False - packages: [etcd, pkg2] + packages: [etcd, pkg2.smoser] config: pkgname: key2: value2 @@ -18,11 +18,15 @@ Example config: - ssh_enabled: This defaults to 'False'. Set to a non-false value to enable ssh service - snap installation and config - The above would install 'etcd', and then install 'pkg2' with a + The above would install 'etcd', and then install 'pkg2.smoser' with a '--config=' argument where 'file' as 'config-blob' inside it. If 'pkgname' is installed already, then 'snappy config pkgname ' will be called where 'file' has 'pkgname-config-blob' as its content. + Entries in 'config' can be namespaced or non-namespaced for a package. + In either case, the config provided to snappy command is non-namespaced. + The package name is provided as it appears. + If 'packages_dir' has files in it that end in '.snap', then they are installed. Given 3 files: /foo.snap @@ -52,6 +56,7 @@ LOG = logging.getLogger(__name__) frequency = PER_INSTANCE SNAPPY_CMD = "snappy" +NAMESPACE_DELIM = '.' BUILTIN_CFG = { 'packages': [], @@ -81,10 +86,20 @@ def makeop(op, name, config=None, path=None, cfgfile=None): 'cfgfile': cfgfile}) +def get_package_config(configs, name): + # load the package's config from the configs dict. + # prefer full-name entry (config-example.canonical) + # over short name entry (config-example) + if name in configs: + return configs[name] + return configs.get(name.partition(NAMESPACE_DELIM)[0]) + + def get_package_ops(packages, configs, installed=None, fspath=None): # get the install an config operations that should be done if installed is None: installed = read_installed_packages() + short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] if not packages: packages = [] @@ -95,21 +110,31 @@ def get_package_ops(packages, configs, installed=None, fspath=None): ops += get_fs_package_ops(fspath) for name in packages: - ops.append(makeop('install', name, configs.get('name'))) + ops.append(makeop('install', name, get_package_config(configs, name))) to_install = [f['name'] for f in ops] + short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] for name in configs: - if name in installed and name not in to_install: - ops.append(makeop('config', name, config=configs[name])) + if name in to_install: + continue + shortname = name.partition(NAMESPACE_DELIM)[0] + if shortname in short_to_install: + continue + if name in installed or shortname in short_installed: + ops.append(makeop('config', name, + config=get_package_config(configs, name))) # prefer config entries to filepath entries for op in ops: + if op['op'] != 'install' or not op['cfgfile']: + continue name = op['name'] - if name in configs and op['op'] == 'install' and 'cfgfile' in op: - LOG.debug("preferring configs[%s] over '%s'", name, op['cfgfile']) + fromcfg = get_package_config(configs, op['name']) + if fromcfg: + LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) op['cfgfile'] = None - op['config'] = configs[op['name']] + op['config'] = fromcfg return ops @@ -118,6 +143,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): if op not in ('install', 'config'): raise ValueError("cannot render op '%s'" % op) + shortname = name.partition(NAMESPACE_DELIM)[0] try: cfg_tmpf = None if config is not None: @@ -126,7 +152,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): # packagename: # config # Note, however, we do not touch config files on disk. - nested_cfg = {'config': {name: config}} + nested_cfg = {'config': {shortname: config}} (fd, cfg_tmpf) = tempfile.mkstemp() os.write(fd, util.yaml_dumps(nested_cfg).encode()) os.close(fd) @@ -151,7 +177,13 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): def read_installed_packages(): - return [p[0] for p in read_pkg_data()] + ret = [] + for (name, date, version, dev) in read_pkg_data(): + if dev: + ret.append(NAMESPACE_DELIM.join(name, dev)) + else: + ret.append(name) + return ret def read_pkg_data(): diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index f56a22f7..f0776259 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -90,6 +90,15 @@ class TestInstallPackages(t_help.TestCase): makeop('install', 'pkg2', b'mycfg2'), makeop('config', 'xinstalled', b'xcfg')]) + def test_package_ops_install_long_config_short(self): + # a package can be installed by full name, but have config by short + cfg = {'k1': 'k2'} + ret = get_package_ops( + packages=['config-example.canonical'], + configs={'config-example': cfg}, installed=[]) + self.assertEqual( + ret, [makeop('install', 'config-example.canonical', cfg)]) + def test_package_ops_with_file(self): self.populate_tmp( {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", @@ -114,6 +123,34 @@ class TestInstallPackages(t_help.TestCase): ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", config="snapf1cfg-config")]) + def test_package_ops_namespacing(self): + cfgs = { + 'config-example': {'k1': 'v1'}, + 'pkg1': {'p1': 'p2'}, + 'ubuntu-core': {'c1': 'c2'}, + 'notinstalled.smoser': {'s1': 's2'}, + } + cfg = {'config-example-k1': 'config-example-k2'} + ret = get_package_ops( + packages=['config-example.canonical'], configs=cfgs, + installed=['config-example.smoser', 'pkg1.canonical', + 'ubuntu-core']) + + expected_configs = [ + makeop('config', 'pkg1', config=cfgs['pkg1']), + makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])] + expected_installs = [ + makeop('install', 'config-example.canonical', + config=cfgs['config-example'])] + + installs = [i for i in ret if i['op'] == 'install'] + configs = [c for c in ret if c['op'] == 'config'] + + self.assertEqual(installs, expected_installs) + # configs are not ordered + self.assertEqual(len(configs), len(expected_configs)) + self.assertTrue(all(found in expected_configs for found in configs)) + def test_render_op_localsnap(self): self.populate_tmp({"snapf1.snap": b"foo1"}) op = makeop_tmpd(self.tmp, 'install', 'snapf1', @@ -190,6 +227,15 @@ class TestInstallPackages(t_help.TestCase): data_found = self.snapcmds[0][2] self.assertEqual(mycfg, data_found['config'][name]) + def test_render_long_configs_short(self): + # install a namespaced package should have un-namespaced config + mycfg = {'k1': 'k2'} + name = 'snapf1' + op = makeop('install', name + ".smoser", config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + def test_render_does_not_pad_cfgfile(self): # package_ops with cfgfile should not modify --file= content. mydata = "foo1: bar1\nk: [l1, l2, l3]\n" -- cgit v1.2.3 From 25a05c3367e024fcee5da0b4f15b5ca599dd92f2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 14:31:06 -0400 Subject: fix read_install --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index f8f67e1f..d9dd9771 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -180,7 +180,7 @@ def read_installed_packages(): ret = [] for (name, date, version, dev) in read_pkg_data(): if dev: - ret.append(NAMESPACE_DELIM.join(name, dev)) + ret.append(NAMESPACE_DELIM.join([name, dev])) else: ret.append(name) return ret -- cgit v1.2.3 From 09cc5909e3d69c03622b7dc2c4cb35fd378743cb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 16:20:05 -0400 Subject: be more user-friendly when looking for matching .config On fspath installs, look for .config files harder. Given a file named: pkg.namespace_0.version_arch.snap We'll search for config files named: pkg.namespace_0.version_arch.config pkg.namespace.config pkg.config --- cloudinit/config/cc_snappy.py | 21 +++++++++---- .../unittests/test_handler/test_handler_snappy.py | 34 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index d9dd9771..74ae3ac0 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -67,15 +67,26 @@ BUILTIN_CFG = { } +def parse_filename(fname): + fname = os.path.basename(fname) + fname_noext = fname.rpartition(".")[0] + name = fname_noext.partition("_")[0] + shortname = name.partition(".")[0] + return(name, shortname, fname_noext) + + def get_fs_package_ops(fspath): if not fspath: return [] ops = [] - for snapfile in glob.glob(os.path.sep.join([fspath, '*.snap'])): - cfg = snapfile.rpartition(".")[0] + ".config" - name = os.path.basename(snapfile).rpartition(".")[0] - if not os.path.isfile(cfg): - cfg = None + for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): + (name, shortname, fname_noext) = parse_filename(snapfile) + cfg = None + for cand in set((fname_noext, name, shortname,)): + fpcand = os.path.sep.join([fspath, cand]) + ".config" + if os.path.isfile(fpcand): + cfg = fpcand + break ops.append(makeop('install', name, config=None, path=snapfile, cfgfile=cfg)) return ops diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index f0776259..84512846 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -112,6 +112,40 @@ class TestInstallPackages(t_help.TestCase): makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), makeop('install', 'pkg1')]) + def test_package_ops_common_filename(self): + # fish package name from filename + # package names likely look like: pkgname.namespace_version_arch.snap + fname = "xkcd-webserver.canonical_0.3.4_all.snap" + name = "xkcd-webserver.canonical" + shortname = "xkcd-webserver" + + # find filenames + self.populate_tmp( + {"pkg-ws.smoser_0.3.4_all.snap": "pkg-ws-snapdata", + "pkg-ws.config": "pkg-ws-config", + "pkg1.smoser_1.2.3_all.snap": "pkg1.snapdata", + "pkg1.smoser.config": "pkg1.smoser.config-data", + "pkg1.config": "pkg1.config-data", + "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", + "pkg2.smoser_0.0_amd64.config": "pkg2.config", + }) + + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + raise Exception("ret: %s" % ret) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'pkg-ws.smoser', + path="pkg-ws.smoser_0.3.4_all.snap", + cfgfile="pkg-ws.config"), + makeop_tmpd(self.tmp, 'install', 'pkg1.smoser', + path="pkg1.smoser_1.2.3_all.snap", + cfgfile="pkg1.smoser.config"), + makeop_tmpd(self.tmp, 'install', 'pkg2.smoser', + path="pkg2.smoser_0.0_amd64.snap", + cfgfile="pkg2.smoser_0.0_amd64.config"), + ]) + def test_package_ops_config_overrides_file(self): # config data overrides local file .config self.populate_tmp( -- cgit v1.2.3 From 6f738bea5d2aa29cdf14d0dc2a6e880517ab2bc2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 16:27:47 -0400 Subject: do not use set --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 74ae3ac0..05676321 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -82,7 +82,7 @@ def get_fs_package_ops(fspath): for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): (name, shortname, fname_noext) = parse_filename(snapfile) cfg = None - for cand in set((fname_noext, name, shortname,)): + for cand in (fname_noext, name, shortname): fpcand = os.path.sep.join([fspath, cand]) + ".config" if os.path.isfile(fpcand): cfg = fpcand -- cgit v1.2.3 From 522a146eadcdb30e68acaaf792c391a7f1da3151 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 Mar 2015 17:03:59 -0400 Subject: allow-unauthenticated when done from local file --- cloudinit/config/cc_snappy.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 05676321..131ee7ea 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -174,6 +174,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): if cfgfile: cmd.append('--config=' + cfgfile) if path: + cmd.append("--allow-unauthenticated") cmd.append(path) else: cmd.append(name) -- cgit v1.2.3 From 8165000c3975db07cb5b8b29410635dd6c9345bd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 Mar 2015 14:20:00 -0400 Subject: adjust cc_snappy for snappy install package with config. It was believed that to install a package with config the command was: snappy install --config=config-file Instead, what was implemented in snappy was: snappy install [] This modifies cloud-init to invoke the latter and changes the tests appropriately. LP: #1438836 --- cloudinit/config/cc_snappy.py | 9 ++++----- tests/unittests/test_handler/test_handler_snappy.py | 10 ++++++---- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 131ee7ea..6a7ae09b 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -19,7 +19,7 @@ Example config: This defaults to 'False'. Set to a non-false value to enable ssh service - snap installation and config The above would install 'etcd', and then install 'pkg2.smoser' with a - '--config=' argument where 'file' as 'config-blob' inside it. + '' argument where 'config-file' has 'config-blob' inside it. If 'pkgname' is installed already, then 'snappy config pkgname ' will be called where 'file' has 'pkgname-config-blob' as its content. @@ -33,8 +33,7 @@ Example config: /foo.config /bar.snap cloud-init will invoke: - snappy install "--config=/foo.config" \ - /foo.snap + snappy install /foo.snap /foo.config snappy install /bar.snap Note, that if provided a 'config' entry for 'ubuntu-core', then @@ -171,13 +170,13 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): cmd = [SNAPPY_CMD, op] if op == 'install': - if cfgfile: - cmd.append('--config=' + cfgfile) if path: cmd.append("--allow-unauthenticated") cmd.append(path) else: cmd.append(name) + if cfgfile: + cmd.append(cfgfile) elif op == 'config': cmd += [name, cfgfile] diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 8effd99d..f3109bac 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -53,15 +53,17 @@ class TestInstallPackages(t_help.TestCase): config = None pkg = None for arg in args[2:]: - if arg.startswith("--config="): - cfgfile = arg.partition("=")[2] + if arg.startswith("-"): + continue + if not pkg: + pkg = arg + elif not config: + cfgfile = arg if cfgfile == "-": config = kwargs.get('data', '') elif cfgfile: with open(cfgfile, "rb") as fp: config = yaml.safe_load(fp.read()) - elif not pkg and not arg.startswith("-"): - pkg = arg self.snapcmds.append(['install', pkg, config]) def test_package_ops_1(self): -- cgit v1.2.3 From dcd4b2b371059bd6249b4e43af371ee1162273e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Apr 2015 16:41:06 -0400 Subject: pep8 fixes --- cloudinit/config/cc_snappy.py | 4 ++-- cloudinit/handlers/__init__.py | 6 +++--- tests/unittests/test_data.py | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 6a7ae09b..bfe76558 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -72,7 +72,7 @@ def parse_filename(fname): name = fname_noext.partition("_")[0] shortname = name.partition(".")[0] return(name, shortname, fname_noext) - + def get_fs_package_ops(fspath): if not fspath: @@ -98,7 +98,7 @@ def makeop(op, name, config=None, path=None, cfgfile=None): def get_package_config(configs, name): # load the package's config from the configs dict. - # prefer full-name entry (config-example.canonical) + # prefer full-name entry (config-example.canonical) # over short name entry (config-example) if name in configs: return configs[name] diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d62fcd19..52defe66 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -170,12 +170,12 @@ def _extract_first_or_bytes(blob, size): start = blob.split("\n", 1)[0] else: # We want to avoid decoding the whole blob (it might be huge) - # By taking 4*size bytes we have a guarantee to decode size utf8 chars - start = blob[:4*size].decode(errors='ignore').split("\n", 1)[0] + # By taking 4*size bytes we guarantee to decode size utf8 chars + start = blob[:4 * size].decode(errors='ignore').split("\n", 1)[0] if len(start) >= size: start = start[:size] except UnicodeDecodeError: - # Bytes array doesn't contain a text object -- return chunk of raw bytes + # Bytes array doesn't contain text so return chunk of raw bytes start = blob[0:size] return start diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4f24e2dd..b950c9a5 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -494,10 +494,10 @@ c: 4 ]) def test_mime_application_octet_stream(self): - """Mime message of type application/octet-stream is ignored but shows warning.""" + """Mime type application/octet-stream is ignored but shows warning.""" ci = stages.Init() message = MIMEBase("application", "octet-stream") - message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc\xbf') + message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc') encoders.encode_base64(message) ci.datasource = FakeDataSource(message.as_string().encode()) @@ -511,6 +511,7 @@ c: 4 mockobj.assert_called_once_with( ci.paths.get_ipath("cloud_config"), "", 0o600) + class TestUDProcess(helpers.ResourceUsingTestCase): def test_bytes_in_userdata(self): -- cgit v1.2.3 From 96854d720d4bd356181acfa093744599a807ea8e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 1 May 2015 05:38:56 -0400 Subject: fix 'Make pyflakes' --- Makefile | 2 +- cloudinit/config/cc_apt_pipelining.py | 2 +- cloudinit/config/cc_snappy.py | 2 -- cloudinit/sources/DataSourceOpenNebula.py | 1 - tests/unittests/test_datasource/test_smartos.py | 2 -- tests/unittests/test_handler/test_handler_apt_configure.py | 1 - tests/unittests/test_handler/test_handler_snappy.py | 5 ----- tests/unittests/test_templating.py | 5 +---- tools/hacking.py | 2 +- tools/validate-yaml.py | 3 +-- 10 files changed, 5 insertions(+), 20 deletions(-) (limited to 'cloudinit/config') diff --git a/Makefile b/Makefile index 009257ca..bb0c5253 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ pep8: @$(CWD)/tools/run-pep8 $(PY_FILES) pyflakes: - pyflakes $(PY_FILES) + @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) pip-requirements: @echo "Installing cloud-init dependencies..." diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index e5629175..40c32c84 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -43,7 +43,7 @@ def handle(_name, cfg, _cloud, log, _args): write_apt_snippet("0", log, DEFAULT_FILE) elif apt_pipe_value_s in ("none", "unchanged", "os"): return - elif apt_pipe_value_s in [str(b) for b in xrange(0, 6)]: + elif apt_pipe_value_s in [str(b) for b in range(0, 6)]: write_apt_snippet(apt_pipe_value_s, log, DEFAULT_FILE) else: log.warn("Invalid option for apt_pipeling: %s", apt_pipe_value) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bfe76558..7aaec94a 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -42,12 +42,10 @@ Example config: """ from cloudinit import log as logging -from cloudinit import templater from cloudinit import util from cloudinit.settings import PER_INSTANCE import glob -import six import tempfile import os diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 61709c1b..ac2c3b45 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -24,7 +24,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import base64 import os import pwd import re diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 28b41eaf..adee9019 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -36,8 +36,6 @@ from binascii import crc32 import serial import six -import six - from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py index 02cad8b2..895728b3 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ b/tests/unittests/test_handler/test_handler_apt_configure.py @@ -7,7 +7,6 @@ import os import re import shutil import tempfile -import unittest class TestAptProxyConfig(TestCase): diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index f3109bac..eceb14d9 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -38,7 +38,6 @@ class TestInstallPackages(t_help.TestCase): if 'args' not in kwargs: kwargs['args'] = args[0] self.subp_called.append(kwargs) - snap_cmds = [] args = kwargs['args'] # here we basically parse the snappy command invoked # and append to snapcmds a list of (mode, pkg, config) @@ -117,9 +116,6 @@ class TestInstallPackages(t_help.TestCase): def test_package_ops_common_filename(self): # fish package name from filename # package names likely look like: pkgname.namespace_version_arch.snap - fname = "xkcd-webserver.canonical_0.3.4_all.snap" - name = "xkcd-webserver.canonical" - shortname = "xkcd-webserver" # find filenames self.populate_tmp( @@ -165,7 +161,6 @@ class TestInstallPackages(t_help.TestCase): 'ubuntu-core': {'c1': 'c2'}, 'notinstalled.smoser': {'s1': 's2'}, } - cfg = {'config-example-k1': 'config-example-k2'} ret = get_package_ops( packages=['config-example.canonical'], configs=cfgs, installed=['config-example.smoser', 'pkg1.canonical', diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index cf7c03b0..0c19a2c2 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -18,10 +18,6 @@ from __future__ import print_function -import sys -import six -import unittest - from . import helpers as test_helpers import textwrap @@ -30,6 +26,7 @@ from cloudinit import templater try: import Cheetah HAS_CHEETAH = True + Cheetah # make pyflakes happy, as Cheetah is not used here except ImportError: HAS_CHEETAH = False diff --git a/tools/hacking.py b/tools/hacking.py index e7797564..3175df38 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -128,7 +128,7 @@ def cloud_docstring_multiline_end(physical_line): """ pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start if (pos != -1 and len(physical_line) == pos): - print physical_line + print(physical_line) if (physical_line[pos + 3] == ' '): return (pos, "N403: multi line docstring end on new line") diff --git a/tools/validate-yaml.py b/tools/validate-yaml.py index eda59cb8..6e164590 100755 --- a/tools/validate-yaml.py +++ b/tools/validate-yaml.py @@ -4,7 +4,6 @@ """ import sys - import yaml @@ -17,7 +16,7 @@ if __name__ == "__main__": yaml.safe_load(fh.read()) fh.close() sys.stdout.write(" - ok\n") - except Exception, e: + except Exception as e: sys.stdout.write(" - bad (%s)\n" % (e)) bads += 1 if bads > 0: -- cgit v1.2.3 From 8b8a90372058205496f42abf2a3d0dc04c7eab3f Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Thu, 14 May 2015 14:29:42 -0500 Subject: This patch adds a cloud-init plugin for helping users register and subscribe their RHEL based systems. As inputs, it can take: - user and password OR activation key and org | requires on of the two pair - auto-attach: True or False | optional - service-level: | optional - add-pool [list, of, pool, ids] | optional - enable-repos [list, of, yum, repos, to, enable] | optional - disable-repos [list, of, yum, repos, to, disable] | optional You can also pass the following to influence your registration via rhsm.conf: - rhsm-baseurl | optional - server-hostname | optional --- cloudinit/config/cc_rh_subscription.py | 399 +++++++++++++++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 cloudinit/config/cc_rh_subscription.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py new file mode 100644 index 00000000..b8056dbb --- /dev/null +++ b/cloudinit/config/cc_rh_subscription.py @@ -0,0 +1,399 @@ +# vi: ts=4 expandtab +# +# Copyright (C) Red Hat, Inc. +# +# Author: Brent Baude +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import subprocess +import itertools + + +def handle(_name, cfg, _cloud, log, _args): + sm = SubscriptionManager(cfg) + sm.log = log + + if not sm.is_registered: + try: + verify, verify_msg = sm._verify_keys() + if verify is not True: + raise SubscriptionError(verify_msg) + cont = sm.rhn_register() + if not cont: + raise SubscriptionError("Registration failed or did not " + "run completely") + + # Splitting up the registration, auto-attach, and servicelevel + # commands because the error codes, messages from subman are not + # specific enough. + + # Attempt to change the service level + if sm.auto_attach and sm.servicelevel is not None: + if not sm._set_service_level(): + raise SubscriptionError("Setting of service-level " + "failed") + else: + sm.log.info("Completed auto-attach with service level") + elif sm.auto_attach: + if not sm._set_auto_attach(): + raise SubscriptionError("Setting auto-attach failed") + else: + sm.log.info("Completed auto-attach") + + if sm.pools is not None: + if type(sm.pools) is not list: + raise SubscriptionError("Pools must in the format of a " + "list.") + return_stat = sm.addPool(sm.pools) + if not return_stat: + raise SubscriptionError("Unable to attach pools {0}" + .format(sm.pools)) + if (sm.enable_repo is not None) or (sm.disable_repo is not None): + return_stat = sm.update_repos(sm.enable_repo, sm.disable_repo) + if not return_stat: + raise SubscriptionError("Unable to add or remove repos") + sm.log.info("rh_subscription plugin completed successfully") + except SubscriptionError as e: + sm.log.warn(e) + sm.log.info("rh_subscription plugin did not complete successfully") + else: + sm.log.info("System is already registered") + + +class SubscriptionError(Exception): + pass + + +class SubscriptionManager(object): + def __init__(self, cfg): + self.cfg = cfg + self.rhel_cfg = self.cfg.get('rh_subscription', {}) + self.rhsm_baseurl = self.rhel_cfg.get('rhsm-baseurl') + self.server_hostname = self.rhel_cfg.get('server-hostname') + self.pools = self.rhel_cfg.get('add-pool') + self.activation_key = self.rhel_cfg.get('activation-key') + self.org = self.rhel_cfg.get('org') + self.userid = self.rhel_cfg.get('username') + self.password = self.rhel_cfg.get('password') + self.auto_attach = self.rhel_cfg.get('auto-attach') + self.enable_repo = self.rhel_cfg.get('enable-repo') + self.disable_repo = self.rhel_cfg.get('disable-repo') + self.servicelevel = self.rhel_cfg.get('service-level') + self.subman = ['/bin/subscription-manager'] + self.valid_rh_keys = ['org', 'activation-key', 'username', 'password', + 'disable-repo', 'enable-repo', 'add-pool', + 'rhsm-baseurl', 'server-hostname', + 'auto-attach', 'service-level'] + self.is_registered = self._is_registered() + + def _verify_keys(self): + ''' + Checks that the keys in the rh_subscription dict from the user-data + are what we expect. + ''' + + for k in self.rhel_cfg: + if k not in self.valid_rh_keys: + bad_key = "{0} is not a valid key for rh_subscription. "\ + "Valid keys are: "\ + "{1}".format(k, ', '.join(self.valid_rh_keys)) + return False, bad_key + + # Check for bad auto-attach value + if (self.auto_attach is not None) and \ + (str(self.auto_attach).upper() not in ['TRUE', 'FALSE']): + not_bool = "The key auto-attach must be a value of "\ + "either True or False" + return False, not_bool + + if (self.servicelevel is not None) and \ + ((not self.auto_attach) or + (str(self.auto_attach).upper() == "FALSE")): + + no_auto = "The service-level key must be used in conjunction with "\ + "the auto-attach key. Please re-run with auto-attach: "\ + "True" + return False, no_auto + return True, None + + def _is_registered(self): + ''' + Checks if the system is already registered and returns + True if so, else False + ''' + cmd = list(itertools.chain(self.subman, ['identity'])) + + if subprocess.call(cmd, stdout=open(os.devnull, 'wb'), + stderr=open(os.devnull, 'wb')) == 1: + return False + else: + return True + + def rhn_register(self): + ''' + Registers the system by userid and password or activation key + and org. Returns True when successful False when not. + ''' + + if (self.activation_key is not None) and (self.org is not None): + # register by activation key + cmd = list(itertools.chain(self.subman, ['register', + '--activationkey={0}'. + format(self.activation_key), + '--org={0}'.format(self.org)])) + + # If the baseurl and/or server url are passed in, we register + # with them. + + if self.rhsm_baseurl is not None: + cmd.append("--baseurl={0}".format(self.rhsm_baseurl)) + + if self.server_hostname is not None: + cmd.append("--serverurl={0}".format(self.server_hostname)) + + return_msg, return_code = self._captureRun(cmd) + + if return_code is not 0: + self.log.warn("Registration with {0} and {1} failed.".format( + self.activation_key, self.org)) + return False + + elif (self.userid is not None) and (self.password is not None): + # register by username and password + cmd = list(itertools.chain(self.subman, ['register', + '--username={0}'.format(self.userid), + '--password={0}'.format(self.password)])) + + # If the baseurl and/or server url are passed in, we register + # with them. + + if self.rhsm_baseurl is not None: + cmd.append("--baseurl={0}".format(self.rhsm_baseurl)) + + if self.server_hostname is not None: + cmd.append("--serverurl={0}".format(self.server_hostname)) + + # Attempting to register the system only + return_msg, return_code = self._captureRun(cmd) + + if return_code is not 0: + # Return message is in a set + if return_msg[0] == "": + self.log.warn("Registration failed") + if return_msg[1] is not "": + self.log.warn(return_msg[1]) + return False + + else: + self.log.warn("Unable to register system due to incomplete " + "information.") + self.log.warn("Use either activationkey and org *or* userid " + "and password") + return False + + reg_id = return_msg[0].split("ID: ")[1].rstrip() + self.log.info("Registered successfully with ID {0}".format(reg_id)) + return True + + def _set_service_level(self): + cmd = list(itertools.chain(self.subman, + ['attach', '--auto', '--servicelevel={0}' + .format(self.servicelevel)])) + + return_msg, return_code = self._captureRun(cmd) + + if return_code is not 0: + self.log.warn("Setting the service level failed with: " + "{0}".format(return_msg[1].strip())) + return False + else: + for line in return_msg[0].split("\n"): + if line is not "": + self.log.info(line) + return True + + def _set_auto_attach(self): + cmd = list(itertools.chain(self.subman, ['attach', '--auto'])) + return_msg, return_code = self._captureRun(cmd) + + if return_code is not 0: + self.log.warn("Auto-attach failed with: " + "{0}]".format(return_msg[1].strip())) + return False + else: + for line in return_msg[0].split("\n"): + if line is not "": + self.log.info(line) + return True + + def _captureRun(self, cmd): + ''' + Subprocess command that captures and returns the output and + return code. + ''' + + r = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return r.communicate(), r.returncode + + def _getPools(self): + ''' + Gets the list pools for the active subscription and returns them + in list form. + ''' + available = [] + consumed = [] + + # Get all available pools + cmd = list(itertools.chain(self.subman, ['list', '--available', + '--pool-only'])) + results = subprocess.check_output(cmd) + available = (results.rstrip()).split("\n") + + # Get all available pools + cmd = list(itertools.chain(self.subman, ['list', '--consumed', + '--pool-only'])) + results = subprocess.check_output(cmd) + consumed = (results.rstrip()).split("\n") + return available, consumed + + def _getRepos(self): + ''' + Obtains the current list of active yum repositories and returns + them in list form. + ''' + + cmd = list(itertools.chain(self.subman, ['repos', '--list-enabled'])) + result, return_code = self._captureRun(cmd) + + active_repos = [] + for repo in result[0].split("\n"): + if "Repo ID:" in repo: + active_repos.append((repo.split(':')[1]).strip()) + + cmd = list(itertools.chain(self.subman, ['repos', '--list-disabled'])) + result, return_code = self._captureRun(cmd) + + inactive_repos = [] + for repo in result[0].split("\n"): + if "Repo ID:" in repo: + inactive_repos.append((repo.split(':')[1]).strip()) + + return active_repos, inactive_repos + + def addPool(self, pools): + ''' + Takes a list of subscription pools and "attaches" them to the + current subscription + ''' + + # An empty list was passed + if len(pools) == 0: + self.log.info("No pools to attach") + return True + + pool_available, pool_consumed = self._getPools() + pool_list = [] + cmd = list(itertools.chain(self.subman, ['attach'])) + for pool in pools: + if (pool not in pool_consumed) and (pool in pool_available): + pool_list.append('--pool={0}'.format(pool)) + else: + self.log.warn("Pool {0} is not available".format(pool)) + if len(pool_list) > 0: + cmd.extend(pool_list) + try: + self._captureRun(cmd) + self.log.info("Attached the following pools to your " + "system: %s" % (", ".join(pool_list)) + .replace('--pool=', '')) + return True + except subprocess.CalledProcessError: + self.log.warn("Unable to attach pool {0}".format(pool)) + return False + + def update_repos(self, erepos, drepos): + ''' + Takes a list of yum repo ids that need to be disabled or enabled; then + it verifies if they are already enabled or disabled and finally + executes the action to disable or enable + ''' + + if (erepos is not None) and (type(erepos) is not list): + self.log.warn("Repo IDs must in the format of a list.") + return False + + if (drepos is not None) and (type(drepos) is not list): + self.log.warn("Repo IDs must in the format of a list.") + return False + + # Bail if both lists are not populated + if (len(erepos) == 0) and (len(drepos) == 0): + self.log.info("No repo IDs to enable or disable") + return True + + active_repos, inactive_repos = self._getRepos() + # Creating a list of repoids to be enabled + enable_list = [] + enable_list_fail = [] + for repoid in erepos: + if (repoid in inactive_repos): + enable_list.append("--enable={0}".format(repoid)) + else: + enable_list_fail.append(repoid) + + # Creating a list of repoids to be disabled + disable_list = [] + disable_list_fail = [] + for repoid in drepos: + if repoid in active_repos: + disable_list.append("--disable={0}".format(repoid)) + else: + disable_list_fail.append(repoid) + + # Logging any repos that are already enabled or disabled + if len(enable_list_fail) > 0: + for fail in enable_list_fail: + # Check if the repo exists or not + if fail in active_repos: + self.log.info("Repo {0} is already enabled".format(fail)) + else: + self.log.warn("Repo {0} does not appear to " + "exist".format(fail)) + if len(disable_list_fail) > 0: + for fail in disable_list_fail: + self.log.info("Repo {0} not disabled " + "because it is not enabled".format(fail)) + + cmd = list(itertools.chain(self.subman, ['repos'])) + if enable_list > 0: + cmd.extend(enable_list) + if disable_list > 0: + cmd.extend(disable_list) + + try: + return_msg, return_code = self._captureRun(cmd) + + except subprocess.CalledProcessError as e: + self.log.warn("Unable to alter repos due to {0}".format(e)) + return False + + if enable_list > 0: + self.log.info("Enabled the following repos: %s" % + (", ".join(enable_list)).replace('--enable=', '')) + if disable_list > 0: + self.log.info("Disabled the following repos: %s" % + (", ".join(disable_list)).replace('--disable=', '')) + return True -- cgit v1.2.3 From cf2b017c8bf2adb02a2c7a9c9f03754402cb73c4 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Thu, 21 May 2015 13:32:30 -0500 Subject: This commit consists of three things based on feedback from smosher: cc_rh_subscription: Use of self.log.info limited, uses the util.subp for subprocesses, removed full path for subscription-manager cloud-config-rh_subscription.txt: A heavily commented example file on how to use rh_subscription and its main keys test_rh_subscription.py: a set of unittests for rh_subscription --- cloudinit/config/cc_rh_subscription.py | 181 +++++++++++++------------ tests/unittests/test_rh_subscription.py | 231 ++++++++++++++++++++++++++++++++ 2 files changed, 323 insertions(+), 89 deletions(-) create mode 100644 tests/unittests/test_rh_subscription.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index b8056dbb..00a88456 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -16,15 +16,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os -import subprocess import itertools +from cloudinit import util def handle(_name, cfg, _cloud, log, _args): sm = SubscriptionManager(cfg) sm.log = log - if not sm.is_registered: try: verify, verify_msg = sm._verify_keys() @@ -41,21 +39,22 @@ def handle(_name, cfg, _cloud, log, _args): # Attempt to change the service level if sm.auto_attach and sm.servicelevel is not None: - if not sm._set_service_level(): - raise SubscriptionError("Setting of service-level " - "failed") - else: - sm.log.info("Completed auto-attach with service level") + if not sm._set_service_level(): + raise SubscriptionError("Setting of service-level " + "failed") + else: + sm.log.debug("Completed auto-attach with service level") elif sm.auto_attach: if not sm._set_auto_attach(): raise SubscriptionError("Setting auto-attach failed") else: - sm.log.info("Completed auto-attach") + sm.log.debug("Completed auto-attach") if sm.pools is not None: if type(sm.pools) is not list: - raise SubscriptionError("Pools must in the format of a " - "list.") + pool_fail = "Pools must in the format of a list" + raise SubscriptionError(pool_fail) + return_stat = sm.addPool(sm.pools) if not return_stat: raise SubscriptionError("Unable to attach pools {0}" @@ -66,8 +65,8 @@ def handle(_name, cfg, _cloud, log, _args): raise SubscriptionError("Unable to add or remove repos") sm.log.info("rh_subscription plugin completed successfully") except SubscriptionError as e: - sm.log.warn(e) - sm.log.info("rh_subscription plugin did not complete successfully") + sm.log.warn(str(e)) + sm.log.warn("rh_subscription plugin did not complete successfully") else: sm.log.info("System is already registered") @@ -91,7 +90,7 @@ class SubscriptionManager(object): self.enable_repo = self.rhel_cfg.get('enable-repo') self.disable_repo = self.rhel_cfg.get('disable-repo') self.servicelevel = self.rhel_cfg.get('service-level') - self.subman = ['/bin/subscription-manager'] + self.subman = ['subscription-manager'] self.valid_rh_keys = ['org', 'activation-key', 'username', 'password', 'disable-repo', 'enable-repo', 'add-pool', 'rhsm-baseurl', 'server-hostname', @@ -135,11 +134,22 @@ class SubscriptionManager(object): ''' cmd = list(itertools.chain(self.subman, ['identity'])) - if subprocess.call(cmd, stdout=open(os.devnull, 'wb'), - stderr=open(os.devnull, 'wb')) == 1: + try: + self._sub_man_cli(cmd) + except util.ProcessExecutionError: return False - else: - return True + + return True + + def _sub_man_cli(self, cmd, logstring_val=False): + ''' + Uses the prefered cloud-init subprocess def of util.subp + and runs subscription-manager. Breaking this to a + separate function for later use in mocking and unittests + ''' + return_out, return_err = util.subp(cmd, logstring=logstring_val) + + return return_out, return_err def rhn_register(self): ''' @@ -163,11 +173,13 @@ class SubscriptionManager(object): if self.server_hostname is not None: cmd.append("--serverurl={0}".format(self.server_hostname)) - return_msg, return_code = self._captureRun(cmd) - - if return_code is not 0: - self.log.warn("Registration with {0} and {1} failed.".format( - self.activation_key, self.org)) + try: + return_out, return_err = self._sub_man_cli(cmd, + logstring_val=True) + except util.ProcessExecutionError as e: + if e.stdout == "": + self.log.warn("Registration failed due " + "to: {0}".format(e.stderr)) return False elif (self.userid is not None) and (self.password is not None): @@ -186,14 +198,13 @@ class SubscriptionManager(object): cmd.append("--serverurl={0}".format(self.server_hostname)) # Attempting to register the system only - return_msg, return_code = self._captureRun(cmd) - - if return_code is not 0: - # Return message is in a set - if return_msg[0] == "": - self.log.warn("Registration failed") - if return_msg[1] is not "": - self.log.warn(return_msg[1]) + try: + return_out, return_err = self._sub_man_cli(cmd, + logstring_val=True) + except util.ProcessExecutionError as e: + if e.stdout == "": + self.log.warn("Registration failed due " + "to: {0}".format(e.stderr)) return False else: @@ -203,8 +214,8 @@ class SubscriptionManager(object): "and password") return False - reg_id = return_msg[0].split("ID: ")[1].rstrip() - self.log.info("Registered successfully with ID {0}".format(reg_id)) + reg_id = return_out.split("ID: ")[1].rstrip() + self.log.debug("Registered successfully with ID {0}".format(reg_id)) return True def _set_service_level(self): @@ -212,41 +223,34 @@ class SubscriptionManager(object): ['attach', '--auto', '--servicelevel={0}' .format(self.servicelevel)])) - return_msg, return_code = self._captureRun(cmd) - - if return_code is not 0: - self.log.warn("Setting the service level failed with: " - "{0}".format(return_msg[1].strip())) + try: + return_out, return_err = self._sub_man_cli(cmd) + except util.ProcessExecutionError as e: + if e.stdout.rstrip() != '': + for line in e.stdout.split("\n"): + if line is not '': + self.log.warn(line) + else: + self.log.warn("Setting the service level failed with: " + "{0}".format(e.stderr.strip())) return False - else: - for line in return_msg[0].split("\n"): - if line is not "": - self.log.info(line) - return True + for line in return_out.split("\n"): + if line is not "": + self.log.debug(line) + return True def _set_auto_attach(self): cmd = list(itertools.chain(self.subman, ['attach', '--auto'])) - return_msg, return_code = self._captureRun(cmd) - - if return_code is not 0: + try: + return_out, return_err = self._sub_man_cli(cmd) + except util.ProcessExecutionError: self.log.warn("Auto-attach failed with: " - "{0}]".format(return_msg[1].strip())) + "{0}]".format(return_err.strip())) return False - else: - for line in return_msg[0].split("\n"): - if line is not "": - self.log.info(line) - return True - - def _captureRun(self, cmd): - ''' - Subprocess command that captures and returns the output and - return code. - ''' - - r = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - return r.communicate(), r.returncode + for line in return_out.split("\n"): + if line is not "": + self.log.debug(line) + return True def _getPools(self): ''' @@ -259,14 +263,15 @@ class SubscriptionManager(object): # Get all available pools cmd = list(itertools.chain(self.subman, ['list', '--available', '--pool-only'])) - results = subprocess.check_output(cmd) + results, errors = self._sub_man_cli(cmd) available = (results.rstrip()).split("\n") - # Get all available pools + # Get all consumed pools cmd = list(itertools.chain(self.subman, ['list', '--consumed', '--pool-only'])) - results = subprocess.check_output(cmd) + results, errors = self._sub_man_cli(cmd) consumed = (results.rstrip()).split("\n") + return available, consumed def _getRepos(self): @@ -276,21 +281,19 @@ class SubscriptionManager(object): ''' cmd = list(itertools.chain(self.subman, ['repos', '--list-enabled'])) - result, return_code = self._captureRun(cmd) - + return_out, return_err = self._sub_man_cli(cmd) active_repos = [] - for repo in result[0].split("\n"): + for repo in return_out.split("\n"): if "Repo ID:" in repo: active_repos.append((repo.split(':')[1]).strip()) cmd = list(itertools.chain(self.subman, ['repos', '--list-disabled'])) - result, return_code = self._captureRun(cmd) + return_out, return_err = self._sub_man_cli(cmd) inactive_repos = [] - for repo in result[0].split("\n"): + for repo in return_out.split("\n"): if "Repo ID:" in repo: inactive_repos.append((repo.split(':')[1]).strip()) - return active_repos, inactive_repos def addPool(self, pools): @@ -301,7 +304,7 @@ class SubscriptionManager(object): # An empty list was passed if len(pools) == 0: - self.log.info("No pools to attach") + self.log.debug("No pools to attach") return True pool_available, pool_consumed = self._getPools() @@ -315,13 +318,14 @@ class SubscriptionManager(object): if len(pool_list) > 0: cmd.extend(pool_list) try: - self._captureRun(cmd) - self.log.info("Attached the following pools to your " - "system: %s" % (", ".join(pool_list)) - .replace('--pool=', '')) + self._sub_man_cli(cmd) + self.log.debug("Attached the following pools to your " + "system: %s" % (", ".join(pool_list)) + .replace('--pool=', '')) return True - except subprocess.CalledProcessError: - self.log.warn("Unable to attach pool {0}".format(pool)) + except util.ProcessExecutionError as e: + self.log.warn("Unable to attach pool {0} " + "due to {1}".format(pool, e)) return False def update_repos(self, erepos, drepos): @@ -341,7 +345,7 @@ class SubscriptionManager(object): # Bail if both lists are not populated if (len(erepos) == 0) and (len(drepos) == 0): - self.log.info("No repo IDs to enable or disable") + self.log.debug("No repo IDs to enable or disable") return True active_repos, inactive_repos = self._getRepos() @@ -368,14 +372,14 @@ class SubscriptionManager(object): for fail in enable_list_fail: # Check if the repo exists or not if fail in active_repos: - self.log.info("Repo {0} is already enabled".format(fail)) + self.log.debug("Repo {0} is already enabled".format(fail)) else: self.log.warn("Repo {0} does not appear to " "exist".format(fail)) if len(disable_list_fail) > 0: for fail in disable_list_fail: - self.log.info("Repo {0} not disabled " - "because it is not enabled".format(fail)) + self.log.debug("Repo {0} not disabled " + "because it is not enabled".format(fail)) cmd = list(itertools.chain(self.subman, ['repos'])) if enable_list > 0: @@ -384,16 +388,15 @@ class SubscriptionManager(object): cmd.extend(disable_list) try: - return_msg, return_code = self._captureRun(cmd) - - except subprocess.CalledProcessError as e: + self._sub_man_cli(cmd) + except util.ProcessExecutionError as e: self.log.warn("Unable to alter repos due to {0}".format(e)) return False if enable_list > 0: - self.log.info("Enabled the following repos: %s" % - (", ".join(enable_list)).replace('--enable=', '')) + self.log.debug("Enabled the following repos: %s" % + (", ".join(enable_list)).replace('--enable=', '')) if disable_list > 0: - self.log.info("Disabled the following repos: %s" % - (", ".join(disable_list)).replace('--disable=', '')) + self.log.debug("Disabled the following repos: %s" % + (", ".join(disable_list)).replace('--disable=', '')) return True diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py new file mode 100644 index 00000000..ba9181ec --- /dev/null +++ b/tests/unittests/test_rh_subscription.py @@ -0,0 +1,231 @@ +from cloudinit import util +from cloudinit.config import cc_rh_subscription +import logging +import mock +import unittest + + +class GoodTests(unittest.TestCase): + def setUp(self): + super(GoodTests, self).setUp() + self.name = "cc_rh_subscription" + self.cloud_init = None + self.log = logging.getLogger("good_tests") + self.args = [] + self.handle = cc_rh_subscription.handle + self.SM = cc_rh_subscription.SubscriptionManager + + self.config = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks' + }} + self.config_full = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'auto-attach': True, + 'service-level': 'self-support', + 'add-pool': ['pool1', 'pool2', 'pool3'], + 'enable-repo': ['repo1', 'repo2', 'repo3'], + 'disable-repo': ['repo4', 'repo5'] + }} + + def test_already_registered(self): + ''' + Emulates a system that is already registered. Ensure it gets + a non-ProcessExecution error from is_registered() + ''' + good_message = 'System is already registered' + with mock.patch.object(cc_rh_subscription.SubscriptionManager, + '_sub_man_cli') as mockobj: + self.log.info = mock.MagicMock(wraps=self.log.info) + self.handle(self.name, self.config, self.cloud_init, + self.log, self.args) + self.assertEqual(mockobj.call_count, 1) + self.log.info.assert_called_with(good_message) + + def test_simple_registration(self): + ''' + Simple registration with username and password + ''' + good_message = 'rh_subscription plugin completed successfully' + self.log.info = mock.MagicMock(wraps=self.log.info) + reg = "The system has been registered with ID:" \ + " 12345678-abde-abcde-1234-1234567890abc" + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError, (reg, 'bar')]) + self.handle(self.name, self.config, self.cloud_init, + self.log, self.args) + self.SM._sub_man_cli.assert_called_with_once(['subscription-manager', + 'identity']) + self.SM._sub_man_cli.assert_called_with_once( + ['subscription-manager', 'register', '--username=scooby@do.com', + '--password=scooby-snacks'], logstring_val=True) + + self.log.info.assert_called_with(good_message) + self.assertEqual(self.SM._sub_man_cli.call_count, 2) + + def test_full_registration(self): + ''' + Registration with auto-attach, service-level, adding pools, + and enabling and disabling yum repos + ''' + pool_message = 'Pool pool2 is not available' + repo_message1 = 'Repo repo1 is already enabled' + repo_message2 = 'Enabled the following repos: repo2, repo3' + good_message = 'rh_subscription plugin completed successfully' + self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.log.debug = mock.MagicMock(wraps=self.log.debug) + reg = "The system has been registered with ID:" \ + " 12345678-abde-abcde-1234-1234567890abc" + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError, (reg, 'bar'), + ('Service level set to: self-support', ''), + ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''), + ('Repo ID: repo1\nRepo ID: repo5\n', ''), + ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: ' + 'repo4', ''), + ('', '')]) + self.handle(self.name, self.config_full, self.cloud_init, + self.log, self.args) + self.log.warn.assert_any_call(pool_message) + self.log.debug.assert_any_call(repo_message1) + self.log.debug.assert_any_call(repo_message2) + self.log.info.assert_any_call(good_message) + self.SM._sub_man_cli.assert_called_with_once(['subscription-manager', + 'attach', '-pool=pool1', + '--pool=pool33']) + self.assertEqual(self.SM._sub_man_cli.call_count, 9) + + +class BadTests(unittest.TestCase): + def setUp(self): + super(BadTests, self).setUp() + self.name = "cc_rh_subscription" + self.cloud_init = None + self.log = logging.getLogger("bad_tests") + self.orig = self.log + self.args = [] + self.handle = cc_rh_subscription.handle + self.SM = cc_rh_subscription.SubscriptionManager + self.reg = "The system has been registered with ID:" \ + " 12345678-abde-abcde-1234-1234567890abc" + + self.config_no_password = {'rh_subscription': + {'username': 'scooby@do.com' + }} + + self.config_no_key = {'rh_subscription': + {'activation-key': '1234abcde', + }} + + self.config_service = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'service-level': 'self-support' + }} + + self.config_badpool = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'add-pool': 'not_a_list' + }} + self.config_badrepo = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'enable-repo': 'not_a_list' + }} + self.config_badkey = {'rh_subscription': + {'activation_key': 'abcdef1234', + 'org': '123', + }} + + def test_no_password(self): + ''' + Attempt to register without the password key/value + ''' + self.missing_info(self.config_no_password) + + def test_no_org(self): + ''' + Attempt to register without the org key/value + ''' + self.missing_info(self.config_no_key) + + def test_service_level_without_auto(self): + ''' + Attempt to register using service-level without the auto-attach key + ''' + good_message = 'The service-level key must be used in conjunction'\ + ' with the auto-attach key. Please re-run with '\ + 'auto-attach: True' + + self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + self.handle(self.name, self.config_service, self.cloud_init, + self.log, self.args) + self.log.warn.assert_any_call(good_message) + self.assertRaises(cc_rh_subscription.SubscriptionError) + self.assertEqual(self.SM._sub_man_cli.call_count, 1) + + def test_pool_not_a_list(self): + ''' + Register with pools that are not in the format of a list + ''' + good_message = "Pools must in the format of a list" + self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + self.handle(self.name, self.config_badpool, self.cloud_init, + self.log, self.args) + self.log.warn.assert_any_call(good_message) + self.assertRaises(cc_rh_subscription.SubscriptionError) + self.assertEqual(self.SM._sub_man_cli.call_count, 2) + + def test_repo_not_a_list(self): + ''' + Register with repos that are not in the format of a list + ''' + good_message = "Repo IDs must in the format of a list." + self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + self.handle(self.name, self.config_badrepo, self.cloud_init, + self.log, self.args) + self.log.warn.assert_any_call(good_message) + self.assertRaises(cc_rh_subscription.SubscriptionError) + self.assertEqual(self.SM._sub_man_cli.call_count, 2) + + def test_bad_key_value(self): + ''' + Attempt to register with a key that we don't know + ''' + good_message = 'activation_key is not a valid key for rh_subscription.'\ + ' Valid keys are: org, activation-key, username, '\ + 'password, disable-repo, enable-repo, add-pool,'\ + ' rhsm-baseurl, server-hostname, auto-attach, '\ + 'service-level' + self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) + self.handle(self.name, self.config_badkey, self.cloud_init, + self.log, self.args) + self.assertRaises(cc_rh_subscription.SubscriptionError) + self.log.warn.assert_any_call(good_message) + self.assertEqual(self.SM._sub_man_cli.call_count, 1) + + def missing_info(self, config): + ''' + Helper def for tests that having missing information + ''' + good_message = "Unable to register system due to incomplete "\ + "information." + self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM._sub_man_cli = mock.MagicMock( + side_effect=[util.ProcessExecutionError]) + self.handle(self.name, config, self.cloud_init, + self.log, self.args) + self.SM._sub_man_cli.assert_called_with(['subscription-manager', + 'identity']) + self.log.warn.assert_any_call(good_message) + self.assertEqual(self.SM._sub_man_cli.call_count, 1) -- cgit v1.2.3 From d9470a429935d4a5e12a5a3d1f57867362f92c57 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Wed, 27 May 2015 13:01:35 -0500 Subject: Updated files with upstream review comments thanks to Dan and Scott --- cloudinit/config/cc_rh_subscription.py | 108 ++++++++++---------- tests/unittests/test_rh_subscription.py | 169 ++++++++++++++------------------ 2 files changed, 128 insertions(+), 149 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 00a88456..db3d5525 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -1,6 +1,6 @@ # vi: ts=4 expandtab # -# Copyright (C) Red Hat, Inc. +# Copyright (C) 2015 Red Hat, Inc. # # Author: Brent Baude # @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import itertools from cloudinit import util @@ -51,10 +50,10 @@ def handle(_name, cfg, _cloud, log, _args): sm.log.debug("Completed auto-attach") if sm.pools is not None: - if type(sm.pools) is not list: + if not isinstance(sm.pools, (list)): pool_fail = "Pools must in the format of a list" raise SubscriptionError(pool_fail) - + return_stat = sm.addPool(sm.pools) if not return_stat: raise SubscriptionError("Unable to attach pools {0}" @@ -63,12 +62,12 @@ def handle(_name, cfg, _cloud, log, _args): return_stat = sm.update_repos(sm.enable_repo, sm.disable_repo) if not return_stat: raise SubscriptionError("Unable to add or remove repos") - sm.log.info("rh_subscription plugin completed successfully") + sm.log_sucess("rh_subscription plugin completed successfully") except SubscriptionError as e: - sm.log.warn(str(e)) - sm.log.warn("rh_subscription plugin did not complete successfully") + sm.log_warn(str(e)) + sm.log_warn("rh_subscription plugin did not complete successfully") else: - sm.log.info("System is already registered") + sm.log_sucess("System is already registered") class SubscriptionError(Exception): @@ -76,6 +75,11 @@ class SubscriptionError(Exception): class SubscriptionManager(object): + valid_rh_keys = ['org', 'activation-key', 'username', 'password', + 'disable-repo', 'enable-repo', 'add-pool', + 'rhsm-baseurl', 'server-hostname', + 'auto-attach', 'service-level'] + def __init__(self, cfg): self.cfg = cfg self.rhel_cfg = self.cfg.get('rh_subscription', {}) @@ -91,12 +95,16 @@ class SubscriptionManager(object): self.disable_repo = self.rhel_cfg.get('disable-repo') self.servicelevel = self.rhel_cfg.get('service-level') self.subman = ['subscription-manager'] - self.valid_rh_keys = ['org', 'activation-key', 'username', 'password', - 'disable-repo', 'enable-repo', 'add-pool', - 'rhsm-baseurl', 'server-hostname', - 'auto-attach', 'service-level'] self.is_registered = self._is_registered() + def log_sucess(self, msg): + '''Simple wrapper for logging info messages. Useful for unittests''' + self.log.info(msg) + + def log_warn(self, msg): + '''Simple wrapper for logging warning messages. Useful for unittests''' + self.log.warn(msg) + def _verify_keys(self): ''' Checks that the keys in the rh_subscription dict from the user-data @@ -112,14 +120,15 @@ class SubscriptionManager(object): # Check for bad auto-attach value if (self.auto_attach is not None) and \ - (str(self.auto_attach).upper() not in ['TRUE', 'FALSE']): + not (util.is_true(self.auto_attach) or + util.is_false(self.auto_attach)): not_bool = "The key auto-attach must be a value of "\ "either True or False" return False, not_bool if (self.servicelevel is not None) and \ - ((not self.auto_attach) or - (str(self.auto_attach).upper() == "FALSE")): + ((not self.auto_attach) + or (util.is_false(str(self.auto_attach)))): no_auto = "The service-level key must be used in conjunction with "\ "the auto-attach key. Please re-run with auto-attach: "\ @@ -132,7 +141,7 @@ class SubscriptionManager(object): Checks if the system is already registered and returns True if so, else False ''' - cmd = list(itertools.chain(self.subman, ['identity'])) + cmd = ['identity'] try: self._sub_man_cli(cmd) @@ -147,9 +156,8 @@ class SubscriptionManager(object): and runs subscription-manager. Breaking this to a separate function for later use in mocking and unittests ''' - return_out, return_err = util.subp(cmd, logstring=logstring_val) - - return return_out, return_err + cmd = self.subman + cmd + return util.subp(cmd, logstring=logstring_val) def rhn_register(self): ''' @@ -159,10 +167,8 @@ class SubscriptionManager(object): if (self.activation_key is not None) and (self.org is not None): # register by activation key - cmd = list(itertools.chain(self.subman, ['register', - '--activationkey={0}'. - format(self.activation_key), - '--org={0}'.format(self.org)])) + cmd = ['register', '--activationkey={0}'. + format(self.activation_key), '--org={0}'.format(self.org)] # If the baseurl and/or server url are passed in, we register # with them. @@ -178,15 +184,14 @@ class SubscriptionManager(object): logstring_val=True) except util.ProcessExecutionError as e: if e.stdout == "": - self.log.warn("Registration failed due " + self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) return False elif (self.userid is not None) and (self.password is not None): # register by username and password - cmd = list(itertools.chain(self.subman, ['register', - '--username={0}'.format(self.userid), - '--password={0}'.format(self.password)])) + cmd = ['register', '--username={0}'.format(self.userid), + '--password={0}'.format(self.password)] # If the baseurl and/or server url are passed in, we register # with them. @@ -203,14 +208,14 @@ class SubscriptionManager(object): logstring_val=True) except util.ProcessExecutionError as e: if e.stdout == "": - self.log.warn("Registration failed due " + self.log_warn("Registration failed due " "to: {0}".format(e.stderr)) return False else: - self.log.warn("Unable to register system due to incomplete " + self.log_warn("Unable to register system due to incomplete " "information.") - self.log.warn("Use either activationkey and org *or* userid " + self.log_warn("Use either activationkey and org *or* userid " "and password") return False @@ -219,9 +224,8 @@ class SubscriptionManager(object): return True def _set_service_level(self): - cmd = list(itertools.chain(self.subman, - ['attach', '--auto', '--servicelevel={0}' - .format(self.servicelevel)])) + cmd = ['attach', '--auto', '--servicelevel={0}' + .format(self.servicelevel)] try: return_out, return_err = self._sub_man_cli(cmd) @@ -229,9 +233,9 @@ class SubscriptionManager(object): if e.stdout.rstrip() != '': for line in e.stdout.split("\n"): if line is not '': - self.log.warn(line) + self.log_warn(line) else: - self.log.warn("Setting the service level failed with: " + self.log_warn("Setting the service level failed with: " "{0}".format(e.stderr.strip())) return False for line in return_out.split("\n"): @@ -240,11 +244,11 @@ class SubscriptionManager(object): return True def _set_auto_attach(self): - cmd = list(itertools.chain(self.subman, ['attach', '--auto'])) + cmd = ['attach', '--auto'] try: return_out, return_err = self._sub_man_cli(cmd) except util.ProcessExecutionError: - self.log.warn("Auto-attach failed with: " + self.log_warn("Auto-attach failed with: " "{0}]".format(return_err.strip())) return False for line in return_out.split("\n"): @@ -261,14 +265,12 @@ class SubscriptionManager(object): consumed = [] # Get all available pools - cmd = list(itertools.chain(self.subman, ['list', '--available', - '--pool-only'])) + cmd = ['list', '--available', '--pool-only'] results, errors = self._sub_man_cli(cmd) available = (results.rstrip()).split("\n") # Get all consumed pools - cmd = list(itertools.chain(self.subman, ['list', '--consumed', - '--pool-only'])) + cmd = ['list', '--consumed', '--pool-only'] results, errors = self._sub_man_cli(cmd) consumed = (results.rstrip()).split("\n") @@ -280,14 +282,14 @@ class SubscriptionManager(object): them in list form. ''' - cmd = list(itertools.chain(self.subman, ['repos', '--list-enabled'])) + cmd = ['repos', '--list-enabled'] return_out, return_err = self._sub_man_cli(cmd) active_repos = [] for repo in return_out.split("\n"): if "Repo ID:" in repo: active_repos.append((repo.split(':')[1]).strip()) - cmd = list(itertools.chain(self.subman, ['repos', '--list-disabled'])) + cmd = ['repos', '--list-disabled'] return_out, return_err = self._sub_man_cli(cmd) inactive_repos = [] @@ -309,12 +311,12 @@ class SubscriptionManager(object): pool_available, pool_consumed = self._getPools() pool_list = [] - cmd = list(itertools.chain(self.subman, ['attach'])) + cmd = ['attach'] for pool in pools: if (pool not in pool_consumed) and (pool in pool_available): pool_list.append('--pool={0}'.format(pool)) else: - self.log.warn("Pool {0} is not available".format(pool)) + self.log_warn("Pool {0} is not available".format(pool)) if len(pool_list) > 0: cmd.extend(pool_list) try: @@ -324,7 +326,7 @@ class SubscriptionManager(object): .replace('--pool=', '')) return True except util.ProcessExecutionError as e: - self.log.warn("Unable to attach pool {0} " + self.log_warn("Unable to attach pool {0} " "due to {1}".format(pool, e)) return False @@ -335,12 +337,12 @@ class SubscriptionManager(object): executes the action to disable or enable ''' - if (erepos is not None) and (type(erepos) is not list): - self.log.warn("Repo IDs must in the format of a list.") + if (erepos is not None) and (not isinstance(erepos, (list))): + self.log_warn("Repo IDs must in the format of a list.") return False - if (drepos is not None) and (type(drepos) is not list): - self.log.warn("Repo IDs must in the format of a list.") + if (drepos is not None) and (not isinstance(drepos, (list))): + self.log_warn("Repo IDs must in the format of a list.") return False # Bail if both lists are not populated @@ -374,14 +376,14 @@ class SubscriptionManager(object): if fail in active_repos: self.log.debug("Repo {0} is already enabled".format(fail)) else: - self.log.warn("Repo {0} does not appear to " + self.log_warn("Repo {0} does not appear to " "exist".format(fail)) if len(disable_list_fail) > 0: for fail in disable_list_fail: self.log.debug("Repo {0} not disabled " "because it is not enabled".format(fail)) - cmd = list(itertools.chain(self.subman, ['repos'])) + cmd = ['repos'] if enable_list > 0: cmd.extend(enable_list) if disable_list > 0: @@ -390,7 +392,7 @@ class SubscriptionManager(object): try: self._sub_man_cli(cmd) except util.ProcessExecutionError as e: - self.log.warn("Unable to alter repos due to {0}".format(e)) + self.log_warn("Unable to alter repos due to {0}".format(e)) return False if enable_list > 0: diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index ba9181ec..2f813f41 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -34,34 +34,33 @@ class GoodTests(unittest.TestCase): Emulates a system that is already registered. Ensure it gets a non-ProcessExecution error from is_registered() ''' - good_message = 'System is already registered' with mock.patch.object(cc_rh_subscription.SubscriptionManager, '_sub_man_cli') as mockobj: - self.log.info = mock.MagicMock(wraps=self.log.info) + self.SM.log_sucess = mock.MagicMock() self.handle(self.name, self.config, self.cloud_init, self.log, self.args) + self.assertEqual(self.SM.log_sucess.call_count, 1) self.assertEqual(mockobj.call_count, 1) - self.log.info.assert_called_with(good_message) def test_simple_registration(self): ''' Simple registration with username and password ''' - good_message = 'rh_subscription plugin completed successfully' - self.log.info = mock.MagicMock(wraps=self.log.info) + self.SM.log_sucess = mock.MagicMock() reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" self.SM._sub_man_cli = mock.MagicMock( side_effect=[util.ProcessExecutionError, (reg, 'bar')]) self.handle(self.name, self.config, self.cloud_init, self.log, self.args) - self.SM._sub_man_cli.assert_called_with_once(['subscription-manager', - 'identity']) - self.SM._sub_man_cli.assert_called_with_once( - ['subscription-manager', 'register', '--username=scooby@do.com', - '--password=scooby-snacks'], logstring_val=True) - - self.log.info.assert_called_with(good_message) + self.assertIn(mock.call(['identity']), + self.SM._sub_man_cli.call_args_list) + self.assertIn(mock.call(['register', '--username=scooby@do.com', + '--password=scooby-snacks'], + logstring_val=True), + self.SM._sub_man_cli.call_args_list) + + self.assertEqual(self.SM.log_sucess.call_count, 1) self.assertEqual(self.SM._sub_man_cli.call_count, 2) def test_full_registration(self): @@ -69,12 +68,12 @@ class GoodTests(unittest.TestCase): Registration with auto-attach, service-level, adding pools, and enabling and disabling yum repos ''' - pool_message = 'Pool pool2 is not available' - repo_message1 = 'Repo repo1 is already enabled' - repo_message2 = 'Enabled the following repos: repo2, repo3' - good_message = 'rh_subscription plugin completed successfully' - self.log.warn = mock.MagicMock(wraps=self.log.warn) - self.log.debug = mock.MagicMock(wraps=self.log.debug) + call_lists = [] + call_lists.append(['attach', '--pool=pool1', '--pool=pool3']) + call_lists.append(['repos', '--enable=repo2', '--enable=repo3', + '--disable=repo5']) + call_lists.append(['attach', '--auto', '--servicelevel=self-support']) + self.SM.log_sucess = mock.MagicMock() reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" self.SM._sub_man_cli = mock.MagicMock( @@ -87,145 +86,123 @@ class GoodTests(unittest.TestCase): ('', '')]) self.handle(self.name, self.config_full, self.cloud_init, self.log, self.args) - self.log.warn.assert_any_call(pool_message) - self.log.debug.assert_any_call(repo_message1) - self.log.debug.assert_any_call(repo_message2) - self.log.info.assert_any_call(good_message) - self.SM._sub_man_cli.assert_called_with_once(['subscription-manager', - 'attach', '-pool=pool1', - '--pool=pool33']) + for call in call_lists: + self.assertIn(mock.call(call), self.SM._sub_man_cli.call_args_list) + self.assertEqual(self.SM.log_sucess.call_count, 1) self.assertEqual(self.SM._sub_man_cli.call_count, 9) -class BadTests(unittest.TestCase): +class TestBadInput(unittest.TestCase): + name = "cc_rh_subscription" + cloud_init = None + log = logging.getLogger("bad_tests") + args = [] + SM = cc_rh_subscription.SubscriptionManager + reg = "The system has been registered with ID:" \ + " 12345678-abde-abcde-1234-1234567890abc" + + config_no_password = {'rh_subscription': + {'username': 'scooby@do.com' + }} + + config_no_key = {'rh_subscription': + {'activation-key': '1234abcde', + }} + + config_service = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'service-level': 'self-support' + }} + + config_badpool = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'add-pool': 'not_a_list' + }} + config_badrepo = {'rh_subscription': + {'username': 'scooby@do.com', + 'password': 'scooby-snacks', + 'enable-repo': 'not_a_list' + }} + config_badkey = {'rh_subscription': + {'activation_key': 'abcdef1234', + 'org': '123', + }} + def setUp(self): - super(BadTests, self).setUp() - self.name = "cc_rh_subscription" - self.cloud_init = None - self.log = logging.getLogger("bad_tests") - self.orig = self.log - self.args = [] + super(TestBadInput, self).setUp() self.handle = cc_rh_subscription.handle - self.SM = cc_rh_subscription.SubscriptionManager - self.reg = "The system has been registered with ID:" \ - " 12345678-abde-abcde-1234-1234567890abc" - - self.config_no_password = {'rh_subscription': - {'username': 'scooby@do.com' - }} - - self.config_no_key = {'rh_subscription': - {'activation-key': '1234abcde', - }} - - self.config_service = {'rh_subscription': - {'username': 'scooby@do.com', - 'password': 'scooby-snacks', - 'service-level': 'self-support' - }} - - self.config_badpool = {'rh_subscription': - {'username': 'scooby@do.com', - 'password': 'scooby-snacks', - 'add-pool': 'not_a_list' - }} - self.config_badrepo = {'rh_subscription': - {'username': 'scooby@do.com', - 'password': 'scooby-snacks', - 'enable-repo': 'not_a_list' - }} - self.config_badkey = {'rh_subscription': - {'activation_key': 'abcdef1234', - 'org': '123', - }} def test_no_password(self): ''' Attempt to register without the password key/value ''' - self.missing_info(self.config_no_password) + self.input_is_missing_data(self.config_no_password) def test_no_org(self): ''' Attempt to register without the org key/value ''' - self.missing_info(self.config_no_key) + self.input_is_missing_data(self.config_no_key) def test_service_level_without_auto(self): ''' Attempt to register using service-level without the auto-attach key ''' - good_message = 'The service-level key must be used in conjunction'\ - ' with the auto-attach key. Please re-run with '\ - 'auto-attach: True' - - self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM.log_warn = mock.MagicMock() self.SM._sub_man_cli = mock.MagicMock( side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) self.handle(self.name, self.config_service, self.cloud_init, self.log, self.args) - self.log.warn.assert_any_call(good_message) - self.assertRaises(cc_rh_subscription.SubscriptionError) self.assertEqual(self.SM._sub_man_cli.call_count, 1) + self.assertEqual(self.SM.log_warn.call_count, 2) def test_pool_not_a_list(self): ''' Register with pools that are not in the format of a list ''' - good_message = "Pools must in the format of a list" - self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM.log_warn = mock.MagicMock() self.SM._sub_man_cli = mock.MagicMock( side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) self.handle(self.name, self.config_badpool, self.cloud_init, self.log, self.args) - self.log.warn.assert_any_call(good_message) - self.assertRaises(cc_rh_subscription.SubscriptionError) self.assertEqual(self.SM._sub_man_cli.call_count, 2) + self.assertEqual(self.SM.log_warn.call_count, 2) def test_repo_not_a_list(self): ''' Register with repos that are not in the format of a list ''' - good_message = "Repo IDs must in the format of a list." - self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM.log_warn = mock.MagicMock() self.SM._sub_man_cli = mock.MagicMock( side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) self.handle(self.name, self.config_badrepo, self.cloud_init, self.log, self.args) - self.log.warn.assert_any_call(good_message) - self.assertRaises(cc_rh_subscription.SubscriptionError) + self.assertEqual(self.SM.log_warn.call_count, 3) self.assertEqual(self.SM._sub_man_cli.call_count, 2) def test_bad_key_value(self): ''' Attempt to register with a key that we don't know ''' - good_message = 'activation_key is not a valid key for rh_subscription.'\ - ' Valid keys are: org, activation-key, username, '\ - 'password, disable-repo, enable-repo, add-pool,'\ - ' rhsm-baseurl, server-hostname, auto-attach, '\ - 'service-level' - self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM.log_warn = mock.MagicMock() self.SM._sub_man_cli = mock.MagicMock( side_effect=[util.ProcessExecutionError, (self.reg, 'bar')]) self.handle(self.name, self.config_badkey, self.cloud_init, self.log, self.args) - self.assertRaises(cc_rh_subscription.SubscriptionError) - self.log.warn.assert_any_call(good_message) + self.assertEqual(self.SM.log_warn.call_count, 2) self.assertEqual(self.SM._sub_man_cli.call_count, 1) - def missing_info(self, config): + def input_is_missing_data(self, config): ''' Helper def for tests that having missing information ''' - good_message = "Unable to register system due to incomplete "\ - "information." - self.log.warn = mock.MagicMock(wraps=self.log.warn) + self.SM.log_warn = mock.MagicMock() self.SM._sub_man_cli = mock.MagicMock( side_effect=[util.ProcessExecutionError]) self.handle(self.name, config, self.cloud_init, self.log, self.args) - self.SM._sub_man_cli.assert_called_with(['subscription-manager', - 'identity']) - self.log.warn.assert_any_call(good_message) + self.SM._sub_man_cli.assert_called_with(['identity']) + self.assertEqual(self.SM.log_warn.call_count, 4) self.assertEqual(self.SM._sub_man_cli.call_count, 1) -- cgit v1.2.3 From 3aa0fcc5983416d743fac6af1d40ca791feb23af Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Thu, 28 May 2015 09:02:11 -0500 Subject: Tightening up an error message and isinstance usage based on feedback from Dan --- cloudinit/config/cc_rh_subscription.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index db3d5525..e57e8a07 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -50,7 +50,7 @@ def handle(_name, cfg, _cloud, log, _args): sm.log.debug("Completed auto-attach") if sm.pools is not None: - if not isinstance(sm.pools, (list)): + if not isinstance(sm.pools, list): pool_fail = "Pools must in the format of a list" raise SubscriptionError(pool_fail) @@ -122,8 +122,8 @@ class SubscriptionManager(object): if (self.auto_attach is not None) and \ not (util.is_true(self.auto_attach) or util.is_false(self.auto_attach)): - not_bool = "The key auto-attach must be a value of "\ - "either True or False" + not_bool = "The key auto-attach must be a boolean value "\ + "(True/False " return False, not_bool if (self.servicelevel is not None) and \ @@ -337,11 +337,11 @@ class SubscriptionManager(object): executes the action to disable or enable ''' - if (erepos is not None) and (not isinstance(erepos, (list))): + if (erepos is not None) and (not isinstance(erepos, list)): self.log_warn("Repo IDs must in the format of a list.") return False - if (drepos is not None) and (not isinstance(drepos, (list))): + if (drepos is not None) and (not isinstance(drepos, list)): self.log_warn("Repo IDs must in the format of a list.") return False -- cgit v1.2.3 From 3c01b8e48400697362f190984ab9c96dee27a369 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Fri, 29 May 2015 09:18:49 -0500 Subject: Corrected spelling error on variable name --- cloudinit/config/cc_rh_subscription.py | 6 +++--- tests/unittests/test_rh_subscription.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index e57e8a07..cabebca4 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -62,12 +62,12 @@ def handle(_name, cfg, _cloud, log, _args): return_stat = sm.update_repos(sm.enable_repo, sm.disable_repo) if not return_stat: raise SubscriptionError("Unable to add or remove repos") - sm.log_sucess("rh_subscription plugin completed successfully") + sm.log_success("rh_subscription plugin completed successfully") except SubscriptionError as e: sm.log_warn(str(e)) sm.log_warn("rh_subscription plugin did not complete successfully") else: - sm.log_sucess("System is already registered") + sm.log_success("System is already registered") class SubscriptionError(Exception): @@ -97,7 +97,7 @@ class SubscriptionManager(object): self.subman = ['subscription-manager'] self.is_registered = self._is_registered() - def log_sucess(self, msg): + def log_success(self, msg): '''Simple wrapper for logging info messages. Useful for unittests''' self.log.info(msg) diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py index 2f813f41..38d5763a 100644 --- a/tests/unittests/test_rh_subscription.py +++ b/tests/unittests/test_rh_subscription.py @@ -36,17 +36,17 @@ class GoodTests(unittest.TestCase): ''' with mock.patch.object(cc_rh_subscription.SubscriptionManager, '_sub_man_cli') as mockobj: - self.SM.log_sucess = mock.MagicMock() + self.SM.log_success = mock.MagicMock() self.handle(self.name, self.config, self.cloud_init, self.log, self.args) - self.assertEqual(self.SM.log_sucess.call_count, 1) + self.assertEqual(self.SM.log_success.call_count, 1) self.assertEqual(mockobj.call_count, 1) def test_simple_registration(self): ''' Simple registration with username and password ''' - self.SM.log_sucess = mock.MagicMock() + self.SM.log_success = mock.MagicMock() reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" self.SM._sub_man_cli = mock.MagicMock( @@ -60,7 +60,7 @@ class GoodTests(unittest.TestCase): logstring_val=True), self.SM._sub_man_cli.call_args_list) - self.assertEqual(self.SM.log_sucess.call_count, 1) + self.assertEqual(self.SM.log_success.call_count, 1) self.assertEqual(self.SM._sub_man_cli.call_count, 2) def test_full_registration(self): @@ -73,7 +73,7 @@ class GoodTests(unittest.TestCase): call_lists.append(['repos', '--enable=repo2', '--enable=repo3', '--disable=repo5']) call_lists.append(['attach', '--auto', '--servicelevel=self-support']) - self.SM.log_sucess = mock.MagicMock() + self.SM.log_success = mock.MagicMock() reg = "The system has been registered with ID:" \ " 12345678-abde-abcde-1234-1234567890abc" self.SM._sub_man_cli = mock.MagicMock( @@ -88,7 +88,7 @@ class GoodTests(unittest.TestCase): self.log, self.args) for call in call_lists: self.assertIn(mock.call(call), self.SM._sub_man_cli.call_args_list) - self.assertEqual(self.SM.log_sucess.call_count, 1) + self.assertEqual(self.SM.log_success.call_count, 1) self.assertEqual(self.SM._sub_man_cli.call_count, 9) -- cgit v1.2.3 From ad403f27e1f8067d7709ed9b184589af8309ba15 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 15 Jun 2015 15:34:03 -0400 Subject: cc_rh_subscription: fixes for python3 --- cloudinit/config/cc_rh_subscription.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index cabebca4..6da26d25 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -384,9 +384,9 @@ class SubscriptionManager(object): "because it is not enabled".format(fail)) cmd = ['repos'] - if enable_list > 0: + if len(enable_list) > 0: cmd.extend(enable_list) - if disable_list > 0: + if len(disable_list) > 0: cmd.extend(disable_list) try: @@ -395,10 +395,10 @@ class SubscriptionManager(object): self.log_warn("Unable to alter repos due to {0}".format(e)) return False - if enable_list > 0: + if len(enable_list) > 0: self.log.debug("Enabled the following repos: %s" % (", ".join(enable_list)).replace('--enable=', '')) - if disable_list > 0: + if len(disable_list) > 0: self.log.debug("Disabled the following repos: %s" % (", ".join(disable_list)).replace('--disable=', '')) return True -- cgit v1.2.3 From 66c13ab5aca67ca3aa3d1536154989f98b85107a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 15 Jun 2015 17:20:51 -0400 Subject: apt_configure: fix importing of apt gpg keys under in python3 LP: #1463373 --- ChangeLog | 1 + cloudinit/config/cc_apt_configure.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index cc12cb6e..6261147e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -48,6 +48,7 @@ [Lars Kellogg-Stedman] - Add an rh_subscription module to handle registration of Red Hat instances. [Brent Baude] + - cc_apt_configure: fix importing keys under python3 (LP: #1463373) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2c51d116..9e9e9e26 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -109,7 +109,7 @@ def handle(name, cfg, cloud, log, _args): # get gpg keyid from keyserver def getkeybyid(keyid, keyserver): - with util.ExtendedTemporaryFile(suffix='.sh') as fh: + with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh: fh.write(EXPORT_GPG_KEYID) fh.flush() cmd = ['/bin/sh', fh.name, keyid, keyserver] -- cgit v1.2.3 From 6e06afffed8614cb143e3a13bab5aa382ccbbce9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 16 Jun 2015 11:18:33 -0400 Subject: growpart: fix specification of 'devices' list. given config: {'growpart': {'devices': ["/"]}} the 'devices' was ignored, it was incorrectly read from the top level non-namespaced location. LP: #1465436 --- ChangeLog | 1 + cloudinit/config/cc_growpart.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index 6261147e..47b8dec2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -49,6 +49,7 @@ - Add an rh_subscription module to handle registration of Red Hat instances. [Brent Baude] - cc_apt_configure: fix importing keys under python3 (LP: #1463373) + - cc_growpart: fix specification of 'devices' list (LP: #1465436) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index f52c41f0..859d69f1 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -276,7 +276,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("use ignore_growroot_disabled to ignore") return - devices = util.get_cfg_option_list(cfg, "devices", ["/"]) + devices = util.get_cfg_option_list(mycfg, "devices", ["/"]) if not len(devices): log.debug("growpart: empty device list") return -- cgit v1.2.3 From 7e9e07608f33f57b620b2dca78cf0e1d9da4d53f Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Mon, 29 Jun 2015 10:58:59 -0400 Subject: - Fix logic change introduced by 1000.1.1 --- cloudinit/config/cc_rightscale_userdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 24880d13..0ecf3a4d 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -58,7 +58,7 @@ def handle(name, _cfg, cloud, log, _args): try: mdict = parse_qs(ud) - if mdict or MY_HOOKNAME not in mdict: + if not mdict or MY_HOOKNAME not in mdict: log.debug(("Skipping module %s, " "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) -- cgit v1.2.3 From 55487d9eb52343bd72271b072734740b52b25c1d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Refactor cc_mounts.sanitize_devname to make it easier to modify. --- cloudinit/config/cc_mounts.py | 104 +++++++--------- .../unittests/test_handler/test_handler_mounts.py | 133 +++++++++++++++++++++ 2 files changed, 177 insertions(+), 60 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_mounts.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 1cb1e839..f970c2ca 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -28,15 +28,15 @@ from cloudinit import type_utils from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1, sr0 -SHORTNAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" -SHORTNAME = re.compile(SHORTNAME_FILTER) +DEVICE_NAME_FILTER = r"^([x]{0,1}[shv]d[a-z][0-9]*|sr[0-9]+)$" +DEVICE_NAME_RE = re.compile(DEVICE_NAME_FILTER) WS = re.compile("[%s]+" % (whitespace)) FSTAB_PATH = "/etc/fstab" LOG = logging.getLogger(__name__) -def is_mdname(name): +def is_meta_device_name(name): # return true if this is a metadata service name if name in ["ami", "root", "swap"]: return True @@ -48,6 +48,23 @@ def is_mdname(name): return False +def _get_nth_partition_for_device(device_path, partition_number): + potential_suffixes = [str(partition_number), 'p%s' % (partition_number,)] + for suffix in potential_suffixes: + potential_partition_device = '%s%s' % (device_path, suffix) + if os.path.exists(potential_partition_device): + return potential_partition_device + return None + + +def _is_block_device(device_path, partition_path=None): + device_name = device_path.split('/')[-1] + sys_path = os.path.join('/sys/block/', device_name) + if partition_path is not None: + sys_path = os.path.join(sys_path, partition_path.split('/')[-1]) + return os.path.exists(sys_path) + + def sanitize_devname(startname, transformer, log): log.debug("Attempting to determine the real name of %s", startname) @@ -58,21 +75,34 @@ def sanitize_devname(startname, transformer, log): devname = "ephemeral0" log.debug("Adjusted mount option from ephemeral to ephemeral0") - (blockdev, part) = util.expand_dotted_devname(devname) + device_path, partition_number = util.expand_dotted_devname(devname) - if is_mdname(blockdev): - orig = blockdev - blockdev = transformer(blockdev) - if not blockdev: + if is_meta_device_name(device_path): + orig = device_path + device_path = transformer(device_path) + if not device_path: return None - if not blockdev.startswith("/"): - blockdev = "/dev/%s" % blockdev - log.debug("Mapped metadata name %s to %s", orig, blockdev) + if not device_path.startswith("/"): + device_path = "/dev/%s" % (device_path,) + log.debug("Mapped metadata name %s to %s", orig, device_path) + else: + if DEVICE_NAME_RE.match(startname): + device_path = "/dev/%s" % (device_path,) + + partition_path = None + if partition_number is None: + partition_path = _get_nth_partition_for_device(device_path, 1) else: - if SHORTNAME.match(startname): - blockdev = "/dev/%s" % blockdev + partition_path = _get_nth_partition_for_device(device_path, + partition_number) + if partition_path is None: + return None - return devnode_for_dev_part(blockdev, part) + if _is_block_device(device_path, partition_path): + if partition_path is not None: + return partition_path + return device_path + return None def suggested_swapsize(memsize=None, maxsize=None, fsys=None): @@ -366,49 +396,3 @@ def handle(_name, cfg, cloud, log, _args): util.subp(("mount", "-a")) except: util.logexc(log, "Activating mounts via 'mount -a' failed") - - -def devnode_for_dev_part(device, partition): - """ - Find the name of the partition. While this might seem rather - straight forward, its not since some devices are '' - while others are 'p'. For example, /dev/xvda3 on EC2 - will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is - a block device. - """ - if not os.path.exists(device): - return None - - short_name = os.path.basename(device) - sys_path = "/sys/block/%s" % short_name - - if not os.path.exists(sys_path): - LOG.debug("did not find entry for %s in /sys/block", short_name) - return None - - sys_long_path = sys_path + "/" + short_name - - if partition is not None: - partition = str(partition) - - if partition is None: - valid_mappings = [sys_long_path + "1", sys_long_path + "p1"] - elif partition != "0": - valid_mappings = [sys_long_path + "%s" % partition, - sys_long_path + "p%s" % partition] - else: - valid_mappings = [] - - for cdisk in valid_mappings: - if not os.path.exists(cdisk): - continue - - dev_path = "/dev/%s" % os.path.basename(cdisk) - if os.path.exists(dev_path): - return dev_path - - if partition is None or partition == "0": - return device - - LOG.debug("Did not fine partition %s for device %s", partition, device) - return None diff --git a/tests/unittests/test_handler/test_handler_mounts.py b/tests/unittests/test_handler/test_handler_mounts.py new file mode 100644 index 00000000..355674b2 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_mounts.py @@ -0,0 +1,133 @@ +import os.path +import shutil +import tempfile + +from cloudinit.config import cc_mounts + +from .. import helpers as test_helpers + +try: + from unittest import mock +except ImportError: + import mock + + +class TestSanitizeDevname(test_helpers.FilesystemMockingTestCase): + + def setUp(self): + super(TestSanitizeDevname, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + self.patchOS(self.new_root) + + def _touch(self, path): + path = os.path.join(self.new_root, path.lstrip('/')) + basedir = os.path.dirname(path) + if not os.path.exists(basedir): + os.makedirs(basedir) + open(path, 'a').close() + + def _makedirs(self, directory): + directory = os.path.join(self.new_root, directory.lstrip('/')) + if not os.path.exists(directory): + os.makedirs(directory) + + def mock_existence_of_disk(self, disk_path): + self._touch(disk_path) + self._makedirs(os.path.join('/sys/block', disk_path.split('/')[-1])) + + def mock_existence_of_partition(self, disk_path, partition_number): + self.mock_existence_of_disk(disk_path) + self._touch(disk_path + str(partition_number)) + disk_name = disk_path.split('/')[-1] + self._makedirs(os.path.join('/sys/block', + disk_name, + disk_name + str(partition_number))) + + def test_existent_full_disk_path_is_returned(self): + disk_path = '/dev/sda' + self.mock_existence_of_disk(disk_path) + self.assertEqual(disk_path, + cc_mounts.sanitize_devname(disk_path, + lambda x: None, + mock.Mock())) + + def test_existent_disk_name_returns_full_path(self): + disk_name = 'sda' + disk_path = '/dev/' + disk_name + self.mock_existence_of_disk(disk_path) + self.assertEqual(disk_path, + cc_mounts.sanitize_devname(disk_name, + lambda x: None, + mock.Mock())) + + def test_existent_meta_disk_is_returned(self): + actual_disk_path = '/dev/sda' + self.mock_existence_of_disk(actual_disk_path) + self.assertEqual( + actual_disk_path, + cc_mounts.sanitize_devname('ephemeral0', + lambda x: actual_disk_path, + mock.Mock())) + + def test_existent_meta_partition_is_returned(self): + disk_name, partition_part = '/dev/sda', '1' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0.1', + lambda x: disk_name, + mock.Mock())) + + def test_existent_meta_partition_with_p_is_returned(self): + disk_name, partition_part = '/dev/sda', 'p1' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0.1', + lambda x: disk_name, + mock.Mock())) + + def test_first_partition_returned_if_existent_disk_is_partitioned(self): + disk_name, partition_part = '/dev/sda', '1' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0', + lambda x: disk_name, + mock.Mock())) + + def test_nth_partition_returned_if_requested(self): + disk_name, partition_part = '/dev/sda', '3' + actual_partition_path = disk_name + partition_part + self.mock_existence_of_partition(disk_name, partition_part) + self.assertEqual( + actual_partition_path, + cc_mounts.sanitize_devname('ephemeral0.3', + lambda x: disk_name, + mock.Mock())) + + def test_transformer_returning_none_returns_none(self): + self.assertIsNone( + cc_mounts.sanitize_devname( + 'ephemeral0', lambda x: None, mock.Mock())) + + def test_missing_device_returns_none(self): + self.assertIsNone( + cc_mounts.sanitize_devname('/dev/sda', None, mock.Mock())) + + def test_missing_sys_returns_none(self): + disk_path = '/dev/sda' + self._makedirs(disk_path) + self.assertIsNone( + cc_mounts.sanitize_devname(disk_path, None, mock.Mock())) + + def test_existent_disk_but_missing_partition_returns_none(self): + disk_path = '/dev/sda' + self.mock_existence_of_disk(disk_path) + self.assertIsNone( + cc_mounts.sanitize_devname( + 'ephemeral0.1', lambda x: disk_path, mock.Mock())) -- cgit v1.2.3 From edc46ee7192376af65640a81c39335ebdfd196b6 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 21 Jul 2015 13:06:11 +0100 Subject: Extend disk_setup and mounts to handle /dev/disk symlinks. --- cloudinit/config/cc_disk_setup.py | 5 +++++ cloudinit/config/cc_mounts.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index e2ce6db4..92fa7a94 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -648,6 +648,8 @@ def mkpart(device, definition): table_type: Which partition table to use, defaults to MBR device: the device to work on. """ + LOG.debug('Ensuring that we have a real device, not a symbolic link') + device = os.path.realpath(device) LOG.debug("Checking values for %s definition" % device) overwrite = definition.get('overwrite', False) @@ -745,6 +747,9 @@ def mkfs(fs_cfg): fs_replace = fs_cfg.get('replace_fs', False) overwrite = fs_cfg.get('overwrite', False) + LOG.debug('Ensuring that we have a real device, not a symbolic link') + device = os.path.realpath(device) + # This allows you to define the default ephemeral or swap LOG.debug("Checking %s against default devices", device) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index f970c2ca..73b42f91 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -49,7 +49,8 @@ def is_meta_device_name(name): def _get_nth_partition_for_device(device_path, partition_number): - potential_suffixes = [str(partition_number), 'p%s' % (partition_number,)] + potential_suffixes = [str(partition_number), 'p%s' % (partition_number,), + '-part%s' % (partition_number,)] for suffix in potential_suffixes: potential_partition_device = '%s%s' % (device_path, suffix) if os.path.exists(potential_partition_device): @@ -58,10 +59,11 @@ def _get_nth_partition_for_device(device_path, partition_number): def _is_block_device(device_path, partition_path=None): - device_name = device_path.split('/')[-1] + device_name = os.path.realpath(device_path).split('/')[-1] sys_path = os.path.join('/sys/block/', device_name) if partition_path is not None: - sys_path = os.path.join(sys_path, partition_path.split('/')[-1]) + sys_path = os.path.join( + sys_path, os.path.realpath(partition_path).split('/')[-1]) return os.path.exists(sys_path) -- cgit v1.2.3 From 7ac13a1ef376a7b461673b90dfcd2c7c8612227a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Jul 2015 20:28:44 -0400 Subject: untested suggested change LP: #1461242 --- cloudinit/config/cc_ssh.py | 53 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index ab6940fa..7a673994 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -20,6 +20,7 @@ import glob import os +import sys # Ensure this is aliased to a name not 'distros' # since the module attribute 'distros' @@ -33,26 +34,17 @@ DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " "rather than the user \\\"root\\\".\';echo;sleep 10\"") -KEY_2_FILE = { - "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0o600), - "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0o644), - "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0o600), - "dsa_public": ("/etc/ssh/ssh_host_dsa_key.pub", 0o644), - "ecdsa_private": ("/etc/ssh/ssh_host_ecdsa_key", 0o600), - "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0o644), -} - -PRIV_2_PUB = { - 'rsa_private': 'rsa_public', - 'dsa_private': 'dsa_public', - 'ecdsa_private': 'ecdsa_public', -} - -KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' +GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] +KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' -GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa'] +KEY_2_FILE = {} +PRIV_2_PUB = {} +for k in GENERATE_KEY_NAMES: + KEY_2_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) + KEY_2_FILE.update({"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + PRIV_2_PUB["%s_private" % k] = "%s_public" % k -KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' +KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' def handle(_name, cfg, cloud, log, _args): @@ -92,18 +84,27 @@ def handle(_name, cfg, cloud, log, _args): genkeys = util.get_cfg_option_list(cfg, 'ssh_genkeytypes', GENERATE_KEY_NAMES) + lang_c = os.environ.copy() + lang_c['LANG'] = 'C' for keytype in genkeys: keyfile = KEY_FILE_TPL % (keytype) + if os.path.exists(keyfile): + continue util.ensure_dir(os.path.dirname(keyfile)) - if not os.path.exists(keyfile): - cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] + + # TODO(harlowja): Is this guard needed? + with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - # TODO(harlowja): Is this guard needed? - with util.SeLinuxGuard("/etc/ssh", recursive=True): - util.subp(cmd, capture=False) - except: - util.logexc(log, "Failed generating key type %s to " - "file %s", keytype, keyfile) + out, err = util.subp(cmd, capture=True, rcs=[0, 1], env=lang_c) + sys.stdout.write(util.encode_text(out)) + except util.ProcessExecutionError as e: + err = util.decode_binary(e.stderr).lower() + if err.lower().startswith("unknown key"): + log.debug("unknown key type %s" % keytype) + else: + util.logexc(log, "Failed generating key type %s to " + "file %s", keytype, keyfile) try: (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) -- cgit v1.2.3 From e86decfd53418ff481cb5db8d8b089417f1dafdf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Jul 2015 13:23:19 -0400 Subject: pep8 line too long --- cloudinit/config/cc_ssh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 7a673994..cfaceac6 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -96,7 +96,8 @@ def handle(_name, cfg, cloud, log, _args): # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - out, err = util.subp(cmd, capture=True, rcs=[0, 1], env=lang_c) + out, err = util.subp(cmd, capture=True, rcs=[0, 1], + env=lang_c) sys.stdout.write(util.encode_text(out)) except util.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() -- cgit v1.2.3 From a21baa2bf5619358250821aa3c3d69dd54b81b18 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Jul 2015 13:25:05 -0400 Subject: replace '2' with 'TO' in globals --- cloudinit/config/cc_ssh.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index cfaceac6..cd0174da 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -37,12 +37,12 @@ DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' -KEY_2_FILE = {} -PRIV_2_PUB = {} +CONFIG_KEY_TO_FILE = {} +PRIV_TO_PUB = {} for k in GENERATE_KEY_NAMES: - KEY_2_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) - KEY_2_FILE.update({"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) - PRIV_2_PUB["%s_private" % k] = "%s_public" % k + CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) + CONFIG_KEY_TO_FILE.update({"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + PRIV_TO_PUB["%s_private" % k] = "%s_public" % k KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' @@ -61,15 +61,15 @@ def handle(_name, cfg, cloud, log, _args): if "ssh_keys" in cfg: # if there are keys in cloud-config, use them for (key, val) in cfg["ssh_keys"].items(): - if key in KEY_2_FILE: - tgt_fn = KEY_2_FILE[key][0] - tgt_perms = KEY_2_FILE[key][1] + if key in CONFIG_KEY_TO_FILE: + tgt_fn = CONFIG_KEY_TO_FILE[key][0] + tgt_perms = CONFIG_KEY_TO_FILE[key][1] util.write_file(tgt_fn, val, tgt_perms) - for (priv, pub) in PRIV_2_PUB.items(): + for (priv, pub) in PRIV_TO_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: continue - pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) + pair = (CONFIG_KEY_TO_FILE[priv][0], CONFIG_KEY_TO_FILE[pub][0]) cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: # TODO(harlowja): Is this guard needed? -- cgit v1.2.3 From 404baf87e58f2c9740c8b31137b727c77d182058 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Jul 2015 14:10:58 -0400 Subject: fixes from testing --- cloudinit/config/cc_ssh.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index cd0174da..7fb13333 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -96,12 +96,11 @@ def handle(_name, cfg, cloud, log, _args): # TODO(harlowja): Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): try: - out, err = util.subp(cmd, capture=True, rcs=[0, 1], - env=lang_c) - sys.stdout.write(util.encode_text(out)) + out, err = util.subp(cmd, capture=True, env=lang_c) + sys.stdout.write(util.decode_binary(out)) except util.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() - if err.lower().startswith("unknown key"): + if e.exit_code == 1 and err.lower().startswith("unknown key"): log.debug("unknown key type %s" % keytype) else: util.logexc(log, "Failed generating key type %s to " -- cgit v1.2.3 From 4c799192a9d3132da0138e1adb640a9ab7e191b0 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Jul 2015 14:15:57 -0400 Subject: improve log message --- cloudinit/config/cc_ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 7fb13333..c2a7af72 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -101,7 +101,7 @@ def handle(_name, cfg, cloud, log, _args): except util.ProcessExecutionError as e: err = util.decode_binary(e.stderr).lower() if e.exit_code == 1 and err.lower().startswith("unknown key"): - log.debug("unknown key type %s" % keytype) + log.debug("ssh-keygen: unknown key type '%s'", keytype) else: util.logexc(log, "Failed generating key type %s to " "file %s", keytype, keyfile) -- cgit v1.2.3 From 452ea086beb8b28b41f5ccc610f4e5433010e35b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 22 Jul 2015 15:14:33 -0400 Subject: remove some overly verbose log messages --- cloudinit/config/cc_disk_setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index 92fa7a94..d5b0d1d7 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -648,7 +648,7 @@ def mkpart(device, definition): table_type: Which partition table to use, defaults to MBR device: the device to work on. """ - LOG.debug('Ensuring that we have a real device, not a symbolic link') + # ensure that we get a real device rather than a symbolic link device = os.path.realpath(device) LOG.debug("Checking values for %s definition" % device) @@ -747,7 +747,7 @@ def mkfs(fs_cfg): fs_replace = fs_cfg.get('replace_fs', False) overwrite = fs_cfg.get('overwrite', False) - LOG.debug('Ensuring that we have a real device, not a symbolic link') + # ensure that we get a real device rather than a symbolic link device = os.path.realpath(device) # This allows you to define the default ephemeral or swap -- cgit v1.2.3 From 6970029c661ab858a55dd467e5c593694ab39512 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 24 Jul 2015 16:58:57 -0400 Subject: commit initial re-work/re-implementation of syslog config --- cloudinit/config/cc_syslog.py | 183 +++++++++++++++++++++ doc/examples/cloud-config-syslog.txt | 30 ++++ .../unittests/test_handler/test_handler_syslog.py | 32 ++++ 3 files changed, 245 insertions(+) create mode 100644 cloudinit/config/cc_syslog.py create mode 100644 doc/examples/cloud-config-syslog.txt create mode 100644 tests/unittests/test_handler/test_handler_syslog.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_syslog.py b/cloudinit/config/cc_syslog.py new file mode 100644 index 00000000..21a8e8a9 --- /dev/null +++ b/cloudinit/config/cc_syslog.py @@ -0,0 +1,183 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import log as logging +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +import re + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE + +BUILTIN_CFG = { + 'remotes_file': '/etc/rsyslog.d/20-cloudinit-remotes.conf', + 'remotes': {}, + 'service_name': 'rsyslog', +} + +COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') +HOST_PORT_RE = re.compile( + r'^(?P[@]{0,2})' + '(([[](?P[^\]]*)[\]])|(?P[^:]*))' + '([:](?P[0-9]+))?$') + + +def parse_remotes_line(line, name=None): + try: + data, comment = COMMENT_RE.split(line) + comment = comment.strip() + except ValueError: + data, comment = (line, None) + + toks = data.strip().split() + match = None + if len(toks) == 1: + host_port = data + elif len(toks) == 2: + match, host_port = toks + else: + raise ValueError("line had multiple spaces: %s" % data) + + toks = HOST_PORT_RE.match(host_port) + + if not toks: + raise ValueError("Invalid host specification '%s'" % host_port) + + proto = toks.group('proto') + addr = toks.group('addr') or toks.group('bracket_addr') + port = toks.group('port') + print("host_port: %s" % addr) + print("port: %s" % port) + + if addr.startswith("[") and not addr.endswith("]"): + raise ValueError("host spec had invalid brackets: %s" % addr) + + if comment and not name: + name = comment + + t = SyslogRemotesLine(name=name, match=match, proto=proto, + addr=addr, port=port) + t.validate() + return t + + +class SyslogRemotesLine(object): + def __init__(self, name=None, match=None, proto=None, addr=None, + port=None): + if not match: + match = "*.*" + self.name = name + self.match = match + if proto == "@": + proto = "udp" + elif proto == "@@": + proto = "tcp" + self.proto = proto + + self.addr = addr + if port: + self.port = int(port) + else: + self.port = None + + def validate(self): + if self.port: + try: + int(self.port) + except ValueError: + raise ValueError("port '%s' is not an integer" % self.port) + + if not self.addr: + raise ValueError("address is required") + + def __repr__(self): + return "[name=%s match=%s proto=%s address=%s port=%s]" % ( + self.name, self.match, self.proto, self.addr, self.port + ) + + def __str__(self): + buf = self.match + " " + if self.proto == "udp": + buf += " @" + elif self.proto == "tcp": + buf += " @@" + + if ":" in self.addr: + buf += "[" + self.addr + "]" + else: + buf += self.addr + + if self.port: + buf += ":%s" % self.port + + if self.name: + buf += " # %s" % self.name + return buf + + +def remotes_to_rsyslog_cfg(remotes, header=None): + if not remotes: + return None + lines = [] + if header is not None: + lines.append(header) + for name, line in remotes.items(): + try: + lines.append(parse_remotes_line(line, name=name)) + except ValueError as e: + LOG.warn("failed loading remote %s: %s [%s]", name, line, e) + return '\n'.join(str(lines)) + '\n' + + +def reload_syslog(systemd, service='rsyslog'): + if systemd: + cmd = ['systemctl', 'reload-or-try-restart', service] + else: + cmd = ['service', service, 'reload'] + try: + util.subp(cmd, capture=True) + except util.ProcessExecutionError as e: + LOG.warn("Failed to reload syslog using '%s': %s", ' '.join(cmd), e) + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('syslog') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) + + remotes_file = mycfg.get('remotes_file') + if util.is_false(remotes_file): + LOG.debug("syslog/remotes_file empty, doing nothing") + return + + remotes = mycfg.get('remotes_dict', {}) + if remotes and not isinstance(remotes, dict): + LOG.warn("syslog/remotes: content is not a dictionary") + return + + config_data = remotes_to_rsyslog_cfg( + remotes, header="#cloud-init syslog module") + + util.write_file(remotes_file, config_data) + + reload_syslog( + systemd=cloud.distro.uses_systemd(), + service=mycfg.get('service_name')) diff --git a/doc/examples/cloud-config-syslog.txt b/doc/examples/cloud-config-syslog.txt new file mode 100644 index 00000000..9ec5e120 --- /dev/null +++ b/doc/examples/cloud-config-syslog.txt @@ -0,0 +1,30 @@ +## syslog module allows you to configure the systems syslog. +## configuration of syslog is under the top level cloud-config +## entry 'syslog'. +## +## "remotes" +## remotes is a dictionary. items are of 'name: remote_info' +## name is simply a name (example 'maas'). It has no importance other than +## for cloud-init merging configs +## +## remote_info is of the format +## * optional filter for log messages +## default if not present: *.* +## * optional leading '@' or '@@' (indicates udp or tcp). +## default if not present (udp): @ +## This is rsyslog format for that. if not present, is '@' which is udp +## * ipv4 or ipv6 or hostname +## ipv6 addresses must be encoded in [::1] format. example: @[fd00::1]:514 +## * optional port +## port defaults to 514 +## +## Example: +#cloud-config +syslog: + remotes: + # udp to host 'maas.mydomain' port 514 + maashost: maas.mydomain + # udp to ipv4 host on port 514 + maas: "@[10.5.1.56]:514" + # tcp to host ipv6 host on port 555 + maasipv6: "*.* @@[FE80::0202:B3FF:FE1E:8329]:555" diff --git a/tests/unittests/test_handler/test_handler_syslog.py b/tests/unittests/test_handler/test_handler_syslog.py new file mode 100644 index 00000000..bbfd521e --- /dev/null +++ b/tests/unittests/test_handler/test_handler_syslog.py @@ -0,0 +1,32 @@ +from cloudinit.config.cc_syslog import ( + parse_remotes_line, SyslogRemotesLine, remotes_to_rsyslog_cfg) +from cloudinit import util +from .. import helpers as t_help + + +class TestParseRemotesLine(t_help.TestCase): + def test_valid_port(self): + r = parse_remotes_line("foo:9") + self.assertEqual(9, r.port) + + def test_invalid_port(self): + with self.assertRaises(ValueError): + parse_remotes_line("*.* foo:abc") + + def test_valid_ipv6(self): + r = parse_remotes_line("*.* [::1]") + self.assertEqual("*.* [::1]", str(r)) + + def test_valid_ipv6_with_port(self): + r = parse_remotes_line("*.* [::1]:100") + self.assertEqual(r.port, 100) + self.assertEqual(r.addr, "::1") + self.assertEqual("*.* [::1]:100", str(r)) + + def test_invalid_multiple_colon(self): + with self.assertRaises(ValueError): + parse_remotes_line("*.* ::1:100") + + def test_name_in_string(self): + r = parse_remotes_line("syslog.host", name="foobar") + self.assertEqual("*.* syslog.host # foobar", str(r)) -- cgit v1.2.3 From 247f2cecb72a852a42f147645e80b538eee05f93 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Jul 2015 14:20:29 -0400 Subject: update existing rsyslog module with better code and doc --- cloudinit/config/cc_rsyslog.py | 190 +++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 37 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 57486edc..7d5657bc 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -17,37 +17,129 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +rsyslog module allows configuration of syslog logging via rsyslog +Configuration is done under the cloud-config top level 'rsyslog'. + +Under 'rsyslog' you can define: + - configs: [default=[]] + this is a list. entries in it are a string or a dictionary. + each entry has 2 parts: + * content + * filename + if the entry is a string, then it is assigned to 'content'. + for each entry, content is written to the provided filename. + if filename is not provided, its default is read from 'config_filename' + + Content here can be any valid rsyslog configuration. No format + specific format is enforced. + + For simply logging to an existing remote syslog server, via udp: + configs: ["*.* @192.168.1.1"] + + - config_filename: [default=20-cloud-config.conf] + this is the file name to use if none is provided in a config entry. + - config_dir: [default=/etc/rsyslog.d] + this directory is used for filenames that are not absolute paths. + - service_reload_command: [default="auto"] + this command is executed if files have been written and thus the syslog + daemon needs to be told. + +Note, since cloud-init 0.5 a legacy version of rsyslog config has been +present and is still supported. See below for the mappings between old +value and new value: + old value -> new value + 'rsyslog' -> rsyslog/configs + 'rsyslog_filename' -> rsyslog/config_filename + 'rsyslog_dir' -> rsyslog/config_dir + +the legacy config does not support 'service_reload_command'. + +Example config: + #cloud-config + rsyslog: + configs: + - "*.* @@192.158.1.1" + - content: "*.* @@192.0.2.1:10514" + - filename: 01-examplecom.conf + - content: | + *.* @@syslogd.example.com + config_dir: config_dir + config_filename: config_filename + service_reload_command: [your, syslog, restart, command] + +Example Legacy config: + #cloud-config + rsyslog: + - "*.* @@192.158.1.1" + rsyslog_dir: /etc/rsyslog-config.d/ + rsyslog_filename: 99-local.conf +""" import os +import six +from cloudinit import log as logging from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" +DEF_RELOAD = "auto" +KEYNAME_CONFIGS = 'configs' +KEYNAME_FILENAME = 'config_filename' +KEYNAME_DIR = 'config_dir' +KEYNAME_RELOAD = 'service_reload_command' +KEYNAME_LEGACY_FILENAME = 'rsyslog_filename' +KEYNAME_LEGACY_DIR = 'rsyslog_dir' -def handle(name, cfg, cloud, log, _args): - # rsyslog: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com +LOG = logging.getLogger(__name__) - # process 'rsyslog' - if 'rsyslog' not in cfg: - log.debug(("Skipping module named %s," - " no 'rsyslog' key in configuration"), name) - return - def_dir = cfg.get('rsyslog_dir', DEF_DIR) - def_fname = cfg.get('rsyslog_filename', DEF_FILENAME) +def reload_syslog(command=DEF_RELOAD, systemd=False): + service = 'rsyslog' + if command == DEF_RELOAD: + if systemd: + cmd = ['systemctl', 'reload-or-try-restart', service] + else: + cmd = ['service', service, 'reload'] + else: + cmd = command + util.subp(cmd, capture=True) + + +def load_config(cfg): + # return an updated config with entries of the correct type + # support converting the old top level format into new format + mycfg = cfg.get('rsyslog', {}) + + if isinstance(mycfg, list): + mycfg[KEYNAME_CONFIGS] = mycfg + if KEYNAME_LEGACY_FILENAME in cfg: + mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME] + if KEYNAME_LEGACY_DIR in cfg: + mycfg[KEYNAME_DIR] = cfg[KEYNAME_DIR] + + fillup = ( + (KEYNAME_DIR, DEF_DIR, six.text_type), + (KEYNAME_FILENAME, DEF_FILENAME, six.text_type) + (KEYNAME_RELOAD, DEF_RELOAD, (six.text_type, list))) + for key, default, vtypes in fillup: + if key not in mycfg or not isinstance(mycfg[key], vtypes): + mycfg[key] = default + + return mycfg + + +def apply_rsyslog_changes(configs, def_fname, cfg_dir): + # apply the changes in 'configs' to the paths in def_fname and cfg_dir + # return a list of the files changed files = [] - for i, ent in enumerate(cfg['rsyslog']): + for cur_pos, ent in enumerate(configs): if isinstance(ent, dict): if "content" not in ent: - log.warn("No 'content' entry in config entry %s", i + 1) + LOG.warn("No 'content' entry in config entry %s", cur_pos + 1) continue content = ent['content'] filename = ent.get("filename", def_fname) @@ -57,15 +149,14 @@ def handle(name, cfg, cloud, log, _args): filename = filename.strip() if not filename: - log.warn("Entry %s has an empty filename", i + 1) + LOG.warn("Entry %s has an empty filename", cur_pos + 1) continue if not filename.startswith("/"): - filename = os.path.join(def_dir, filename) + filename = os.path.join(cfg_dir, filename) # Truncate filename first time you see it - omode = "ab" - if filename not in files: + omode = "ab" if filename not in files: omode = "wb" files.append(filename) @@ -73,24 +164,49 @@ def handle(name, cfg, cloud, log, _args): contents = "%s\n" % (content) util.write_file(filename, contents, omode=omode) except Exception: - util.logexc(log, "Failed to write to %s", filename) + util.logexc(LOG, "Failed to write to %s", filename) + + return files + + +def handle(name, cfg, cloud, log, _args): + # rsyslog: + # configs: + # - "*.* @@192.158.1.1" + # - content: "*.* @@192.0.2.1:10514" + # - filename: 01-examplecom.conf + # content: | + # *.* @@syslogd.example.com + # config_dir: DEF_DIR + # config_filename: DEF_FILENAME + # service_reload: "auto" + + if 'rsyslog' not in cfg: + log.debug(("Skipping module named %s," + " no 'rsyslog' key in configuration"), name) + return + + mycfg = load_config(cfg) + if not mycfg['configs']: + log.debug("Empty config rsyslog['configs'], nothing to do") + return + + changes = apply_rsyslog_changes( + configs=mycfg[KEYNAME_CONFIGS], + def_fname=mycfg[KEYNAME_FILENAME], + cfg_dir=mycfg[KEYNAME_DIR]) + + if not changes: + log.debug("restart of syslog not necessary, no changes made") + return - # Attempt to restart syslogd - restarted = False try: - # If this config module is running at cloud-init time - # (before rsyslog is running) we don't actually have to - # restart syslog. - # - # Upstart actually does what we want here, in that it doesn't - # start a service that wasn't running already on 'restart' - # it will also return failure on the attempt, so 'restarted' - # won't get set. - log.debug("Restarting rsyslog") - util.subp(['service', 'rsyslog', 'restart']) - restarted = True - except Exception: - util.logexc(log, "Failed restarting rsyslog") + restarted = reload_syslog( + service=mycfg[KEYNAME_RELOAD], + systemd=cloud.distro.uses_systemd()), + except util.ProcessExecutionError as e: + restarted = False + log.warn("Failed to reload syslog", e) if restarted: # This only needs to run if we *actually* restarted @@ -98,4 +214,4 @@ def handle(name, cfg, cloud, log, _args): cloud.cycle_logging() # This should now use rsyslog if # the logging was setup to use it... - log.debug("%s configured %s files", name, files) + log.debug("%s configured %s files", name, changes) -- cgit v1.2.3 From 22cb92421234c31b783ed9df01439c734535ba01 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Jul 2015 16:49:30 -0400 Subject: add rsyslog tests reasonable test of reworked rsyslog module --- cloudinit/config/cc_rsyslog.py | 27 ++--- .../unittests/test_handler/test_handler_rsyslog.py | 113 +++++++++++++++++++++ 2 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_rsyslog.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 7d5657bc..a07200c3 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -113,17 +113,18 @@ def load_config(cfg): # support converting the old top level format into new format mycfg = cfg.get('rsyslog', {}) - if isinstance(mycfg, list): - mycfg[KEYNAME_CONFIGS] = mycfg + if isinstance(cfg.get('rsyslog'), list): + mycfg = {KEYNAME_CONFIGS: cfg.get('rsyslog')} if KEYNAME_LEGACY_FILENAME in cfg: mycfg[KEYNAME_FILENAME] = cfg[KEYNAME_LEGACY_FILENAME] if KEYNAME_LEGACY_DIR in cfg: - mycfg[KEYNAME_DIR] = cfg[KEYNAME_DIR] + mycfg[KEYNAME_DIR] = cfg[KEYNAME_LEGACY_DIR] fillup = ( - (KEYNAME_DIR, DEF_DIR, six.text_type), - (KEYNAME_FILENAME, DEF_FILENAME, six.text_type) - (KEYNAME_RELOAD, DEF_RELOAD, (six.text_type, list))) + (KEYNAME_CONFIGS, [], list), + (KEYNAME_DIR, DEF_DIR, six.string_types), + (KEYNAME_FILENAME, DEF_FILENAME, six.string_types), + (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,))) for key, default, vtypes in fillup: if key not in mycfg or not isinstance(mycfg[key], vtypes): @@ -156,7 +157,8 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): filename = os.path.join(cfg_dir, filename) # Truncate filename first time you see it - omode = "ab" if filename not in files: + omode = "ab" + if filename not in files: omode = "wb" files.append(filename) @@ -170,17 +172,6 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): def handle(name, cfg, cloud, log, _args): - # rsyslog: - # configs: - # - "*.* @@192.158.1.1" - # - content: "*.* @@192.0.2.1:10514" - # - filename: 01-examplecom.conf - # content: | - # *.* @@syslogd.example.com - # config_dir: DEF_DIR - # config_filename: DEF_FILENAME - # service_reload: "auto" - if 'rsyslog' not in cfg: log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py new file mode 100644 index 00000000..3501ff95 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -0,0 +1,113 @@ +import os +import shutil +import tempfile + +from cloudinit.config.cc_rsyslog import ( + load_config, DEF_FILENAME, DEF_DIR, DEF_RELOAD, apply_rsyslog_changes) +from cloudinit import util + +from .. import helpers as t_help + + +class TestLoadConfig(t_help.TestCase): + def setUp(self): + super(TestLoadConfig, self).setUp() + self.basecfg = { + 'config_filename': DEF_FILENAME, + 'config_dir': DEF_DIR, + 'service_reload_command': DEF_RELOAD, + 'configs': [], + } + + def test_legacy_full(self): + found = load_config({ + 'rsyslog': ['*.* @192.168.1.1'], + 'rsyslog_dir': "mydir", + 'rsyslog_filename': "myfilename"}) + expected = { + 'configs': ['*.* @192.168.1.1'], + 'config_dir': "mydir", + 'config_filename': 'myfilename', + 'service_reload_command': 'auto'} + self.assertEqual(found, expected) + + def test_legacy_defaults(self): + found = load_config({ + 'rsyslog': ['*.* @192.168.1.1']}) + self.basecfg.update({ + 'configs': ['*.* @192.168.1.1']}) + self.assertEqual(found, self.basecfg) + + def test_new_defaults(self): + self.assertEqual(load_config({}), self.basecfg) + + def test_new_configs(self): + cfgs = ['*.* myhost', '*.* my2host'] + self.basecfg.update({'configs': cfgs}) + self.assertEqual( + load_config({'rsyslog': {'configs': cfgs}}), + self.basecfg) + + +class TestApplyChanges(t_help.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + + def test_simple(self): + cfgline = "*.* foohost" + changed = apply_rsyslog_changes( + configs=[cfgline], def_fname="foo.cfg", cfg_dir=self.tmp) + + fname = os.path.join(self.tmp, "foo.cfg") + self.assertEqual([fname], changed) + self.assertEqual( + util.load_file(fname), cfgline + "\n") + + def test_multiple_files(self): + configs = [ + '*.* foohost', + {'content': 'abc', 'filename': 'my.cfg'}, + {'content': 'filefoo-content', + 'filename': os.path.join(self.tmp, 'mydir/mycfg')}, + ] + + changed = apply_rsyslog_changes( + configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) + + expected = [ + (os.path.join(self.tmp, "default.cfg"), + "*.* foohost\n"), + (os.path.join(self.tmp, "my.cfg"), "abc\n"), + (os.path.join(self.tmp, "mydir/mycfg"), "filefoo-content\n"), + ] + self.assertEqual([f[0] for f in expected], changed) + actual = [] + for fname, _content in expected: + util.load_file(fname) + actual.append((fname, util.load_file(fname),)) + self.assertEqual(expected, actual) + + def test_repeat_def(self): + configs = ['*.* foohost', "*.warn otherhost"] + + changed = apply_rsyslog_changes( + configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) + + fname = os.path.join(self.tmp, "default.cfg") + self.assertEqual([fname], changed) + + expected_content = '\n'.join([c for c in configs]) + '\n' + found_content = util.load_file(fname) + self.assertEqual(expected_content, found_content) + + def test_multiline_content(self): + configs = ['line1', 'line2\nline3\n'] + + changed = apply_rsyslog_changes( + configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) + + fname = os.path.join(self.tmp, "default.cfg") + expected_content = '\n'.join([c for c in configs]) + '\n' + found_content = util.load_file(fname) + self.assertEqual(expected_content, found_content) -- cgit v1.2.3 From 39f25660714e5640be3dce576a6cfdee9a1672c8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Jul 2015 20:53:44 -0400 Subject: use 'restart' rather than 'reload' on non-systemd systems Testing on trusty shows that: service rsyslog reload does produce a message like: rsyslogd was HUPed but does not result in new config being in honored. Using restart does, and with upstart that should be fine (as upstart will start only if previously running). --- cloudinit/config/cc_rsyslog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index a07200c3..82e382e8 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -102,7 +102,7 @@ def reload_syslog(command=DEF_RELOAD, systemd=False): if systemd: cmd = ['systemctl', 'reload-or-try-restart', service] else: - cmd = ['service', service, 'reload'] + cmd = ['service', service, 'restart'] else: cmd = command util.subp(cmd, capture=True) -- cgit v1.2.3 From cae9122f88f8369454b03b97a5386d3135941fd9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 27 Jul 2015 21:02:51 -0400 Subject: fix kwarg --- cloudinit/config/cc_rsyslog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 82e382e8..9599e925 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -193,7 +193,7 @@ def handle(name, cfg, cloud, log, _args): try: restarted = reload_syslog( - service=mycfg[KEYNAME_RELOAD], + command=mycfg[KEYNAME_RELOAD], systemd=cloud.distro.uses_systemd()), except util.ProcessExecutionError as e: restarted = False -- cgit v1.2.3 From 6dd505fd02e0933d8770c8932a927940f6a0e025 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jul 2015 09:27:26 -0400 Subject: add support for 'remotes' --- cloudinit/config/cc_rsyslog.py | 156 ++++++++++++++++++++- cloudinit/config/cc_syslog.py | 2 +- doc/examples/cloud-config-rsyslog.txt | 37 +++++ doc/examples/cloud-config-syslog.txt | 30 ---- .../unittests/test_handler/test_handler_rsyslog.py | 38 ++++- 5 files changed, 227 insertions(+), 36 deletions(-) create mode 100644 doc/examples/cloud-config-rsyslog.txt delete mode 100644 doc/examples/cloud-config-syslog.txt (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 9599e925..8c02e826 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -37,10 +37,33 @@ Under 'rsyslog' you can define: For simply logging to an existing remote syslog server, via udp: configs: ["*.* @192.168.1.1"] + - remotes: [default={}] + This is a dictionary of name / value pairs. + In comparison to 'config's, it is more focused in that it only supports + remote syslog configuration. It is not rsyslog specific, and could + convert to other syslog implementations. + + Each entry in remotes is a 'name' and a 'value'. + * name: an string identifying the entry. good practice would indicate + using a consistent and identifiable string for the producer. + For example, the MAAS service could use 'maas' as the key. + * value consists of the following parts: + * optional filter for log messages + default if not present: *.* + * optional leading '@' or '@@' (indicates udp or tcp respectively). + default if not present (udp): @ + This is rsyslog format for that. if not present, is '@'. + * ipv4 or ipv6 or hostname + ipv6 addresses must be in [::1] format. (@[fd00::1]:514) + * optional port + port defaults to 514 + - config_filename: [default=20-cloud-config.conf] this is the file name to use if none is provided in a config entry. + - config_dir: [default=/etc/rsyslog.d] this directory is used for filenames that are not absolute paths. + - service_reload_command: [default="auto"] this command is executed if files have been written and thus the syslog daemon needs to be told. @@ -61,9 +84,12 @@ Example config: configs: - "*.* @@192.158.1.1" - content: "*.* @@192.0.2.1:10514" - - filename: 01-examplecom.conf + filename: 01-example.conf - content: | *.* @@syslogd.example.com + remotes: + maas: "192.168.1.1" + juju: "10.0.4.1" config_dir: config_dir config_filename: config_filename service_reload_command: [your, syslog, restart, command] @@ -77,6 +103,7 @@ Example Legacy config: """ import os +import re import six from cloudinit import log as logging @@ -85,6 +112,7 @@ from cloudinit import util DEF_FILENAME = "20-cloud-config.conf" DEF_DIR = "/etc/rsyslog.d" DEF_RELOAD = "auto" +DEF_REMOTES = {} KEYNAME_CONFIGS = 'configs' KEYNAME_FILENAME = 'config_filename' @@ -92,9 +120,15 @@ KEYNAME_DIR = 'config_dir' KEYNAME_RELOAD = 'service_reload_command' KEYNAME_LEGACY_FILENAME = 'rsyslog_filename' KEYNAME_LEGACY_DIR = 'rsyslog_dir' +KEYNAME_REMOTES = 'remotes' LOG = logging.getLogger(__name__) +COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') +HOST_PORT_RE = re.compile( + r'^(?P[@]{0,2})' + '(([[](?P[^\]]*)[\]])|(?P[^:]*))' + '([:](?P[0-9]+))?$') def reload_syslog(command=DEF_RELOAD, systemd=False): service = 'rsyslog' @@ -124,7 +158,8 @@ def load_config(cfg): (KEYNAME_CONFIGS, [], list), (KEYNAME_DIR, DEF_DIR, six.string_types), (KEYNAME_FILENAME, DEF_FILENAME, six.string_types), - (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,))) + (KEYNAME_RELOAD, DEF_RELOAD, six.string_types + (list,)), + (KEYNAME_REMOTES, DEF_REMOTES, dict)) for key, default, vtypes in fillup: if key not in mycfg or not isinstance(mycfg[key], vtypes): @@ -171,6 +206,113 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): return files +def parse_remotes_line(line, name=None): + try: + data, comment = COMMENT_RE.split(line) + comment = comment.strip() + except ValueError: + data, comment = (line, None) + + toks = data.strip().split() + match = None + if len(toks) == 1: + host_port = data + elif len(toks) == 2: + match, host_port = toks + else: + raise ValueError("line had multiple spaces: %s" % data) + + toks = HOST_PORT_RE.match(host_port) + + if not toks: + raise ValueError("Invalid host specification '%s'" % host_port) + + proto = toks.group('proto') + addr = toks.group('addr') or toks.group('bracket_addr') + port = toks.group('port') + + if addr.startswith("[") and not addr.endswith("]"): + raise ValueError("host spec had invalid brackets: %s" % addr) + + if comment and not name: + name = comment + + t = SyslogRemotesLine(name=name, match=match, proto=proto, + addr=addr, port=port) + t.validate() + return t + + +class SyslogRemotesLine(object): + def __init__(self, name=None, match=None, proto=None, addr=None, + port=None): + if not match: + match = "*.*" + self.name = name + self.match = match + if proto == "@": + proto = "udp" + elif proto == "@@": + proto = "tcp" + self.proto = proto + + self.addr = addr + if port: + self.port = int(port) + else: + self.port = None + + def validate(self): + if self.port: + try: + int(self.port) + except ValueError: + raise ValueError("port '%s' is not an integer" % self.port) + + if not self.addr: + raise ValueError("address is required") + + def __repr__(self): + return "[name=%s match=%s proto=%s address=%s port=%s]" % ( + self.name, self.match, self.proto, self.addr, self.port + ) + + def __str__(self): + buf = self.match + " " + if self.proto == "udp": + buf += " @" + elif self.proto == "tcp": + buf += " @@" + + if ":" in self.addr: + buf += "[" + self.addr + "]" + else: + buf += self.addr + + if self.port: + buf += ":%s" % self.port + + if self.name: + buf += " # %s" % self.name + return buf + + +def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): + if not remotes: + return None + lines = [] + if header is not None: + lines.append(header) + for name, line in remotes.items(): + try: + lines.append(parse_remotes_line(line, name=name)) + except ValueError as e: + LOG.warn("failed loading remote %s: %s [%s]", name, line, e) + if footer is not None: + lines.append(footer) + return '\n'.join(str(lines)) + '\n' + + def handle(name, cfg, cloud, log, _args): if 'rsyslog' not in cfg: log.debug(("Skipping module named %s," @@ -178,6 +320,16 @@ def handle(name, cfg, cloud, log, _args): return mycfg = load_config(cfg) + configs = mycfg[KEYNAME_CONFIGS] + + if mycfg[KEYNAME_REMOTES]: + configs.append( + remotes_to_rsyslog_cfg( + mycfg[KEYNAME_REMOTES], + header="# begin remotes", + footer="# end remotes", + )) + if not mycfg['configs']: log.debug("Empty config rsyslog['configs'], nothing to do") return diff --git a/cloudinit/config/cc_syslog.py b/cloudinit/config/cc_syslog.py index 21a8e8a9..27793f8b 100644 --- a/cloudinit/config/cc_syslog.py +++ b/cloudinit/config/cc_syslog.py @@ -168,7 +168,7 @@ def handle(name, cfg, cloud, log, args): LOG.debug("syslog/remotes_file empty, doing nothing") return - remotes = mycfg.get('remotes_dict', {}) + remotes = mycfg.get('remotes', {}) if remotes and not isinstance(remotes, dict): LOG.warn("syslog/remotes: content is not a dictionary") return diff --git a/doc/examples/cloud-config-rsyslog.txt b/doc/examples/cloud-config-rsyslog.txt new file mode 100644 index 00000000..ff60e3a8 --- /dev/null +++ b/doc/examples/cloud-config-rsyslog.txt @@ -0,0 +1,37 @@ +## the rsyslog module allows you to configure the systems syslog. +## configuration of syslog is under the top level cloud-config +## entry 'rsyslog'. +## +## Example: +#cloud-config +rsyslog: + remotes: + # udp to host 'maas.mydomain' port 514 + maashost: maas.mydomain + # udp to ipv4 host on port 514 + maas: "@[10.5.1.56]:514" + # tcp to host ipv6 host on port 555 + maasipv6: "*.* @@[FE80::0202:B3FF:FE1E:8329]:555" + configs: + - "*.* @@192.158.1.1" + - content: "*.* @@192.0.2.1:10514" + filename: 01-example.conf + - content: | + *.* @@syslogd.example.com + config_dir: /etc/rsyslog.d + config_filename: 20-cloud-config.conf + service_reload_command: [your, syslog, reload, command] + +## Additionally the following legacy format is supported +## it is converted into the format above before use. +## rsyslog_filename -> rsyslog/config_filename +## rsyslog_dir -> rsyslog/config_dir +## rsyslog -> rsyslog/configs +# rsyslog: +# - "*.* @@192.158.1.1" +# - content: "*.* @@192.0.2.1:10514" +# filename: 01-example.conf +# - content: | +# *.* @@syslogd.example.com +# rsyslog_filename: 20-cloud-config.conf +# rsyslog_dir: /etc/rsyslog.d diff --git a/doc/examples/cloud-config-syslog.txt b/doc/examples/cloud-config-syslog.txt deleted file mode 100644 index 9ec5e120..00000000 --- a/doc/examples/cloud-config-syslog.txt +++ /dev/null @@ -1,30 +0,0 @@ -## syslog module allows you to configure the systems syslog. -## configuration of syslog is under the top level cloud-config -## entry 'syslog'. -## -## "remotes" -## remotes is a dictionary. items are of 'name: remote_info' -## name is simply a name (example 'maas'). It has no importance other than -## for cloud-init merging configs -## -## remote_info is of the format -## * optional filter for log messages -## default if not present: *.* -## * optional leading '@' or '@@' (indicates udp or tcp). -## default if not present (udp): @ -## This is rsyslog format for that. if not present, is '@' which is udp -## * ipv4 or ipv6 or hostname -## ipv6 addresses must be encoded in [::1] format. example: @[fd00::1]:514 -## * optional port -## port defaults to 514 -## -## Example: -#cloud-config -syslog: - remotes: - # udp to host 'maas.mydomain' port 514 - maashost: maas.mydomain - # udp to ipv4 host on port 514 - maas: "@[10.5.1.56]:514" - # tcp to host ipv6 host on port 555 - maasipv6: "*.* @@[FE80::0202:B3FF:FE1E:8329]:555" diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index 3501ff95..0bace685 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -3,7 +3,8 @@ import shutil import tempfile from cloudinit.config.cc_rsyslog import ( - load_config, DEF_FILENAME, DEF_DIR, DEF_RELOAD, apply_rsyslog_changes) + apply_rsyslog_changes, DEF_DIR, DEF_FILENAME, DEF_RELOAD, load_config, + parse_remotes_line) from cloudinit import util from .. import helpers as t_help @@ -17,6 +18,7 @@ class TestLoadConfig(t_help.TestCase): 'config_dir': DEF_DIR, 'service_reload_command': DEF_RELOAD, 'configs': [], + 'remotes': {}, } def test_legacy_full(self): @@ -24,12 +26,14 @@ class TestLoadConfig(t_help.TestCase): 'rsyslog': ['*.* @192.168.1.1'], 'rsyslog_dir': "mydir", 'rsyslog_filename': "myfilename"}) - expected = { + self.basecfg.update({ 'configs': ['*.* @192.168.1.1'], 'config_dir': "mydir", 'config_filename': 'myfilename', 'service_reload_command': 'auto'} - self.assertEqual(found, expected) + ) + + self.assertEqual(found, self.basecfg) def test_legacy_defaults(self): found = load_config({ @@ -111,3 +115,31 @@ class TestApplyChanges(t_help.TestCase): expected_content = '\n'.join([c for c in configs]) + '\n' found_content = util.load_file(fname) self.assertEqual(expected_content, found_content) + + +class TestParseRemotesLine(t_help.TestCase): + def test_valid_port(self): + r = parse_remotes_line("foo:9") + self.assertEqual(9, r.port) + + def test_invalid_port(self): + with self.assertRaises(ValueError): + parse_remotes_line("*.* foo:abc") + + def test_valid_ipv6(self): + r = parse_remotes_line("*.* [::1]") + self.assertEqual("*.* [::1]", str(r)) + + def test_valid_ipv6_with_port(self): + r = parse_remotes_line("*.* [::1]:100") + self.assertEqual(r.port, 100) + self.assertEqual(r.addr, "::1") + self.assertEqual("*.* [::1]:100", str(r)) + + def test_invalid_multiple_colon(self): + with self.assertRaises(ValueError): + parse_remotes_line("*.* ::1:100") + + def test_name_in_string(self): + r = parse_remotes_line("syslog.host", name="foobar") + self.assertEqual("*.* syslog.host # foobar", str(r)) -- cgit v1.2.3 From f61a62434b36ab873b2b82a5ba69eda826755bfc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jul 2015 10:12:02 -0400 Subject: fix bug in remotes_to_rsyslog_cfg, add test --- cloudinit/config/cc_rsyslog.py | 4 +-- .../unittests/test_handler/test_handler_rsyslog.py | 32 ++++++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 8c02e826..915ab420 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -305,12 +305,12 @@ def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): lines.append(header) for name, line in remotes.items(): try: - lines.append(parse_remotes_line(line, name=name)) + lines.append(str(parse_remotes_line(line, name=name))) except ValueError as e: LOG.warn("failed loading remote %s: %s [%s]", name, line, e) if footer is not None: lines.append(footer) - return '\n'.join(str(lines)) + '\n' + return '\n'.join(lines) + "\n" def handle(name, cfg, cloud, log, _args): diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index 0bace685..292559c5 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -4,7 +4,7 @@ import tempfile from cloudinit.config.cc_rsyslog import ( apply_rsyslog_changes, DEF_DIR, DEF_FILENAME, DEF_RELOAD, load_config, - parse_remotes_line) + parse_remotes_line, remotes_to_rsyslog_cfg) from cloudinit import util from .. import helpers as t_help @@ -80,10 +80,10 @@ class TestApplyChanges(t_help.TestCase): configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) expected = [ - (os.path.join(self.tmp, "default.cfg"), - "*.* foohost\n"), - (os.path.join(self.tmp, "my.cfg"), "abc\n"), - (os.path.join(self.tmp, "mydir/mycfg"), "filefoo-content\n"), + (os.path.join(self.tmp, "default.cfg"), + "*.* foohost\n"), + (os.path.join(self.tmp, "my.cfg"), "abc\n"), + (os.path.join(self.tmp, "mydir/mycfg"), "filefoo-content\n"), ] self.assertEqual([f[0] for f in expected], changed) actual = [] @@ -108,7 +108,7 @@ class TestApplyChanges(t_help.TestCase): def test_multiline_content(self): configs = ['line1', 'line2\nline3\n'] - changed = apply_rsyslog_changes( + apply_rsyslog_changes( configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) fname = os.path.join(self.tmp, "default.cfg") @@ -143,3 +143,23 @@ class TestParseRemotesLine(t_help.TestCase): def test_name_in_string(self): r = parse_remotes_line("syslog.host", name="foobar") self.assertEqual("*.* syslog.host # foobar", str(r)) + + +class TestRemotesToSyslog(t_help.TestCase): + def test_simple(self): + # str rendered line must appear in remotes_to_ryslog_cfg return + mycfg = "*.* myhost" + myline = str(parse_remotes_line(mycfg, name="myname")) + r = remotes_to_rsyslog_cfg({'myname': mycfg}) + lines = r.splitlines() + self.assertEqual(1, len(lines)) + self.assertTrue(myline in r.splitlines()) + + def test_header_footer(self): + header = "#foo head" + footer = "#foo foot" + r = remotes_to_rsyslog_cfg( + {'myname': "*.* myhost"}, header=header, footer=footer) + lines = r.splitlines() + self.assertTrue(header, lines[0]) + self.assertTrue(footer, lines[-1]) -- cgit v1.2.3 From c6e7fb1752a93ed534080adf0588e4c7cdd99071 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jul 2015 10:34:31 -0400 Subject: add trailing newline only if necessary --- cloudinit/config/cc_rsyslog.py | 9 +++++---- tests/unittests/test_handler/test_handler_rsyslog.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 915ab420..2bb00728 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -188,8 +188,7 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): LOG.warn("Entry %s has an empty filename", cur_pos + 1) continue - if not filename.startswith("/"): - filename = os.path.join(cfg_dir, filename) + filename = os.path.join(cfg_dir, filename) # Truncate filename first time you see it omode = "ab" @@ -198,8 +197,10 @@ def apply_rsyslog_changes(configs, def_fname, cfg_dir): files.append(filename) try: - contents = "%s\n" % (content) - util.write_file(filename, contents, omode=omode) + endl = "" + if not content.endswith("\n"): + endl = "\n" + util.write_file(filename, content + endl, omode=omode) except Exception: util.logexc(LOG, "Failed to write to %s", filename) diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index 292559c5..e7666615 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -112,7 +112,7 @@ class TestApplyChanges(t_help.TestCase): configs=configs, def_fname="default.cfg", cfg_dir=self.tmp) fname = os.path.join(self.tmp, "default.cfg") - expected_content = '\n'.join([c for c in configs]) + '\n' + expected_content = '\n'.join([c for c in configs]) found_content = util.load_file(fname) self.assertEqual(expected_content, found_content) -- cgit v1.2.3 From 0a581732a40ff814b1fc0dace9f519b7a5c779e6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jul 2015 10:48:32 -0400 Subject: must declare proto of '@' --- cloudinit/config/cc_rsyslog.py | 6 ++++-- tests/unittests/test_handler/test_handler_rsyslog.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 2bb00728..5ecf1629 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -251,6 +251,8 @@ class SyslogRemotesLine(object): match = "*.*" self.name = name self.match = match + if not proto: + proto = "udp" if proto == "@": proto = "udp" elif proto == "@@": @@ -281,9 +283,9 @@ class SyslogRemotesLine(object): def __str__(self): buf = self.match + " " if self.proto == "udp": - buf += " @" + buf += "@" elif self.proto == "tcp": - buf += " @@" + buf += "@@" if ":" in self.addr: buf += "[" + self.addr + "]" diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index e7666615..7bfa65a9 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -128,13 +128,13 @@ class TestParseRemotesLine(t_help.TestCase): def test_valid_ipv6(self): r = parse_remotes_line("*.* [::1]") - self.assertEqual("*.* [::1]", str(r)) + self.assertEqual("*.* @[::1]", str(r)) def test_valid_ipv6_with_port(self): r = parse_remotes_line("*.* [::1]:100") self.assertEqual(r.port, 100) self.assertEqual(r.addr, "::1") - self.assertEqual("*.* [::1]:100", str(r)) + self.assertEqual("*.* @[::1]:100", str(r)) def test_invalid_multiple_colon(self): with self.assertRaises(ValueError): @@ -142,7 +142,7 @@ class TestParseRemotesLine(t_help.TestCase): def test_name_in_string(self): r = parse_remotes_line("syslog.host", name="foobar") - self.assertEqual("*.* syslog.host # foobar", str(r)) + self.assertEqual("*.* @syslog.host # foobar", str(r)) class TestRemotesToSyslog(t_help.TestCase): -- cgit v1.2.3 From d5f93dbd908c349548554cb69ca3afd05077cf57 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jul 2015 10:49:48 -0400 Subject: remove 'syslog' module (its been moved to rsyslog) --- cloudinit/config/cc_syslog.py | 183 --------------------- .../unittests/test_handler/test_handler_syslog.py | 32 ---- 2 files changed, 215 deletions(-) delete mode 100644 cloudinit/config/cc_syslog.py delete mode 100644 tests/unittests/test_handler/test_handler_syslog.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_syslog.py b/cloudinit/config/cc_syslog.py deleted file mode 100644 index 27793f8b..00000000 --- a/cloudinit/config/cc_syslog.py +++ /dev/null @@ -1,183 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2015 Canonical Ltd. -# -# Author: Scott Moser -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit import log as logging -from cloudinit import util -from cloudinit.settings import PER_INSTANCE - -import re - -LOG = logging.getLogger(__name__) - -frequency = PER_INSTANCE - -BUILTIN_CFG = { - 'remotes_file': '/etc/rsyslog.d/20-cloudinit-remotes.conf', - 'remotes': {}, - 'service_name': 'rsyslog', -} - -COMMENT_RE = re.compile(r'[ ]*[#]+[ ]*') -HOST_PORT_RE = re.compile( - r'^(?P[@]{0,2})' - '(([[](?P[^\]]*)[\]])|(?P[^:]*))' - '([:](?P[0-9]+))?$') - - -def parse_remotes_line(line, name=None): - try: - data, comment = COMMENT_RE.split(line) - comment = comment.strip() - except ValueError: - data, comment = (line, None) - - toks = data.strip().split() - match = None - if len(toks) == 1: - host_port = data - elif len(toks) == 2: - match, host_port = toks - else: - raise ValueError("line had multiple spaces: %s" % data) - - toks = HOST_PORT_RE.match(host_port) - - if not toks: - raise ValueError("Invalid host specification '%s'" % host_port) - - proto = toks.group('proto') - addr = toks.group('addr') or toks.group('bracket_addr') - port = toks.group('port') - print("host_port: %s" % addr) - print("port: %s" % port) - - if addr.startswith("[") and not addr.endswith("]"): - raise ValueError("host spec had invalid brackets: %s" % addr) - - if comment and not name: - name = comment - - t = SyslogRemotesLine(name=name, match=match, proto=proto, - addr=addr, port=port) - t.validate() - return t - - -class SyslogRemotesLine(object): - def __init__(self, name=None, match=None, proto=None, addr=None, - port=None): - if not match: - match = "*.*" - self.name = name - self.match = match - if proto == "@": - proto = "udp" - elif proto == "@@": - proto = "tcp" - self.proto = proto - - self.addr = addr - if port: - self.port = int(port) - else: - self.port = None - - def validate(self): - if self.port: - try: - int(self.port) - except ValueError: - raise ValueError("port '%s' is not an integer" % self.port) - - if not self.addr: - raise ValueError("address is required") - - def __repr__(self): - return "[name=%s match=%s proto=%s address=%s port=%s]" % ( - self.name, self.match, self.proto, self.addr, self.port - ) - - def __str__(self): - buf = self.match + " " - if self.proto == "udp": - buf += " @" - elif self.proto == "tcp": - buf += " @@" - - if ":" in self.addr: - buf += "[" + self.addr + "]" - else: - buf += self.addr - - if self.port: - buf += ":%s" % self.port - - if self.name: - buf += " # %s" % self.name - return buf - - -def remotes_to_rsyslog_cfg(remotes, header=None): - if not remotes: - return None - lines = [] - if header is not None: - lines.append(header) - for name, line in remotes.items(): - try: - lines.append(parse_remotes_line(line, name=name)) - except ValueError as e: - LOG.warn("failed loading remote %s: %s [%s]", name, line, e) - return '\n'.join(str(lines)) + '\n' - - -def reload_syslog(systemd, service='rsyslog'): - if systemd: - cmd = ['systemctl', 'reload-or-try-restart', service] - else: - cmd = ['service', service, 'reload'] - try: - util.subp(cmd, capture=True) - except util.ProcessExecutionError as e: - LOG.warn("Failed to reload syslog using '%s': %s", ' '.join(cmd), e) - - -def handle(name, cfg, cloud, log, args): - cfgin = cfg.get('syslog') - if not cfgin: - cfgin = {} - mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) - - remotes_file = mycfg.get('remotes_file') - if util.is_false(remotes_file): - LOG.debug("syslog/remotes_file empty, doing nothing") - return - - remotes = mycfg.get('remotes', {}) - if remotes and not isinstance(remotes, dict): - LOG.warn("syslog/remotes: content is not a dictionary") - return - - config_data = remotes_to_rsyslog_cfg( - remotes, header="#cloud-init syslog module") - - util.write_file(remotes_file, config_data) - - reload_syslog( - systemd=cloud.distro.uses_systemd(), - service=mycfg.get('service_name')) diff --git a/tests/unittests/test_handler/test_handler_syslog.py b/tests/unittests/test_handler/test_handler_syslog.py deleted file mode 100644 index bbfd521e..00000000 --- a/tests/unittests/test_handler/test_handler_syslog.py +++ /dev/null @@ -1,32 +0,0 @@ -from cloudinit.config.cc_syslog import ( - parse_remotes_line, SyslogRemotesLine, remotes_to_rsyslog_cfg) -from cloudinit import util -from .. import helpers as t_help - - -class TestParseRemotesLine(t_help.TestCase): - def test_valid_port(self): - r = parse_remotes_line("foo:9") - self.assertEqual(9, r.port) - - def test_invalid_port(self): - with self.assertRaises(ValueError): - parse_remotes_line("*.* foo:abc") - - def test_valid_ipv6(self): - r = parse_remotes_line("*.* [::1]") - self.assertEqual("*.* [::1]", str(r)) - - def test_valid_ipv6_with_port(self): - r = parse_remotes_line("*.* [::1]:100") - self.assertEqual(r.port, 100) - self.assertEqual(r.addr, "::1") - self.assertEqual("*.* [::1]:100", str(r)) - - def test_invalid_multiple_colon(self): - with self.assertRaises(ValueError): - parse_remotes_line("*.* ::1:100") - - def test_name_in_string(self): - r = parse_remotes_line("syslog.host", name="foobar") - self.assertEqual("*.* syslog.host # foobar", str(r)) -- cgit v1.2.3 From 55472eb02eaa5b88676a96e006f6838020f8ffe3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 28 Jul 2015 11:44:32 -0400 Subject: rsyslog: skip empty or None in remotes format This allows user to specify the following to overwrite a previously declared entry without warnings. rsyslog: {'remotes': {'foo': None}} --- cloudinit/config/cc_rsyslog.py | 2 ++ tests/unittests/test_handler/test_handler_rsyslog.py | 9 +++++++++ 2 files changed, 11 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 5ecf1629..a0132d28 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -307,6 +307,8 @@ def remotes_to_rsyslog_cfg(remotes, header=None, footer=None): if header is not None: lines.append(header) for name, line in remotes.items(): + if not line: + continue try: lines.append(str(parse_remotes_line(line, name=name))) except ValueError as e: diff --git a/tests/unittests/test_handler/test_handler_rsyslog.py b/tests/unittests/test_handler/test_handler_rsyslog.py index 7bfa65a9..b932165c 100644 --- a/tests/unittests/test_handler/test_handler_rsyslog.py +++ b/tests/unittests/test_handler/test_handler_rsyslog.py @@ -163,3 +163,12 @@ class TestRemotesToSyslog(t_help.TestCase): lines = r.splitlines() self.assertTrue(header, lines[0]) self.assertTrue(footer, lines[-1]) + + def test_with_empty_or_null(self): + mycfg = "*.* myhost" + myline = str(parse_remotes_line(mycfg, name="myname")) + r = remotes_to_rsyslog_cfg( + {'myname': mycfg, 'removed': None, 'removed2': ""}) + lines = r.splitlines() + self.assertEqual(1, len(lines)) + self.assertTrue(myline in r.splitlines()) -- cgit v1.2.3 From 328cc7fbaf4d60b51193fb8c14e52d8c6f3273f2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 4 Aug 2015 21:57:57 -0500 Subject: pep8 fixes --- cloudinit/config/cc_rh_subscription.py | 6 +++--- cloudinit/config/cc_rsyslog.py | 1 + cloudinit/sources/DataSourceCloudStack.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 6da26d25..3b30c47e 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -130,9 +130,9 @@ class SubscriptionManager(object): ((not self.auto_attach) or (util.is_false(str(self.auto_attach)))): - no_auto = "The service-level key must be used in conjunction with "\ - "the auto-attach key. Please re-run with auto-attach: "\ - "True" + no_auto = ("The service-level key must be used in conjunction " + "with the auto-attach key. Please re-run with " + "auto-attach: True") return False, no_auto return True, None diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index a0132d28..b8642d65 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -130,6 +130,7 @@ HOST_PORT_RE = re.compile( '(([[](?P[^\]]*)[\]])|(?P[^:]*))' '([:](?P[0-9]+))?$') + def reload_syslog(command=DEF_RELOAD, systemd=False): service = 'rsyslog' if command == DEF_RELOAD: diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index d0cac5bb..64595020 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -41,10 +41,12 @@ class CloudStackPasswordServerClient(object): """ Implements password fetching from the CloudStack password server. - http://cloudstack-administration.readthedocs.org/en/latest/templates.html#adding-password-management-to-your-templates + http://cloudstack-administration.readthedocs.org/ + en/latest/templates.html#adding-password-management-to-your-templates has documentation about the system. This implementation is following that found at - https://github.com/shankerbalan/cloudstack-scripts/blob/master/cloud-set-guest-password-debian + https://github.com/shankerbalan/cloudstack-scripts/ + blob/master/cloud-set-guest-password-debian """ def __init__(self, virtual_router_address): -- cgit v1.2.3 From 827b7b903abc07d5fb04591bbae5587e6dc44993 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 12 Aug 2015 12:51:39 -0400 Subject: swap: create swap with fallocate if possible fallocate is much faster than 'dd' for creating and initializing a swap file. LP: #1482994 --- cloudinit/config/cc_mounts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 73b42f91..47b63dfc 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -206,7 +206,8 @@ def setup_swapfile(fname, size=None, maxsize=None): util.log_time(LOG.debug, msg, func=util.subp, args=[['sh', '-c', ('rm -f "$1" && umask 0066 && ' - 'dd if=/dev/zero "of=$1" bs=1M "count=$2" && ' + '{ fallocate -l "${2}M" "$1" || ' + ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), 'setup_swap', fname, mbsize]]) -- cgit v1.2.3 From ba3e59cbb5ae58a2267fcbcd23eecaaa26f2c396 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 8 Sep 2015 16:53:59 -0400 Subject: power_state: support 'condition' argument if 'condition' is provided to config in power_state, then consult it before powering off. This allows the user to shut down only if a condition is met, and leave the system in a debuggable state otherwise. An example is as simple as: power_state: mode: poweroff condition: ['sh', '-c', '[ -f /disable-poweroff ]'] --- ChangeLog | 1 + cloudinit/config/cc_power_state_change.py | 57 +++++++++++++++++++--- doc/examples/cloud-config-power-state.txt | 9 ++++ .../test_handler/test_handler_power_state.py | 48 ++++++++++++++++-- 4 files changed, 105 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index 6fb70696..bbb7e990 100644 --- a/ChangeLog +++ b/ChangeLog @@ -61,6 +61,7 @@ - status_wrapper in main: fix use of print_exc when handling exception - reporting: add reporting module for web hook or logging of events. - NoCloud: fix consumption of vendordata (LP: #1493453) + - power_state_change: support 'condition' to disable or enable poweroff 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 09d37371..7d9567e3 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,6 +22,7 @@ from cloudinit import util import errno import os import re +import six import subprocess import time @@ -48,10 +49,40 @@ def givecmdline(pid): return None +def check_condition(cond, log=None): + if isinstance(cond, bool): + if log: + log.debug("Static Condition: %s" % cond) + return cond + + pre = "check_condition command (%s): " % cond + try: + proc = subprocess.Popen(cond, shell=not isinstance(cond, list)) + proc.communicate() + ret = proc.returncode + if ret == 0: + if log: + log.debug(pre + "exited 0. condition met.") + return True + elif ret == 1: + if log: + log.debug(pre + "exited 1. condition not met.") + return False + else: + if log: + log.warn(pre + "unexpected exit %s. " % ret + + "do not apply change.") + return False + except Exception as e: + if log: + log.warn(pre + "Unexpected error: %s" % e) + return False + + def handle(_name, cfg, _cloud, log, _args): try: - (args, timeout) = load_power_state(cfg) + (args, timeout, condition) = load_power_state(cfg) if args is None: log.debug("no power_state provided. doing nothing") return @@ -59,6 +90,10 @@ def handle(_name, cfg, _cloud, log, _args): log.warn("%s Not performing power state change!" % str(e)) return + if condition is False: + log.debug("Condition was false. Will not perform state change.") + return + mypid = os.getpid() cmdline = givecmdline(mypid) @@ -70,8 +105,8 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, execmd, - [args, devnull_fp]) + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + condition, execmd, [args, devnull_fp]) def load_power_state(cfg): @@ -80,7 +115,7 @@ def load_power_state(cfg): pstate = cfg.get('power_state') if pstate is None: - return (None, None) + return (None, None, None) if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") @@ -115,7 +150,10 @@ def load_power_state(cfg): raise ValueError("failed to convert timeout '%s' to float." % pstate['timeout']) - return (args, timeout) + condition = pstate.get("condition", True) + if not isinstance(condition, six.string_types + (list, bool)): + raise TypeError("condition type %s invalid. must be list, bool, str") + return (args, timeout, condition) def doexit(sysexit): @@ -133,7 +171,7 @@ def execmd(exe_args, output=None, data_in=None): doexit(ret) -def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args): +def run_after_pid_gone(pid, pidcmdline, timeout, log, condition, func, args): # wait until pid, with /proc/pid/cmdline contents of pidcmdline # is no longer alive. After it is gone, or timeout has passed # execute func(args) @@ -175,4 +213,11 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args): if log: log.debug(msg) + + try: + if not check_condition(condition, log): + return + except Exception as e: + fatal("Unexpected Exception when checking condition: %s" % e) + func(*args) diff --git a/doc/examples/cloud-config-power-state.txt b/doc/examples/cloud-config-power-state.txt index 8df14366..b470153d 100644 --- a/doc/examples/cloud-config-power-state.txt +++ b/doc/examples/cloud-config-power-state.txt @@ -23,9 +23,18 @@ # message: provided as the message argument to 'shutdown'. default is none. # timeout: the amount of time to give the cloud-init process to finish # before executing shutdown. +# condition: apply state change only if condition is met. +# May be boolean True (always met), or False (never met), +# or a command string or list to be executed. +# command's exit code indicates: +# 0: condition met +# 1: condition not met +# other exit codes will result in 'not met', but are reserved +# for future use. # power_state: delay: "+30" mode: poweroff message: Bye Bye timeout: 30 + condition: True diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 2f86b8f8..5687b10d 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -1,6 +1,9 @@ +import sys + from cloudinit.config import cc_power_state_change as psc from .. import helpers as t_help +from ..helpers import mock class TestLoadPowerState(t_help.TestCase): @@ -9,12 +12,12 @@ class TestLoadPowerState(t_help.TestCase): def test_no_config(self): # completely empty config should mean do nothing - (cmd, _timeout) = psc.load_power_state({}) + (cmd, _timeout, _condition) = psc.load_power_state({}) self.assertEqual(cmd, None) def test_irrelevant_config(self): # no power_state field in config should return None for cmd - (cmd, _timeout) = psc.load_power_state({'foo': 'bar'}) + (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}) self.assertEqual(cmd, None) def test_invalid_mode(self): @@ -53,23 +56,60 @@ class TestLoadPowerState(t_help.TestCase): def test_no_message(self): # if message is not present, then no argument should be passed for it cfg = {'power_state': {'mode': 'poweroff'}} - (cmd, _timeout) = psc.load_power_state(cfg) + (cmd, _timeout, _condition) = psc.load_power_state(cfg) self.assertNotIn("", cmd) check_lps_ret(psc.load_power_state(cfg)) self.assertTrue(len(cmd) == 3) + def test_condition_null_raises(self): + cfg = {'power_state': {'mode': 'poweroff', 'condition': None}} + self.assertRaises(TypeError, psc.load_power_state, cfg) + + def test_condition_default_is_true(self): + cfg = {'power_state': {'mode': 'poweroff'}} + _cmd, _timeout, cond = psc.load_power_state(cfg) + self.assertEqual(cond, True) + + +class TestCheckCondition(t_help.TestCase): + def cmd_with_exit(self, rc): + return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc]) + + def test_true_is_true(self): + self.assertEqual(psc.check_condition(True), True) + + def test_false_is_false(self): + self.assertEqual(psc.check_condition(False), False) + + def test_cmd_exit_zero_true(self): + self.assertEqual(psc.check_condition(self.cmd_with_exit(0)), True) + + def test_cmd_exit_one_false(self): + self.assertEqual(psc.check_condition(self.cmd_with_exit(1)), False) + + def test_cmd_exit_nonzero_warns(self): + mocklog = mock.Mock() + self.assertEqual( + psc.check_condition(self.cmd_with_exit(2), mocklog), False) + self.assertEqual(mocklog.warn.call_count, 1) + + def check_lps_ret(psc_return, mode=None): - if len(psc_return) != 2: + if len(psc_return) != 3: raise TypeError("length returned = %d" % len(psc_return)) errs = [] cmd = psc_return[0] timeout = psc_return[1] + condition = psc_return[2] if 'shutdown' not in psc_return[0][0]: errs.append("string 'shutdown' not in cmd") + if 'condition' is None: + errs.append("condition was not returned") + if mode is not None: opt = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'}[mode] if opt not in psc_return[0]: -- cgit v1.2.3 From 6f2b8551e72596adfc685357d8471c454bd96d63 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 11 Sep 2015 13:38:14 -0600 Subject: Ubuntu Snappy: conditionally enable SSH on Snappy When a user provides authentication tokens, enable SSH unless SSH has been explicitly disabled (LP: #1494816). --- cloudinit/config/cc_snappy.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 7aaec94a..e36542bf 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -274,7 +274,20 @@ def handle(name, cfg, cloud, log, args): LOG.warn("'%s' failed for '%s': %s", pkg_op['op'], pkg_op['name'], e) - disable_enable_ssh(mycfg.get('ssh_enabled', False)) + # Default to disabling SSH + ssh_enabled = mycfg.get('ssh_enabled', False) + + # If the user has not explicitly enabled or disabled SSH, then enable it + # when password SSH authentication is requested or there are SSH keys + if mycfg.get('ssh_enabled', None) is not False: + if len(mycfg.get('public-keys', [])) > 0: + LOG.debug("Enabling SSH, user SSH keys provided") + ssh_enabled = True + elif mycfg.get('ssh_pwauth', False): + LOG.debug("Enabling SSH, password authentication requested") + ssh_enabled = True + + disable_enable_ssh(ssh_enabled) if fails: raise Exception("failed to install/configure snaps") -- cgit v1.2.3 From fd6b08c4d03b07be67398450e40e7e2f91e8db51 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 11 Sep 2015 14:04:52 -0600 Subject: Refinements on SSH enablement --- cloudinit/config/cc_snappy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index e36542bf..899df10c 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -280,10 +280,12 @@ def handle(name, cfg, cloud, log, args): # If the user has not explicitly enabled or disabled SSH, then enable it # when password SSH authentication is requested or there are SSH keys if mycfg.get('ssh_enabled', None) is not False: - if len(mycfg.get('public-keys', [])) > 0: + user_ssh_keys = cloud.get_public_ssh_keys() or None + password_auth_enabled = cfg.get('ssh_pwauth', False) + if user_ssh_keys: LOG.debug("Enabling SSH, user SSH keys provided") ssh_enabled = True - elif mycfg.get('ssh_pwauth', False): + elif password_auth_enabled: LOG.debug("Enabling SSH, password authentication requested") ssh_enabled = True -- cgit v1.2.3 From 988174dca9e4e5593b357c6def82c857f718282d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 11 Sep 2015 16:52:26 -0400 Subject: cc_snappy: update doc string, change default to 'auto' --- cloudinit/config/cc_snappy.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 899df10c..124452c0 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -6,7 +6,7 @@ Example config: #cloud-config snappy: system_snappy: auto - ssh_enabled: False + ssh_enabled: auto packages: [etcd, pkg2.smoser] config: pkgname: @@ -16,7 +16,12 @@ Example config: packages_dir: '/writable/user-data/cloud-init/snaps' - ssh_enabled: - This defaults to 'False'. Set to a non-false value to enable ssh service + This controls the system's ssh service. The default value is 'auto'. + True: enable ssh service + False: disable ssh service + auto: enable ssh service if either ssh keys have been provided + or user has requested password authentication (ssh_pwauth). + - snap installation and config The above would install 'etcd', and then install 'pkg2.smoser' with a '' argument where 'config-file' has 'config-blob' inside it. @@ -275,19 +280,23 @@ def handle(name, cfg, cloud, log, args): pkg_op['op'], pkg_op['name'], e) # Default to disabling SSH - ssh_enabled = mycfg.get('ssh_enabled', False) + ssh_enabled = mycfg.get('ssh_enabled', "auto") # If the user has not explicitly enabled or disabled SSH, then enable it # when password SSH authentication is requested or there are SSH keys - if mycfg.get('ssh_enabled', None) is not False: + if ssh_enabled == "auto": user_ssh_keys = cloud.get_public_ssh_keys() or None password_auth_enabled = cfg.get('ssh_pwauth', False) if user_ssh_keys: - LOG.debug("Enabling SSH, user SSH keys provided") + LOG.debug("Enabling SSH, ssh keys found in datasource") ssh_enabled = True + elif cfg.get('ssh_authorized_keys'): + LOG.debug("Enabling SSH, ssh keys found in config") elif password_auth_enabled: LOG.debug("Enabling SSH, password authentication requested") ssh_enabled = True + elif ssh_enabled not in (True, False): + LOG.warn("Unknown value '%s' in ssh_enabled", ssh_enabled) disable_enable_ssh(ssh_enabled) -- cgit v1.2.3 From 03b5cac37154476b89e67b231c2888a9cfdc92ca Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 15 Sep 2015 11:53:36 -0600 Subject: Change Snappy SSH enabled default from false to 'auto' (LP: #1494816) --- cloudinit/config/cc_snappy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 124452c0..fa9d54a0 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -63,7 +63,7 @@ NAMESPACE_DELIM = '.' BUILTIN_CFG = { 'packages': [], 'packages_dir': '/writable/user-data/cloud-init/snaps', - 'ssh_enabled': False, + 'ssh_enabled': "auto", 'system_snappy': "auto", 'config': {}, } -- cgit v1.2.3 From 86bd318e41b0bec10765d0498a125de062afe1f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 9 Oct 2015 12:39:23 -0400 Subject: support configuring and installing the Ubuntu fan driver #cloud-config fan: config: | # fan 240 10.0.0.0/8 eth0/16 dhcp 10.0.0.0/8 eth1/16 dhcp off # fan 241 241.0.0.0/8 eth0/16 dhcp config_path: /etc/network/fan LP: #1504604 --- ChangeLog | 1 + cloudinit/config/cc_fan.py | 101 +++++++++++++++++++++++++++++++++++++++++++++ config/cloud.cfg | 1 + 3 files changed, 103 insertions(+) create mode 100644 cloudinit/config/cc_fan.py (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index bbb7e990..b7a66aa1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -62,6 +62,7 @@ - reporting: add reporting module for web hook or logging of events. - NoCloud: fix consumption of vendordata (LP: #1493453) - power_state_change: support 'condition' to disable or enable poweroff + - ubuntu fan: support for config and installing of ubuntu fan (LP: #1504604) 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py new file mode 100644 index 00000000..39e3850e --- /dev/null +++ b/cloudinit/config/cc_fan.py @@ -0,0 +1,101 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2015 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +fan module allows configuration of Ubuntu Fan + https://wiki.ubuntu.com/FanNetworking + +Example config: + #cloud-config + fan: + config: | + # fan 240 + 10.0.0.0/8 eth0/16 dhcp + 10.0.0.0/8 eth1/16 dhcp off + # fan 241 + 241.0.0.0/8 eth0/16 dhcp + config_path: /etc/network/fan + +If cloud-init sees a 'fan' entry in cloud-config it will + a.) write 'config_path' with the contents + b.) install the package 'ubuntu-fan' if it is not installed + c.) ensure the service is started (or restarted if was previously running) +""" + +from cloudinit import log as logging +from cloudinit import util +from cloudinit.settings import PER_INSTANCE + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE + +BUILTIN_CFG = { + 'config': None, + 'config_path': '/etc/network/fan', +} + + +def stop_update_start(service, config_file, content, systemd=False): + if systemd: + cmds = {'stop': ['systemctl', 'stop', service], + 'start': ['systemctl', 'start', service], + 'enable': ['systemctl', 'enable', service]} + else: + cmds = {'stop': ['service', 'stop'], + 'start': ['service', 'start']} + + def run(cmd, msg): + try: + return util.subp(cmd, capture=True) + except util.ProcessExecutionError as e: + LOG.warn("failed: %s (%s): %s", service, cmd, e) + return False + + stop_failed = not run(cmds['stop'], msg='stop %s' % service) + if not content.endswith('\n'): + content += '\n' + util.write_file(config_file, content, omode="w") + + ret = run(cmds['start'], msg='start %s' % service) + if ret and stop_failed: + LOG.warn("success: %s started", service) + + if 'enable' in cmds: + ret = run(cmds['enable'], msg='enable %s' % service) + + return ret + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('fan') + if not cfgin: + cfgin = {} + mycfg = util.mergemanydict([cfgin, BUILTIN_CFG]) + + if not mycfg.get('config'): + LOG.debug("%s: no 'fan' config entry. disabling", name) + return + + util.write_file(mycfg.get('config_path'), mycfg.get('config'), omode="w") + distro = cloud.distro + if not util.which('fanctl'): + distro.install_packages(['ubuntu-fan']) + + stop_update_start( + service='ubuntu-fan', config_file=mycfg.get('config_path'), + content=mycfg.get('config'), systemd=distro.uses_systemd()) diff --git a/config/cloud.cfg b/config/cloud.cfg index 2b27f379..74794ab0 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -53,6 +53,7 @@ cloud_config_modules: - apt-pipelining - apt-configure - package-update-upgrade-install + - fan - landscape - timezone - puppet -- cgit v1.2.3 From f1db8eaa68dadaae6a591339f69994e3afb589c3 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Mon, 9 Nov 2015 16:40:43 -0700 Subject: With Ubuntu 15.10, "nobootwait" != "nofail". The "nobootwait" was an Ubuntu specific option. This change was dropped in 15.10 (LP: #1514485). --- cloudinit/config/cc_mounts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 47b63dfc..11089d8d 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -263,7 +263,11 @@ def handle_swapcfg(swapcfg): def handle(_name, cfg, cloud, log, _args): # fs_spec, fs_file, fs_vfstype, fs_mntops, fs-freq, fs_passno - defvals = [None, None, "auto", "defaults,nobootwait", "0", "2"] + def_mnt_opts = "defaults,nobootwait" + if cloud.distro.uses_systemd(): + def_mnt_opts = "defaults,nofail" + + defvals = [None, None, "auto", def_mnt_opts, "0", "2"] defvals = cfg.get("mount_default_fields", defvals) # these are our default set of mounts -- cgit v1.2.3 From ee40614b0a34a110265493c176c64db823aa34b3 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Wed, 3 Feb 2016 22:21:40 -0600 Subject: lxd: add support for setting up lxd using 'lxd init' If lxd key is present in cfg, then run 'lxd init' with values from the 'init' entry in lxd configuration as flags. --- ChangeLog | 1 + cloudinit/config/cc_lxd.py | 50 +++++++++++++++++++ config/cloud.cfg | 1 + tests/unittests/test_handler/test_handler_lxd.py | 62 ++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 cloudinit/config/cc_lxd.py create mode 100644 tests/unittests/test_handler/test_handler_lxd.py (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index 0ba16492..9fbc920d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -71,6 +71,7 @@ - Azure: get instance id from dmi instead of SharedConfig (LP: #1506187) - systemd/power_state: fix power_state to work even if cloud-final exited non-zero (LP: #1449318) + - lxd: add support for setting up lxd using 'lxd init' 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py new file mode 100644 index 00000000..0db8356b --- /dev/null +++ b/cloudinit/config/cc_lxd.py @@ -0,0 +1,50 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +This module initializes lxd using 'lxd init' + +Example config: + #cloud-config + lxd: + init: + network_address: + network_port: + storage_backend: + storage_create_device: + storage_create_loop: + storage_pool: + trust_password: +""" + +from cloudinit import util + + +def handle(name, cfg, cloud, log, args): + if not cfg.get('lxd') and cfg['lxd'].get('init'): + log.debug("Skipping module named %s, not present or disabled by cfg") + return + lxd_conf = cfg['lxd']['init'] + keys = ('network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', 'storage_pool', + 'trust_password') + cmd = ['lxd', 'init', '--auto'] + for k in keys: + if lxd_conf.get(k): + cmd.extend(["--%s" % k.replace('_', '-'), lxd_conf[k]]) + util.subp(cmd) diff --git a/config/cloud.cfg b/config/cloud.cfg index 74794ab0..795df19f 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -56,6 +56,7 @@ cloud_config_modules: - fan - landscape - timezone + - lxd - puppet - chef - salt-minion diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py new file mode 100644 index 00000000..89863d52 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -0,0 +1,62 @@ +from cloudinit.config import cc_lxd +from cloudinit import (util, distros, helpers, cloud) +from cloudinit.sources import DataSourceNoCloud +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestLxd(t_help.TestCase): + def setUp(self): + super(TestLxd, self).setUp() + self.unapply = [] + apply_patches([(util, 'subp', self._mock_subp)]) + self.subp_called = [] + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + + def _mock_subp(self, *args, **kwargs): + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + return + + def _get_cloud(self, distro): + cls = distros.fetch(distro) + paths = helpers.Paths({}) + d = cls(distro, {}, paths) + ds = DataSourceNoCloud.DataSourceNoCloud({}, d, paths) + cc = cloud.Cloud(ds, paths, {}, d, None) + return cc + + def test_lxd_init(self): + cfg = { + 'lxd': { + 'init': { + 'network_address': '0.0.0.0', + 'storage_backend': 'zfs', + 'storage_pool': 'poolname', + } + } + } + cc = self._get_cloud('ubuntu') + cc_lxd.handle('cc_lxd', cfg, cc, LOG, []) + + self.assertEqual( + self.subp_called[0].get('args'), + ['lxd', 'init', '--auto', '--network-address', '0.0.0.0', + '--storage-backend', 'zfs', '--storage-pool', 'poolname']) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret -- cgit v1.2.3 From a2e251c46307fed0b91e34084c361816829f251d Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Thu, 4 Feb 2016 19:09:05 -0600 Subject: - Ensure that lxd is installed before running lxd init. - Handle init cfg separately from main cfg to allow multiple sections under lxd config to be handled independantly. - Check for properly formatted lxd init cfg --- cloudinit/config/cc_lxd.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 0db8356b..c9cf8704 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -36,15 +36,31 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): - if not cfg.get('lxd') and cfg['lxd'].get('init'): + # Get config + lxd_cfg = cfg.get('lxd') + if not lxd_cfg and isinstance(lxd_cfg, dict): log.debug("Skipping module named %s, not present or disabled by cfg") return - lxd_conf = cfg['lxd']['init'] - keys = ('network_address', 'network_port', 'storage_backend', - 'storage_create_device', 'storage_create_loop', 'storage_pool', - 'trust_password') - cmd = ['lxd', 'init', '--auto'] - for k in keys: - if lxd_conf.get(k): - cmd.extend(["--%s" % k.replace('_', '-'), lxd_conf[k]]) - util.subp(cmd) + + # Ensure lxd is installed + if not util.which("lxd"): + try: + cloud.distro.install_packages(("lxd",)) + except util.ProcessExecutionError as e: + log.warn("no lxd executable and could not install lxd: '%s'" % e) + return + + # Set up lxd if init config is given + init_cfg = lxd_cfg.get('init') + if init_cfg: + if not isinstance(init_cfg, dict): + log.warn("lxd init config must be a dict of flag: val pairs") + return + init_keys = ('network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', + 'storage_pool', 'trust_password') + cmd = ['lxd', 'init', '--auto'] + for k in init_keys: + if init_cfg.get(k): + cmd.extend(["--%s" % k.replace('_', '-'), init_cfg[k]]) + util.subp(cmd) -- cgit v1.2.3 From b20191f04c586147165a304b88a2b89c89f79225 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Feb 2016 14:32:14 -0500 Subject: minor cleanups --- cloudinit/config/cc_lxd.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index c9cf8704..aaafb643 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -47,18 +47,20 @@ def handle(name, cfg, cloud, log, args): try: cloud.distro.install_packages(("lxd",)) except util.ProcessExecutionError as e: - log.warn("no lxd executable and could not install lxd: '%s'" % e) + log.warn("no lxd executable and could not install lxd:", e) return # Set up lxd if init config is given + init_keys = ( + 'network_address', 'network_port', 'storage_backend', + 'storage_create_device', 'storage_create_loop', + 'storage_pool', 'trust_password') init_cfg = lxd_cfg.get('init') if init_cfg: if not isinstance(init_cfg, dict): - log.warn("lxd init config must be a dict of flag: val pairs") + log.warn("lxd/init config must be a dictionary. found a '%s'", + type(f)) return - init_keys = ('network_address', 'network_port', 'storage_backend', - 'storage_create_device', 'storage_create_loop', - 'storage_pool', 'trust_password') cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): -- cgit v1.2.3 From 14915526ca67bbf7842028d48170015b85f87469 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 1 Mar 2016 00:19:55 -0500 Subject: lxd: general fix after testing A few changes: a.) change to using '--name=value' rather than '--name' 'value' b.) make sure only strings are passed to command (useful for storage_create_loop: which is likely an integer) c.) document simple working example d.) support installing zfs if not present and storage_backedn has it. --- cloudinit/config/cc_lxd.py | 35 ++++++++++++++++++------ doc/examples/cloud-config-lxd.txt | 7 +++++ tests/unittests/test_handler/test_handler_lxd.py | 9 +++--- 3 files changed, 38 insertions(+), 13 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index aaafb643..84eec7a5 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -38,16 +38,36 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') - if not lxd_cfg and isinstance(lxd_cfg, dict): + if not lxd_cfg: log.debug("Skipping module named %s, not present or disabled by cfg") return + if not isinstance(lxd_cfg, dict): + log.warn("lxd config must be a dictionary. found a '%s'", + type(lxd_cfg)) + return + + init_cfg = lxd_cfg.get('init') + if not init_cfg: + init_cfg = {} + + if not isinstance(init_cfg, dict): + log.warn("lxd/init config must be a dictionary. found a '%s'", + type(init_cfg)) + init_cfg = {} + + packages = [] + if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): + packages.append('zfs') # Ensure lxd is installed if not util.which("lxd"): + packages.append('lxd') + + if len(packages): try: - cloud.distro.install_packages(("lxd",)) + cloud.distro.install_packages(packages) except util.ProcessExecutionError as e: - log.warn("no lxd executable and could not install lxd:", e) + log.warn("failed to install packages %s: %s", packages, e) return # Set up lxd if init config is given @@ -55,14 +75,11 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - init_cfg = lxd_cfg.get('init') + if init_cfg: - if not isinstance(init_cfg, dict): - log.warn("lxd/init config must be a dictionary. found a '%s'", - type(f)) - return cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): - cmd.extend(["--%s" % k.replace('_', '-'), init_cfg[k]]) + cmd.extend(["--%s=%s" % + (k.replace('_', '-'), str(init_cfg[k]))]) util.subp(cmd) diff --git a/doc/examples/cloud-config-lxd.txt b/doc/examples/cloud-config-lxd.txt index f66da4c3..b9bb4aa5 100644 --- a/doc/examples/cloud-config-lxd.txt +++ b/doc/examples/cloud-config-lxd.txt @@ -19,3 +19,10 @@ lxd: network_port: 8443 storage_backend: zfs storage_pool: datapool + storage_create_loop: 10 + + +# The simplist working configuration is +# lxd: +# init: +# storage_backend: dir diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 4d858b8f..65794a41 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -43,9 +43,10 @@ class TestLxd(t_help.TestCase): self.assertTrue(mock_util.which.called) init_call = mock_util.subp.call_args_list[0][0][0] self.assertEquals(init_call, - ['lxd', 'init', '--auto', '--network-address', - '0.0.0.0', '--storage-backend', 'zfs', - '--storage-pool', 'poolname']) + ['lxd', 'init', '--auto', + '--network-address=0.0.0.0', + '--storage-backend=zfs', + '--storage-pool=poolname']) @mock.patch("cloudinit.config.cc_lxd.util") def test_lxd_install(self, mock_util): @@ -55,4 +56,4 @@ class TestLxd(t_help.TestCase): cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) self.assertTrue(cc.distro.install_packages.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] - self.assertEquals(install_pkg, ('lxd',)) + self.assertEquals(sorted(install_pkg), ['lxd', 'zfs']) -- cgit v1.2.3 From 568d15d7fb239e609fb70cc7c7a08205e640bf25 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 2 Mar 2016 13:23:55 -0600 Subject: Fix logic error in lxd config check If the cloud-config does not contain and lxd dictionary then we should not attempt to install the package. Change the latter half of the check to negate the dictionary type check. This fix prevents us from always installing lxd, rather than only installing when we have a config. Fix pyflakes check on init_cfg dict error message. --- cloudinit/config/cc_lxd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index aaafb643..7d8a0202 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -38,7 +38,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') - if not lxd_cfg and isinstance(lxd_cfg, dict): + if not lxd_cfg and not isinstance(lxd_cfg, dict): log.debug("Skipping module named %s, not present or disabled by cfg") return @@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args): if init_cfg: if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(f)) + type(init_cfg)) return cmd = ['lxd', 'init', '--auto'] for k in init_keys: -- cgit v1.2.3 From 96f1742b36241cee152aa2cb5b4a5e1a267a4770 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 15:17:24 -0500 Subject: fix lxd module to not do anything unless config provided --- cloudinit/config/cc_lxd.py | 30 ++++++++++++------------ tests/unittests/test_handler/test_handler_lxd.py | 16 +++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 84eec7a5..80a4d219 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -47,22 +47,24 @@ def handle(name, cfg, cloud, log, args): return init_cfg = lxd_cfg.get('init') - if not init_cfg: - init_cfg = {} - if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", type(init_cfg)) init_cfg = {} - packages = [] - if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): - packages.append('zfs') + if not init_cfg: + log.debug("no lxd/init config. disabled.") + return + packages = [] # Ensure lxd is installed if not util.which("lxd"): packages.append('lxd') - + + # if using zfs, get the utils + if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): + packages.append('zfs') + if len(packages): try: cloud.distro.install_packages(packages) @@ -75,11 +77,9 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') - - if init_cfg: - cmd = ['lxd', 'init', '--auto'] - for k in init_keys: - if init_cfg.get(k): - cmd.extend(["--%s=%s" % - (k.replace('_', '-'), str(init_cfg[k]))]) - util.subp(cmd) + cmd = ['lxd', 'init', '--auto'] + for k in init_keys: + if init_cfg.get(k): + cmd.extend(["--%s=%s" % + (k.replace('_', '-'), str(init_cfg[k]))]) + util.subp(cmd) diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 65794a41..7ffa2a53 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -57,3 +57,19 @@ class TestLxd(t_help.TestCase): self.assertTrue(cc.distro.install_packages.called) install_pkg = cc.distro.install_packages.call_args_list[0][0][0] self.assertEquals(sorted(install_pkg), ['lxd', 'zfs']) + + @mock.patch("cloudinit.config.cc_lxd.util") + def test_no_init_does_nothing(self, mock_util): + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, LOG, []) + self.assertFalse(cc.distro.install_packages.called) + self.assertFalse(mock_util.subp.called) + + @mock.patch("cloudinit.config.cc_lxd.util") + def test_no_lxd_does_nothing(self, mock_util): + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc_lxd.handle('cc_lxd', {'package_update': True}, cc, LOG, []) + self.assertFalse(cc.distro.install_packages.called) + self.assertFalse(mock_util.subp.called) -- cgit v1.2.3 From cb64cf1e14a474794654f5d1586b117912bed4f9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 3 Mar 2016 15:21:48 -0500 Subject: fix some of pylints complaints --- cloudinit/config/cc_lxd.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 80a4d219..63b8fb63 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args): init_cfg = lxd_cfg.get('init') if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + type(init_cfg)) init_cfg = {} if not init_cfg: @@ -62,14 +62,14 @@ def handle(name, cfg, cloud, log, args): packages.append('lxd') # if using zfs, get the utils - if (init_cfg.get("storage_backend") == "zfs" and not util.which('zfs')): + if init_cfg.get("storage_backend") == "zfs" and not util.which('zfs'): packages.append('zfs') if len(packages): try: cloud.distro.install_packages(packages) - except util.ProcessExecutionError as e: - log.warn("failed to install packages %s: %s", packages, e) + except util.ProcessExecutionError as exc: + log.warn("failed to install packages %s: %s", packages, exc) return # Set up lxd if init config is given -- cgit v1.2.3 From 8cb7c3f7b5339e686bfbf95996b51afafeaf9c9e Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 3 Mar 2016 16:20:10 -0600 Subject: Update pep8 runner and fix pep8 issues --- Makefile | 9 ++-- bin/cloud-init | 43 +++++++++--------- cloudinit/config/cc_apt_configure.py | 6 ++- cloudinit/config/cc_disk_setup.py | 31 +++++++------ cloudinit/config/cc_grub_dpkg.py | 8 ++-- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 12 ++--- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_puppet.py | 6 +-- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rh_subscription.py | 4 +- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_ssh.py | 7 +-- cloudinit/config/cc_update_etc_hosts.py | 6 +-- cloudinit/config/cc_update_hostname.py | 2 +- cloudinit/config/cc_yum_add_repo.py | 2 +- cloudinit/distros/__init__.py | 12 ++--- cloudinit/distros/arch.py | 6 +-- cloudinit/distros/debian.py | 5 ++- cloudinit/distros/freebsd.py | 4 +- cloudinit/distros/gentoo.py | 4 +- cloudinit/distros/parsers/hostname.py | 2 +- cloudinit/distros/parsers/resolv_conf.py | 2 +- cloudinit/distros/parsers/sys_conf.py | 7 ++- cloudinit/filters/launch_index.py | 2 +- cloudinit/helpers.py | 7 +-- cloudinit/sources/DataSourceAzure.py | 21 +++++---- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceEc2.py | 10 ++--- cloudinit/sources/DataSourceMAAS.py | 15 ++++--- cloudinit/sources/DataSourceOVF.py | 4 +- cloudinit/sources/DataSourceOpenNebula.py | 3 +- cloudinit/sources/DataSourceSmartOS.py | 7 ++- cloudinit/ssh_util.py | 3 +- cloudinit/stages.py | 18 ++++---- cloudinit/url_helper.py | 6 +-- cloudinit/util.py | 15 ++++--- tests/unittests/test_data.py | 5 ++- tests/unittests/test_datasource/test_altcloud.py | 23 +++++----- tests/unittests/test_datasource/test_azure.py | 15 ++++--- .../unittests/test_datasource/test_configdrive.py | 12 ++--- tests/unittests/test_datasource/test_maas.py | 16 +++---- tests/unittests/test_datasource/test_smartos.py | 6 +-- .../test_handler/test_handler_power_state.py | 3 +- .../test_handler/test_handler_seed_random.py | 3 +- .../unittests/test_handler/test_handler_snappy.py | 3 +- tests/unittests/test_sshutil.py | 3 +- tests/unittests/test_templating.py | 3 +- tools/hacking.py | 16 +++---- tools/mock-meta.py | 27 +++++++----- tools/run-pep8 | 51 ++++++++-------------- 52 files changed, 244 insertions(+), 243 deletions(-) (limited to 'cloudinit/config') diff --git a/Makefile b/Makefile index 058ac199..fb65b70b 100644 --- a/Makefile +++ b/Makefile @@ -20,13 +20,14 @@ all: test check_version check: pep8 pyflakes pyflakes3 unittest pep8: - @$(CWD)/tools/run-pep8 $(PY_FILES) + @$(CWD)/tools/run-pep8 pyflakes: - @$(CWD)/tools/tox-venv py27 pyflakes $(PY_FILES) + @$(CWD)/tools/run-pyflakes -pyflakes: - @$(CWD)/tools/tox-venv py34 pyflakes $(PY_FILES) +pyflakes3: + @$(CWD)/tools/run-pyflakes3 + unittest: nosetests $(noseopts) tests/unittests diff --git a/bin/cloud-init b/bin/cloud-init index 9b90c45e..7f665e7e 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -194,7 +194,7 @@ def main_init(name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(init.cfg) apply_reporting_cfg(init.cfg) @@ -276,9 +276,9 @@ def main_init(name, args): # This may run user-data handlers and/or perform # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_data', - init.consume_data, - args=[PER_INSTANCE], - freq=PER_INSTANCE) + init.consume_data, + args=[PER_INSTANCE], + freq=PER_INSTANCE) if not ran: # Just consume anything that is set to run per-always # if nothing ran in the per-instance code @@ -349,7 +349,7 @@ def main_modules(action_name, args): if args.debug: # Reset so that all the debug handlers are closed out LOG.debug(("Logging being reset, this logger may no" - " longer be active shortly")) + " longer be active shortly")) logging.resetLogging() logging.setupLogging(mods.cfg) apply_reporting_cfg(init.cfg) @@ -534,7 +534,8 @@ def status_wrapper(name, args, data_d=None, link_d=None): errors.extend(v1[m].get('errors', [])) atomic_write_json(result_path, - {'v1': {'datasource': v1['datasource'], 'errors': errors}}) + {'v1': {'datasource': v1['datasource'], + 'errors': errors}}) util.sym_link(os.path.relpath(result_path, link_d), result_link, force=True) @@ -578,13 +579,13 @@ def main(): # These settings are used for the 'config' and 'final' stages parser_mod = subparsers.add_parser('modules', - help=('activates modules ' - 'using a given configuration key')) + help=('activates modules using ' + 'a given configuration key')) parser_mod.add_argument("--mode", '-m', action='store', - help=("module configuration name " - "to use (default: %(default)s)"), - default='config', - choices=('init', 'config', 'final')) + help=("module configuration name " + "to use (default: %(default)s)"), + default='config', + choices=('init', 'config', 'final')) parser_mod.set_defaults(action=('modules', main_modules)) # These settings are used when you want to query information @@ -600,22 +601,22 @@ def main(): # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', - help=('run a single module ')) + help=('run a single module ')) parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", - help="module name to run", - required=True) + help="module name to run", + required=True) parser_single.add_argument("--frequency", action="store", - help=("frequency of the module"), - required=False, - choices=list(FREQ_SHORT_NAMES.keys())) + help=("frequency of the module"), + required=False, + choices=list(FREQ_SHORT_NAMES.keys())) parser_single.add_argument("--report", action="store_true", help="enable reporting", required=False) parser_single.add_argument("module_args", nargs="*", - metavar='argument', - help=('any additional arguments to' - ' pass to this module')) + metavar='argument', + help=('any additional arguments to' + ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) args = parser.parse_args() diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 9e9e9e26..702977cb 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -91,7 +91,8 @@ def handle(name, cfg, cloud, log, _args): if matchcfg: matcher = re.compile(matchcfg).search else: - matcher = lambda f: False + def matcher(x): + return False errors = add_sources(cfg['apt_sources'], params, aa_repo_match=matcher) @@ -173,7 +174,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - aa_repo_match = lambda f: False + def aa_repo_match(x): + return False errorlist = [] for ent in srclist: diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index d5b0d1d7..0ecc2e4c 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -167,11 +167,12 @@ def enumerate_disk(device, nodeps=False): parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0] for part in parts: - d = {'name': None, - 'type': None, - 'fstype': None, - 'label': None, - } + d = { + 'name': None, + 'type': None, + 'fstype': None, + 'label': None, + } for key, value in value_splitter(part): d[key.lower()] = value @@ -701,11 +702,12 @@ def lookup_force_flag(fs): """ A force flag might be -F or -F, this look it up """ - flags = {'ext': '-F', - 'btrfs': '-f', - 'xfs': '-f', - 'reiserfs': '-f', - } + flags = { + 'ext': '-F', + 'btrfs': '-f', + 'xfs': '-f', + 'reiserfs': '-f', + } if 'ext' in fs.lower(): fs = 'ext' @@ -824,10 +826,11 @@ def mkfs(fs_cfg): # Create the commands if fs_cmd: - fs_cmd = fs_cfg['cmd'] % {'label': label, - 'filesystem': fs_type, - 'device': device, - } + fs_cmd = fs_cfg['cmd'] % { + 'label': label, + 'filesystem': fs_type, + 'device': device, + } else: # Find the mkfs command mkfs_cmd = util.which("mkfs.%s" % fs_type) diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 456597af..acd3e60a 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -38,11 +38,11 @@ def handle(name, cfg, _cloud, log, _args): idevs = util.get_cfg_option_str(mycfg, "grub-pc/install_devices", None) idevs_empty = util.get_cfg_option_str(mycfg, - "grub-pc/install_devices_empty", None) + "grub-pc/install_devices_empty", + None) if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or - (os.path.exists("/dev/xvda1") - and not os.path.exists("/dev/xvda"))): + (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))): if idevs is None: idevs = "" if idevs_empty is None: @@ -66,7 +66,7 @@ def handle(name, cfg, _cloud, log, _args): (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % - (idevs, idevs_empty)) + (idevs, idevs_empty)) try: util.subp(['debconf-set-selections'], dconf_sel) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index f1c1adff..aa844ee9 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -48,7 +48,7 @@ def handle(name, cfg, cloud, log, _args): "ssh_fp_console_blacklist", []) key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", - ["ssh-dss"]) + ["ssh-dss"]) try: cmd = [helper_path] diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 7d8a0202..e2fdf68e 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -59,7 +59,7 @@ def handle(name, cfg, cloud, log, args): if init_cfg: if not isinstance(init_cfg, dict): log.warn("lxd/init config must be a dictionary. found a '%s'", - type(init_cfg)) + type(init_cfg)) return cmd = ['lxd', 'init', '--auto'] for k in init_keys: diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 11089d8d..4fe3ee21 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -204,12 +204,12 @@ def setup_swapfile(fname, size=None, maxsize=None): try: util.ensure_dir(tdir) util.log_time(LOG.debug, msg, func=util.subp, - args=[['sh', '-c', - ('rm -f "$1" && umask 0066 && ' - '{ fallocate -l "${2}M" "$1" || ' - ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' - 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), - 'setup_swap', fname, mbsize]]) + args=[['sh', '-c', + ('rm -f "$1" && umask 0066 && ' + '{ fallocate -l "${2}M" "$1" || ' + ' dd if=/dev/zero "of=$1" bs=1M "count=$2"; } && ' + 'mkswap "$1" || { r=$?; rm -f "$1"; exit $r; }'), + 'setup_swap', fname, mbsize]]) except Exception as e: raise IOError("Failed %s: %s" % (msg, e)) diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 7d9567e3..cc3f7f70 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -105,7 +105,7 @@ def handle(_name, cfg, _cloud, log, _args): log.debug("After pid %s ends, will execute: %s" % (mypid, ' '.join(args))) - util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, + util.fork_cb(run_after_pid_gone, mypid, cmdline, timeout, log, condition, execmd, [args, devnull_fp]) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 4501598e..774d3322 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -36,8 +36,8 @@ def _autostart_puppet(log): # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): util.subp(['sed', '-i', - '-e', 's/^START=.*/START=yes/', - '/etc/default/puppet'], capture=False) + '-e', 's/^START=.*/START=yes/', + '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, _args): " doing nothing.")) elif install: log.debug(("Attempting to install puppet %s,"), - version if version else 'latest') + version if version else 'latest') cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index cbc07853..2a2a9f59 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -166,7 +166,7 @@ def handle(name, cfg, _cloud, log, args): func=do_resize, args=(resize_cmd, log)) else: util.log_time(logfunc=log.debug, msg="Resizing", - func=do_resize, args=(resize_cmd, log)) + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 3b30c47e..6f474aed 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -127,8 +127,8 @@ class SubscriptionManager(object): return False, not_bool if (self.servicelevel is not None) and \ - ((not self.auto_attach) - or (util.is_false(str(self.auto_attach)))): + ((not self.auto_attach) or + (util.is_false(str(self.auto_attach)))): no_auto = ("The service-level key must be used in conjunction " "with the auto-attach key. Please re-run with " diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 5d7f4331..f43d8d5a 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in module %s"), name) + " not setting the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 5bd2dec6..d24e43c0 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -30,9 +30,10 @@ from cloudinit import distros as ds from cloudinit import ssh_util from cloudinit import util -DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " -"rather than the user \\\"root\\\".\';echo;sleep 10\"") +DISABLE_ROOT_OPTS = ( + "no-port-forwarding,no-agent-forwarding," + "no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\"" + " rather than the user \\\"root\\\".\';echo;sleep 10\"") GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index d3dd1f32..15703efe 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -41,10 +41,10 @@ def handle(name, cfg, cloud, log, _args): if not tpl_fn_name: raise RuntimeError(("No hosts template could be" " found for distro %s") % - (cloud.distro.osfamily)) + (cloud.distro.osfamily)) templater.render_to_file(tpl_fn_name, '/etc/hosts', - {'hostname': hostname, 'fqdn': fqdn}) + {'hostname': hostname, 'fqdn': fqdn}) elif manage_hosts == "localhost": (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) @@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in module %s"), name) + " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index e396ba13..5b78afe1 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -29,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in module %s"), name) + " not updating the hostname in module %s"), name) return (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 3b821af9..64fba869 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -92,7 +92,7 @@ def handle(name, cfg, _cloud, log, _args): for req_field in ['baseurl']: if req_field not in repo_config: log.warn(("Repository %s does not contain a %s" - " configuration 'required' entry"), + " configuration 'required' entry"), repo_id, req_field) missing_required += 1 if not missing_required: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 71884b32..661a9fd2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -211,8 +211,8 @@ class Distro(object): # If the system hostname is different than the previous # one or the desired one lets update it as well - if (not sys_hostname) or (sys_hostname == prev_hostname - and sys_hostname != hostname): + if ((not sys_hostname) or (sys_hostname == prev_hostname and + sys_hostname != hostname)): update_files.append(sys_fn) # If something else has changed the hostname after we set it @@ -221,7 +221,7 @@ class Distro(object): if (sys_hostname and prev_hostname and sys_hostname != prev_hostname): LOG.info("%s differs from %s, assuming user maintained hostname.", - prev_hostname_fn, sys_fn) + prev_hostname_fn, sys_fn) return # Remove duplicates (incase the previous config filename) @@ -289,7 +289,7 @@ class Distro(object): def _bring_up_interface(self, device_name): cmd = ['ifup', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -548,7 +548,7 @@ class Distro(object): for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist.", member, name) + "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) @@ -886,7 +886,7 @@ def fetch(name): locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: raise ImportError("No distribution found for distro %s (searched %s)" - % (name, looked_locs)) + % (name, looked_locs)) mod = importer.import_module(locs[0]) cls = getattr(mod, 'Distro') return cls diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 45fcf26f..93a2e008 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -74,7 +74,7 @@ class Distro(distros.Distro): 'Interface': dev, 'IP': info.get('bootproto'), 'Address': "('%s/%s')" % (info.get('address'), - info.get('netmask')), + info.get('netmask')), 'Gateway': info.get('gateway'), 'DNS': str(tuple(info.get('dns-nameservers'))).replace(',', '') } @@ -86,7 +86,7 @@ class Distro(distros.Distro): if nameservers: util.write_file(self.resolve_conf_fn, - convert_resolv_conf(nameservers)) + convert_resolv_conf(nameservers)) return dev_names @@ -102,7 +102,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['netctl', 'restart', device_name] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 6d3a82bf..db5890b1 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -159,8 +159,9 @@ class Distro(distros.Distro): # Allow the output of this to flow outwards (ie not be captured) util.log_time(logfunc=LOG.debug, - msg="apt-%s [%s]" % (command, ' '.join(cmd)), func=util.subp, - args=(cmd,), kwargs={'env': e, 'capture': False}) + msg="apt-%s [%s]" % (command, ' '.join(cmd)), + func=util.subp, + args=(cmd,), kwargs={'env': e, 'capture': False}) def update_package_sources(self): self._runner.run("update-sources", self.package_command, diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 4c484639..72012056 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -205,8 +205,8 @@ class Distro(distros.Distro): redact_opts = ['passwd'] for key, val in kwargs.items(): - if (key in adduser_opts and val - and isinstance(val, six.string_types)): + if (key in adduser_opts and val and + isinstance(val, six.string_types)): adduser_cmd.extend([adduser_opts[key], val]) # Redact certain fields from the logs diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 9e80583c..6267dd6e 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -66,7 +66,7 @@ class Distro(distros.Distro): def _bring_up_interface(self, device_name): cmd = ['/etc/init.d/net.%s' % device_name, 'restart'] LOG.debug("Attempting to run bring up interface %s using command %s", - device_name, cmd) + device_name, cmd) try: (_out, err) = util.subp(cmd) if len(err): @@ -88,7 +88,7 @@ class Distro(distros.Distro): (_out, err) = util.subp(cmd) if len(err): LOG.warn("Running %s resulted in stderr output: %s", cmd, - err) + err) except util.ProcessExecutionError: util.logexc(LOG, "Running interface command %s failed", cmd) return False diff --git a/cloudinit/distros/parsers/hostname.py b/cloudinit/distros/parsers/hostname.py index 84a1de42..efb185d4 100644 --- a/cloudinit/distros/parsers/hostname.py +++ b/cloudinit/distros/parsers/hostname.py @@ -84,5 +84,5 @@ class HostnameConf(object): hostnames_found.add(head) if len(hostnames_found) > 1: raise IOError("Multiple hostnames (%s) found!" - % (hostnames_found)) + % (hostnames_found)) return entries diff --git a/cloudinit/distros/parsers/resolv_conf.py b/cloudinit/distros/parsers/resolv_conf.py index 8aee03a4..2ed13d9c 100644 --- a/cloudinit/distros/parsers/resolv_conf.py +++ b/cloudinit/distros/parsers/resolv_conf.py @@ -132,7 +132,7 @@ class ResolvConf(object): # Some hard limit on 256 chars total raise ValueError(("Adding %r would go beyond the " "256 maximum search list character limit") - % (search_domain)) + % (search_domain)) self._remove_option('search') self._contents.append(('option', ['search', s_list, ''])) return flat_sds diff --git a/cloudinit/distros/parsers/sys_conf.py b/cloudinit/distros/parsers/sys_conf.py index d795e12f..6157cf32 100644 --- a/cloudinit/distros/parsers/sys_conf.py +++ b/cloudinit/distros/parsers/sys_conf.py @@ -77,8 +77,7 @@ class SysConf(configobj.ConfigObj): quot_func = None if value[0] in ['"', "'"] and value[-1] in ['"', "'"]: if len(value) == 1: - quot_func = (lambda x: - self._get_single_quote(x) % x) + quot_func = (lambda x: self._get_single_quote(x) % x) else: # Quote whitespace if it isn't the start + end of a shell command if value.strip().startswith("$(") and value.strip().endswith(")"): @@ -91,10 +90,10 @@ class SysConf(configobj.ConfigObj): # to use single quotes which won't get expanded... if re.search(r"[\n\"']", value): quot_func = (lambda x: - self._get_triple_quote(x) % x) + self._get_triple_quote(x) % x) else: quot_func = (lambda x: - self._get_single_quote(x) % x) + self._get_single_quote(x) % x) else: quot_func = pipes.quote if not quot_func: diff --git a/cloudinit/filters/launch_index.py b/cloudinit/filters/launch_index.py index 5bebd318..baecdac9 100644 --- a/cloudinit/filters/launch_index.py +++ b/cloudinit/filters/launch_index.py @@ -61,7 +61,7 @@ class Filter(object): discarded += 1 LOG.debug(("Discarding %s multipart messages " "which do not match launch index %s"), - discarded, self.wanted_idx) + discarded, self.wanted_idx) new_message = copy.copy(message) new_message.set_payload(new_msgs) new_message[ud.ATTACHMENT_FIELD] = str(len(new_msgs)) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 5e99d185..a6eb20fe 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -139,9 +139,10 @@ class FileSemaphores(object): # but the item had run before we did canon_sem_name. if cname != name and os.path.exists(self._get_path(name, freq)): LOG.warn("%s has run without canonicalized name [%s].\n" - "likely the migrator has not yet run. It will run next boot.\n" - "run manually with: cloud-init single --name=migrator" - % (name, cname)) + "likely the migrator has not yet run. " + "It will run next boot.\n" + "run manually with: cloud-init single --name=migrator" + % (name, cname)) return True return False diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index bd80a8a6..b03ab895 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -38,7 +38,8 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BOUNCE_COMMAND = ['sh', '-xc', +BOUNCE_COMMAND = [ + 'sh', '-xc', "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] BUILTIN_DS_CONFIG = { @@ -91,9 +92,9 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): """ policy = cfg['hostname_bounce']['policy'] previous_hostname = get_hostname(hostname_command) - if (not util.is_true(cfg.get('set_hostname')) - or util.is_false(policy) - or (previous_hostname == temp_hostname and policy != 'force')): + if (not util.is_true(cfg.get('set_hostname')) or + util.is_false(policy) or + (previous_hostname == temp_hostname and policy != 'force')): yield None return set_hostname(temp_hostname, hostname_command) @@ -123,8 +124,8 @@ class DataSourceAzureNet(sources.DataSource): with temporary_hostname(temp_hostname, self.ds_cfg, hostname_command=hostname_command) \ as previous_hostname: - if (previous_hostname is not None - and util.is_true(self.ds_cfg.get('set_hostname'))): + if (previous_hostname is not None and + util.is_true(self.ds_cfg.get('set_hostname'))): cfg = self.ds_cfg['hostname_bounce'] try: perform_hostname_bounce(hostname=temp_hostname, @@ -152,7 +153,8 @@ class DataSourceAzureNet(sources.DataSource): else: bname = str(pk['fingerprint'] + ".crt") fp_files += [os.path.join(ddir, bname)] - LOG.debug("ssh authentication: using fingerprint from fabirc") + LOG.debug("ssh authentication: " + "using fingerprint from fabirc") missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", func=wait_for_files, @@ -506,7 +508,7 @@ def read_azure_ovf(contents): raise BrokenAzureDataSource("invalid xml: %s" % e) results = find_child(dom.documentElement, - lambda n: n.localName == "ProvisioningSection") + lambda n: n.localName == "ProvisioningSection") if len(results) == 0: raise NonAzureDataSource("No ProvisioningSection") @@ -516,7 +518,8 @@ def read_azure_ovf(contents): provSection = results[0] lpcs_nodes = find_child(provSection, - lambda n: n.localName == "LinuxProvisioningConfigurationSet") + lambda n: + n.localName == "LinuxProvisioningConfigurationSet") if len(results) == 0: raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index eb474079..e3916208 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -39,7 +39,7 @@ FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') OPTICAL_DEVICES = tuple(('/dev/%s%s' % (z, i) for z in POSSIBLE_MOUNTS - for i in range(0, 2))) + for i in range(0, 2))) class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0032d06c..6a897f7d 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -61,12 +61,12 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2.get_instance_userdata(self.api_ver, - self.metadata_address) + self.userdata_raw = \ + ec2.get_instance_userdata(self.api_ver, self.metadata_address) self.metadata = ec2.get_instance_metadata(self.api_ver, self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", - int(time.time() - start_time)) + int(time.time() - start_time)) return True except Exception: util.logexc(LOG, "Failed reading from metadata address %s", @@ -132,13 +132,13 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) else: LOG.critical("Giving up on md from %s after %s seconds", - urls, int(time.time() - start_time)) + urls, int(time.time() - start_time)) self.metadata_address = url2base.get(url) return bool(url) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index cfc59ca5..f18c4cee 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -275,17 +275,18 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='Interact with MAAS DS') parser.add_argument("--config", metavar="file", - help="specify DS config file", default=None) + help="specify DS config file", default=None) parser.add_argument("--ckey", metavar="key", - help="the consumer key to auth with", default=None) + help="the consumer key to auth with", default=None) parser.add_argument("--tkey", metavar="key", - help="the token key to auth with", default=None) + help="the token key to auth with", default=None) parser.add_argument("--csec", metavar="secret", - help="the consumer secret (likely '')", default="") + help="the consumer secret (likely '')", default="") parser.add_argument("--tsec", metavar="secret", - help="the token secret to auth with", default=None) + help="the token secret to auth with", default=None) parser.add_argument("--apiver", metavar="version", - help="the apiver to use ("" can be used)", default=MD_VERSION) + help="the apiver to use ("" can be used)", + default=MD_VERSION) subcmds = parser.add_subparsers(title="subcommands", dest="subcmd") subcmds.add_parser('crawl', help="crawl the datasource") @@ -297,7 +298,7 @@ if __name__ == "__main__": args = parser.parse_args() creds = {'consumer_key': args.ckey, 'token_key': args.tkey, - 'token_secret': args.tsec, 'consumer_secret': args.csec} + 'token_secret': args.tsec, 'consumer_secret': args.csec} if args.config: cfg = util.read_conf(args.config) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 58a4b2a2..adf9b12e 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -264,14 +264,14 @@ def get_properties(contents): # could also check here that elem.namespaceURI == # "http://schemas.dmtf.org/ovf/environment/1" propSections = find_child(dom.documentElement, - lambda n: n.localName == "PropertySection") + lambda n: n.localName == "PropertySection") if len(propSections) == 0: raise XmlError("No 'PropertySection's") props = {} propElems = find_child(propSections[0], - (lambda n: n.localName == "Property")) + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index ac2c3b45..b26940d1 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -404,7 +404,8 @@ def read_context_disk_dir(source_dir, asuser=None): if ssh_key_var: lines = context.get(ssh_key_var).splitlines() results['metadata']['public-keys'] = [l for l in lines - if len(l) and not l.startswith("#")] + if len(l) and not + l.startswith("#")] # custom hostname -- try hostname or leave cloud-init # itself create hostname from IP address later diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7453379a..139ee52c 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -90,8 +90,7 @@ BUILTIN_DS_CONFIG = { 'user-data', 'user-script', 'sdc:datacenter_name', - 'sdc:uuid', - ], + 'sdc:uuid'], 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -450,7 +449,7 @@ class JoyentMetadataClient(object): response = bytearray() response.extend(self.metasource.read(1)) - while response[-1:] != b'\n': + while response[-1:] != b'\n': response.extend(self.metasource.read(1)) response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) @@ -513,7 +512,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, except Exception as e: util.logexc(LOG, ("Failed to identify script type for %s" % - content_f, e)) + content_f, e)) if link: try: diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 9b2f5ed5..c74a7ae2 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -31,7 +31,8 @@ LOG = logging.getLogger(__name__) DEF_SSHD_CFG = "/etc/ssh/sshd_config" # taken from openssh source key.c/key_type_from_name -VALID_KEY_TYPES = ("rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", +VALID_KEY_TYPES = ( + "rsa", "dsa", "ssh-rsa", "ssh-dss", "ecdsa", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v00@openssh.com", "ssh-dss-cert-v00@openssh.com", "ssh-rsa-cert-v01@openssh.com", "ssh-dss-cert-v01@openssh.com", diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9f192c8d..dbcf3d55 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -509,13 +509,13 @@ class Init(object): def consume_data(self, frequency=PER_INSTANCE): # Consume the userdata first, because we need want to let the part # handlers run first (for merging stuff) - with events.ReportEventStack( - "consume-user-data", "reading and applying user-data", - parent=self.reporter): + with events.ReportEventStack("consume-user-data", + "reading and applying user-data", + parent=self.reporter): self._consume_userdata(frequency) - with events.ReportEventStack( - "consume-vendor-data", "reading and applying vendor-data", - parent=self.reporter): + with events.ReportEventStack("consume-vendor-data", + "reading and applying vendor-data", + parent=self.reporter): self._consume_vendordata(frequency) # Perform post-consumption adjustments so that @@ -655,7 +655,7 @@ class Modules(object): else: raise TypeError(("Failed to read '%s' item in config," " unknown type %s") % - (item, type_utils.obj_name(item))) + (item, type_utils.obj_name(item))) return module_list def _fixup_modules(self, raw_mods): @@ -762,8 +762,8 @@ class Modules(object): if skipped: LOG.info("Skipping modules %s because they are not verified " - "on distro '%s'. To run anyway, add them to " - "'unverified_modules' in config.", skipped, d_name) + "on distro '%s'. To run anyway, add them to " + "'unverified_modules' in config.", skipped, d_name) if forced: LOG.info("running unverified_modules: %s", forced) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index f2e1390e..936f7da5 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -252,9 +252,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - if (isinstance(e, (exceptions.HTTPError)) - and hasattr(e, 'response') # This appeared in v 0.10.8 - and hasattr(e.response, 'status_code')): + if (isinstance(e, (exceptions.HTTPError)) and + hasattr(e, 'response') and # This appeared in v 0.10.8 + hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers, url=url)) diff --git a/cloudinit/util.py b/cloudinit/util.py index 45d49e66..de37b0f5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -612,7 +612,7 @@ def redirect_output(outfmt, errfmt, o_out=None, o_err=None): def make_url(scheme, host, port=None, - path='', params='', query='', fragment=''): + path='', params='', query='', fragment=''): pieces = [] pieces.append(scheme or '') @@ -804,8 +804,8 @@ def load_yaml(blob, default=None, allowed=(dict,)): blob = decode_binary(blob) try: LOG.debug("Attempting to load yaml from string " - "of length %s with allowed root types %s", - len(blob), allowed) + "of length %s with allowed root types %s", + len(blob), allowed) converted = safeyaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... @@ -878,7 +878,7 @@ def read_conf_with_confd(cfgfile): if not isinstance(confd, six.string_types): raise TypeError(("Config file %s contains 'conf_d' " "with non-string type %s") % - (cfgfile, type_utils.obj_name(confd))) + (cfgfile, type_utils.obj_name(confd))) else: confd = str(confd).strip() elif os.path.isdir("%s.d" % cfgfile): @@ -1041,7 +1041,8 @@ def is_resolvable(name): for iname in badnames: try: result = socket.getaddrinfo(iname, None, 0, 0, - socket.SOCK_STREAM, socket.AI_CANONNAME) + socket.SOCK_STREAM, + socket.AI_CANONNAME) badresults[iname] = [] for (_fam, _stype, _proto, cname, sockaddr) in result: badresults[iname].append("%s: %s" % (cname, sockaddr[0])) @@ -1109,7 +1110,7 @@ def close_stdin(): def find_devs_with(criteria=None, oformat='device', - tag=None, no_cache=False, path=None): + tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) criteria can be *one* of: @@ -1628,7 +1629,7 @@ def write_file(filename, content, mode=0o644, omode="wb"): content = decode_binary(content) write_type = 'characters' LOG.debug("Writing to %s - %s: [%s] %s %s", - filename, omode, mode, len(content), write_type) + filename, omode, mode, len(content), write_type) with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index c603bfdb..9c1ec1d4 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -27,11 +27,12 @@ from cloudinit import stages from cloudinit import user_data as ud from cloudinit import util -INSTANCE_ID = "i-testing" - from . import helpers +INSTANCE_ID = "i-testing" + + class FakeDataSource(sources.DataSource): def __init__(self, userdata=None, vendordata=None): diff --git a/tests/unittests/test_datasource/test_altcloud.py b/tests/unittests/test_datasource/test_altcloud.py index e9cd2fa5..85759c68 100644 --- a/tests/unittests/test_datasource/test_altcloud.py +++ b/tests/unittests/test_datasource/test_altcloud.py @@ -134,8 +134,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('RHEV') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('RHEV', \ - dsrc.get_cloud_type()) + self.assertEquals('RHEV', dsrc.get_cloud_type()) def test_vsphere(self): ''' @@ -144,8 +143,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('VMware Virtual Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('VSPHERE', \ - dsrc.get_cloud_type()) + self.assertEquals('VSPHERE', dsrc.get_cloud_type()) def test_unknown(self): ''' @@ -154,8 +152,7 @@ class TestGetCloudType(TestCase): ''' util.read_dmi_data = _dmi_data('Unrecognized Platform') dsrc = DataSourceAltCloud({}, None, self.paths) - self.assertEquals('UNKNOWN', \ - dsrc.get_cloud_type()) + self.assertEquals('UNKNOWN', dsrc.get_cloud_type()) class TestGetDataCloudInfoFile(TestCase): @@ -412,27 +409,27 @@ class TestReadUserDataCallback(TestCase): '''Test read_user_data_callback() with both files.''' self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_dc(self): '''Test read_user_data_callback() with only DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=False, - non_dc_file=True) + dc_file=False, + non_dc_file=True) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_non_dc(self): '''Test read_user_data_callback() with only non-DC file.''' _remove_user_data_files(self.mount_dir, - dc_file=True, - non_dc_file=False) + dc_file=True, + non_dc_file=False) self.assertEquals('test user data', - read_user_data_callback(self.mount_dir)) + read_user_data_callback(self.mount_dir)) def test_callback_none(self): '''Test read_user_data_callback() no files are found.''' diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3933794f..4c9c7d8b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -207,7 +207,7 @@ class TestAzureDataSource(TestCase): yaml_cfg = "{agent_command: my_command}\n" cfg = yaml.safe_load(yaml_cfg) odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} + 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -219,8 +219,8 @@ class TestAzureDataSource(TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(cfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(cfg)), + 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) @@ -267,7 +267,8 @@ class TestAzureDataSource(TestCase): # should equal that after the '$' pos = defuser['passwd'].rfind("$") + 1 self.assertEqual(defuser['passwd'], - crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos])) + crypt.crypt(odata['UserPassword'], + defuser['passwd'][0:pos])) def test_userdata_plain(self): mydata = "FOOBAR" @@ -364,8 +365,8 @@ class TestAzureDataSource(TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': b64e(yaml.dump(dscfg)), - 'encoding': 'base64'}} + 'dscfg': {'text': b64e(yaml.dump(dscfg)), + 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} userdata = '#cloud-config' + yaml.dump(usercfg) + "\n" @@ -634,7 +635,7 @@ class TestReadAzureOvf(TestCase): def test_invalid_xml_raises_non_azure_ds(self): invalid_xml = "" + construct_valid_ovf_env(data={}) self.assertRaises(DataSourceAzure.BrokenAzureDataSource, - DataSourceAzure.read_azure_ovf, invalid_xml) + DataSourceAzure.read_azure_ovf, invalid_xml) def test_load_with_pubkeys(self): mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}] diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 83aca505..3954ceb3 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -293,9 +293,8 @@ class TestConfigDriveDataSource(TestCase): util.is_partition = my_is_partition devs_with_answers = {"TYPE=vfat": [], - "TYPE=iso9660": ["/dev/vdb"], - "LABEL=config-2": ["/dev/vdb"], - } + "TYPE=iso9660": ["/dev/vdb"], + "LABEL=config-2": ["/dev/vdb"]} self.assertEqual(["/dev/vdb"], ds.find_candidate_devs()) # add a vfat item @@ -306,9 +305,10 @@ class TestConfigDriveDataSource(TestCase): # verify that partitions are considered, that have correct label. devs_with_answers = {"TYPE=vfat": ["/dev/sda1"], - "TYPE=iso9660": [], "LABEL=config-2": ["/dev/vdb3"]} + "TYPE=iso9660": [], + "LABEL=config-2": ["/dev/vdb3"]} self.assertEqual(["/dev/vdb3"], - ds.find_candidate_devs()) + ds.find_candidate_devs()) finally: util.find_devs_with = orig_find_devs_with @@ -319,7 +319,7 @@ class TestConfigDriveDataSource(TestCase): populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) self.assertEqual(myds.get_public_ssh_keys(), - [OSTACK_META['public_keys']['mykey']]) + [OSTACK_META['public_keys']['mykey']]) def cfg_ds_from_dir(seed_d): diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index eb97b692..77d15cac 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -25,9 +25,9 @@ class TestMAASDataSource(TestCase): """Verify a valid seeddir is read as such.""" data = {'instance-id': 'i-valid01', - 'local-hostname': 'valid01-hostname', - 'user-data': b'valid01-userdata', - 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} + 'local-hostname': 'valid01-hostname', + 'user-data': b'valid01-userdata', + 'public-keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname'} my_d = os.path.join(self.tmp, "valid") populate_dir(my_d, data) @@ -45,8 +45,8 @@ class TestMAASDataSource(TestCase): """Verify extra files do not affect seed_dir validity.""" data = {'instance-id': 'i-valid-extra', - 'local-hostname': 'valid-extra-hostname', - 'user-data': b'valid-extra-userdata', 'foo': 'bar'} + 'local-hostname': 'valid-extra-hostname', + 'user-data': b'valid-extra-userdata', 'foo': 'bar'} my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) @@ -64,7 +64,7 @@ class TestMAASDataSource(TestCase): """Verify that invalid seed_dir raises MAASSeedDirMalformed.""" valid = {'instance-id': 'i-instanceid', - 'local-hostname': 'test-hostname', 'user-data': ''} + 'local-hostname': 'test-hostname', 'user-data': ''} my_based = os.path.join(self.tmp, "valid_extra") @@ -94,8 +94,8 @@ class TestMAASDataSource(TestCase): def test_seed_dir_missing(self): """Verify that missing seed_dir raises MAASSeedDirNone.""" self.assertRaises(DataSourceMAAS.MAASSeedDirNone, - DataSourceMAAS.read_maas_seed_dir, - os.path.join(self.tmp, "nonexistantdirectory")) + DataSourceMAAS.read_maas_seed_dir, + os.path.join(self.tmp, "nonexistantdirectory")) def test_seed_url_valid(self): """Verify that valid seed_url is read as such.""" diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 1235436d..5e617b83 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -463,8 +463,8 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): payloadstr = ' {0}'.format(self.response_parts['payload']) return ('V2 {length} {crc} {request_id} ' '{command}{payloadstr}\n'.format( - payloadstr=payloadstr, - **self.response_parts).encode('ascii')) + payloadstr=payloadstr, + **self.response_parts).encode('ascii')) self.metasource_data = None @@ -501,7 +501,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): written_line = self.serial.write.call_args[0][0] print(type(written_line)) self.assertEndsWith(written_line.decode('ascii'), - b'\n'.decode('ascii')) + b'\n'.decode('ascii')) self.assertEqual(1, written_line.count(b'\n')) def _get_written_line(self, key='some_key'): diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 5687b10d..f9660ff6 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -74,7 +74,7 @@ class TestLoadPowerState(t_help.TestCase): class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): return([sys.executable, '-c', 'import sys; sys.exit(%s)' % rc]) - + def test_true_is_true(self): self.assertEqual(psc.check_condition(True), True) @@ -94,7 +94,6 @@ class TestCheckCondition(t_help.TestCase): self.assertEqual(mocklog.warn.call_count, 1) - def check_lps_ret(psc_return, mode=None): if len(psc_return) != 3: raise TypeError("length returned = %d" % len(psc_return)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 0bcdcb31..34d11f21 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -190,7 +190,8 @@ class TestRandomSeed(t_help.TestCase): c = self._get_cloud('ubuntu', {}) self.whichdata = {} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, c, LOG, []) + 'test', {'random_seed': {'command_required': True}}, + c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index eceb14d9..8aeff53c 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -125,8 +125,7 @@ class TestInstallPackages(t_help.TestCase): "pkg1.smoser.config": "pkg1.smoser.config-data", "pkg1.config": "pkg1.config-data", "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", - "pkg2.smoser_0.0_amd64.config": "pkg2.config", - }) + "pkg2.smoser_0.0_amd64.config": "pkg2.config"}) ret = get_package_ops( packages=[], configs={}, installed=[], fspath=self.tmp) diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py index 3b317121..9aeb1cde 100644 --- a/tests/unittests/test_sshutil.py +++ b/tests/unittests/test_sshutil.py @@ -32,7 +32,8 @@ VALID_CONTENT = { ), } -TEST_OPTIONS = ("no-port-forwarding,no-agent-forwarding,no-X11-forwarding," +TEST_OPTIONS = ( + "no-port-forwarding,no-agent-forwarding,no-X11-forwarding," 'command="echo \'Please login as the user \"ubuntu\" rather than the' 'user \"root\".\';echo;sleep 10"') diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 0c19a2c2..b9863650 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -114,5 +114,6 @@ $a,$b''' codename) out_data = templater.basic_render(in_data, - {'mirror': mirror, 'codename': codename}) + {'mirror': mirror, + 'codename': codename}) self.assertEqual(ex_data, out_data) diff --git a/tools/hacking.py b/tools/hacking.py index 3175df38..1a0631c2 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -47,10 +47,10 @@ def import_normalize(line): # handle "from x import y as z" to "import x.y as z" split_line = line.split() if (line.startswith("from ") and "," not in line and - split_line[2] == "import" and split_line[3] != "*" and - split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): + split_line[2] == "import" and split_line[3] != "*" and + split_line[1] != "__future__" and + (len(split_line) == 4 or + (len(split_line) == 6 and split_line[4] == "as"))): return "import %s.%s" % (split_line[1], split_line[3]) else: return line @@ -74,7 +74,7 @@ def cloud_import_alphabetical(physical_line, line_number, lines): split_line[0] == "import" and split_previous[0] == "import"): if split_line[1] < split_previous[1]: return (0, "N306: imports not in alphabetical order (%s, %s)" - % (split_previous[1], split_line[1])) + % (split_previous[1], split_line[1])) def cloud_docstring_start_space(physical_line): @@ -87,8 +87,8 @@ def cloud_docstring_start_space(physical_line): pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start if (pos != -1 and len(physical_line) > pos + 1): if (physical_line[pos + 3] == ' '): - return (pos, "N401: one line docstring should not start with" - " a space") + return (pos, + "N401: one line docstring should not start with a space") def cloud_todo_format(physical_line): @@ -167,4 +167,4 @@ if __name__ == "__main__": finally: if len(_missingImport) > 0: print >> sys.stderr, ("%i imports missing in this test environment" - % len(_missingImport)) + % len(_missingImport)) diff --git a/tools/mock-meta.py b/tools/mock-meta.py index dfbc2a71..1c746f17 100755 --- a/tools/mock-meta.py +++ b/tools/mock-meta.py @@ -126,11 +126,11 @@ class WebException(Exception): def yamlify(data): formatted = yaml.dump(data, - line_break="\n", - indent=4, - explicit_start=True, - explicit_end=True, - default_flow_style=False) + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) return formatted @@ -282,7 +282,7 @@ class MetaDataHandler(object): else: log.warn(("Did not implement action %s, " "returning empty response: %r"), - action, NOT_IMPL_RESPONSE) + action, NOT_IMPL_RESPONSE) return NOT_IMPL_RESPONSE @@ -404,14 +404,17 @@ def setup_logging(log_level, fmt='%(levelname)s: @%(name)s : %(message)s'): def extract_opts(): parser = OptionParser() parser.add_option("-p", "--port", dest="port", action="store", type=int, - default=80, metavar="PORT", - help="port from which to serve traffic (default: %default)") + default=80, metavar="PORT", + help=("port from which to serve traffic" + " (default: %default)")) parser.add_option("-a", "--addr", dest="address", action="store", type=str, - default='0.0.0.0', metavar="ADDRESS", - help="address from which to serve traffic (default: %default)") + default='0.0.0.0', metavar="ADDRESS", + help=("address from which to serve traffic" + " (default: %default)")) parser.add_option("-f", '--user-data-file', dest='user_data_file', - action='store', metavar='FILE', - help="user data filename to serve back to incoming requests") + action='store', metavar='FILE', + help=("user data filename to serve back to" + "incoming requests")) (options, args) = parser.parse_args() out = dict() out['extra'] = args diff --git a/tools/run-pep8 b/tools/run-pep8 index ccd6be5a..086400fc 100755 --- a/tools/run-pep8 +++ b/tools/run-pep8 @@ -1,39 +1,22 @@ #!/bin/bash -if [ $# -eq 0 ]; then - files=( bin/cloud-init $(find * -name "*.py" -type f) ) +pycheck_dirs=( "cloudinit/" "bin/" "tests/" "tools/" ) +# FIXME: cloud-init modifies sys module path, pep8 does not like +# bin_files=( "bin/cloud-init" ) +CR=" +" +[ "$1" = "-v" ] && { verbose="$1"; shift; } || verbose="" + +set -f +if [ $# -eq 0 ]; then unset IFS + IFS="$CR" + files=( "${bin_files[@]}" "${pycheck_dirs[@]}" ) + unset IFS else - files=( "$@" ); + files=( "$@" ) fi -if [ -f 'hacking.py' ] -then - base=`pwd` -else - base=`pwd`/tools/ -fi - -IGNORE="" - -# King Arthur: Be quiet! ... Be Quiet! I Order You to Be Quiet. -IGNORE="$IGNORE,E121" # Continuation line indentation is not a multiple of four -IGNORE="$IGNORE,E123" # Closing bracket does not match indentation of opening bracket's line -IGNORE="$IGNORE,E124" # Closing bracket missing visual indentation -IGNORE="$IGNORE,E125" # Continuation line does not distinguish itself from next logical line -IGNORE="$IGNORE,E126" # Continuation line over-indented for hanging indent -IGNORE="$IGNORE,E127" # Continuation line over-indented for visual indent -IGNORE="$IGNORE,E128" # Continuation line under-indented for visual indent -IGNORE="$IGNORE,E502" # The backslash is redundant between brackets -IGNORE="${IGNORE#,}" # remove the leading ',' added above - -cmd=( - ${base}/hacking.py - - --ignore="$IGNORE" - - "${files[@]}" -) - -echo -e "\nRunning 'cloudinit' pep8:" -echo "${cmd[@]}" -"${cmd[@]}" +myname=${0##*/} +cmd=( "${myname#run-}" $verbose "${files[@]}" ) +echo "Running: " "${cmd[@]}" 1>&2 +exec "${cmd[@]}" -- cgit v1.2.3 From 70acc910c3368980d7cb8971391a2c9dfaf3fda8 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Fri, 4 Mar 2016 09:51:05 -0600 Subject: pep8: update formatting to pass pep8 1.4.6 (trusty) and 1.6.2 (xenial) make check fails in a trusty sbuild due to different rules on older pep8. Fix formatting to pass in older and newer pep8. --- cloudinit/config/cc_rh_subscription.py | 4 +--- tests/unittests/test_datasource/test_azure.py | 2 +- tools/hacking.py | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 6f474aed..6087c45c 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -126,10 +126,8 @@ class SubscriptionManager(object): "(True/False " return False, not_bool - if (self.servicelevel is not None) and \ - ((not self.auto_attach) or + if (self.servicelevel is not None) and ((not self.auto_attach) or (util.is_false(str(self.auto_attach)))): - no_auto = ("The service-level key must be used in conjunction " "with the auto-attach key. Please re-run with " "auto-attach: True") diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 4c9c7d8b..444e2799 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -268,7 +268,7 @@ class TestAzureDataSource(TestCase): pos = defuser['passwd'].rfind("$") + 1 self.assertEqual(defuser['passwd'], crypt.crypt(odata['UserPassword'], - defuser['passwd'][0:pos])) + defuser['passwd'][0:pos])) def test_userdata_plain(self): mydata = "FOOBAR" diff --git a/tools/hacking.py b/tools/hacking.py index 1a0631c2..716c1154 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -49,8 +49,8 @@ def import_normalize(line): if (line.startswith("from ") and "," not in line and split_line[2] == "import" and split_line[3] != "*" and split_line[1] != "__future__" and - (len(split_line) == 4 or - (len(split_line) == 6 and split_line[4] == "as"))): + (len(split_line) == 4 or (len(split_line) == 6 and + split_line[4] == "as"))): return "import %s.%s" % (split_line[1], split_line[3]) else: return line -- cgit v1.2.3 From 6e31038b9cccbcb4a33693060b96fc4f71d86789 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 7 Mar 2016 21:31:25 -0500 Subject: No longer run pollinate by default in seed_random The user can still choose to run pollinate here to seed their random data. And in an environment with network datasource, that would be expected to work. However, we do not want to run it any more from cloud-init because a.) pollinate's own init system jobs should get it ran before ssh, which is the primary purpose of wanting cloud-init to run it. b.) with a local datasource, there is no network guarantee when init_modules run, so pollinate -q would often cause issues then. c.) cloud-init would run pollinate and log the failure causing many cloud-init specific failures that it could do nothing about. LP: #1554152 --- ChangeLog | 1 + cloudinit/config/cc_seed_random.py | 2 +- tests/unittests/test_handler/test_handler_seed_random.py | 14 ++++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index a80a5d5f..6da276b5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -85,6 +85,7 @@ unless it is already a file (LP: #1543025). - Enable password changing via a hashed string [Alex Sirbu] - Added BigStep datasource [Alex Sirbu] + - No longer run pollinate in seed_random (LP: #1554152) 0.7.6: - open 0.7.6 diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 3288a853..1b011216 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -83,7 +83,7 @@ def handle(name, cfg, cloud, log, _args): len(seed_data), seed_path) util.append_file(seed_path, seed_data) - command = mycfg.get('command', ['pollinate', '-q']) + command = mycfg.get('command', None) req = mycfg.get('command_required', False) try: env = os.environ.copy() diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 34d11f21..98bc9b81 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -170,28 +170,30 @@ class TestRandomSeed(t_help.TestCase): contents = util.load_file(self._seed_file) self.assertEquals('tiny-tim-was-here-so-was-josh', contents) - def test_seed_command_not_provided_pollinate_available(self): + def test_seed_command_provided_and_available(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {'pollinate': '/usr/bin/pollinate'} - cc_seed_random.handle('test', {}, c, LOG, []) + cfg = {'random_seed': {'command': ['pollinate', '-q']}} + cc_seed_random.handle('test', cfg, c, LOG, []) subp_args = [f['args'] for f in self.subp_called] self.assertIn(['pollinate', '-q'], subp_args) - def test_seed_command_not_provided_pollinate_not_available(self): + def test_seed_command_not_provided(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {} cc_seed_random.handle('test', {}, c, LOG, []) # subp should not have been called as which would say not available - self.assertEquals(self.subp_called, list()) + self.assertFalse(self.subp_called) def test_unavailable_seed_command_and_required_raises_error(self): c = self._get_cloud('ubuntu', {}) self.whichdata = {} + cfg = {'random_seed': {'command': ['THIS_NO_COMMAND'], + 'command_required': True}} self.assertRaises(ValueError, cc_seed_random.handle, - 'test', {'random_seed': {'command_required': True}}, - c, LOG, []) + 'test', cfg, c, LOG, []) def test_seed_command_and_required(self): c = self._get_cloud('ubuntu', {}) -- cgit v1.2.3 From c3ece3129228ad7f2206d049af0f4635da8e8eb5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 14 Mar 2016 14:24:27 -0400 Subject: fix long line --- cloudinit/config/cc_set_passwords.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index ff3b9ba5..58e1b713 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -118,8 +118,8 @@ def handle(_name, cfg, cloud, log, args): log.debug('Leaving auth line unchanged') change_pwauth = False else: - util.logexc(log, 'Unrecognized value %r for ssh_pwauth' % cfg['ssh_pwauth']) - + msg = 'Unrecognized value %s for ssh_pwauth' % cfg['ssh_pwauth'] + util.logexc(log, msg) if change_pwauth: replaced_auth = False -- cgit v1.2.3