From 26ea813d293467921ab6b1e32abd2ab8fcefa3bd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 11 May 2016 14:18:02 -0700 Subject: Fix py26 for rhel (and older versions of python) --- cloudinit/util.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..7562b97a 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,6 +37,7 @@ import pwd import random import re import shutil +import shlex import socket import stat import string @@ -81,6 +82,7 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None +PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -171,7 +173,8 @@ class ProcessExecutionError(IOError): def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None, reason=None): + description=None, reason=None, + errno=None): if not cmd: self.cmd = '-' else: @@ -202,6 +205,7 @@ class ProcessExecutionError(IOError): else: self.reason = '-' + self.errno = errno message = self.MESSAGE_TMPL % { 'description': self.description, 'cmd': self.cmd, @@ -1147,7 +1151,14 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options # See man blkid for why 2 is added - (out, _err) = subp(cmd, rcs=[0, 2]) + try: + (out, _err) = subp(cmd, rcs=[0, 2]) + except ProcessExecutionError as e: + if e.errno == errno.ENOENT: + # blkid not found... + out = "" + else: + raise entries = [] for line in out.splitlines(): line = line.strip() @@ -1191,6 +1202,13 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents +def shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + + def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] @@ -1696,7 +1714,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, sp = subprocess.Popen(args, **kws) (out, err) = sp.communicate(data) except OSError as e: - raise ProcessExecutionError(cmd=args, reason=e) + raise ProcessExecutionError(cmd=args, reason=e, + errno=e.errno) rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, -- cgit v1.2.3 From f11827aee59cf2678c3d2a157218a3fe2831f5c2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 10:56:26 -0700 Subject: Fix up a ton of flake8 issues --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/config/cc_bootcmd.py | 4 ++-- cloudinit/config/cc_disk_setup.py | 2 +- cloudinit/config/cc_emit_upstart.py | 2 +- cloudinit/config/cc_fan.py | 2 +- cloudinit/config/cc_final_message.py | 2 +- cloudinit/config/cc_grub_dpkg.py | 2 +- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 10 ++++----- cloudinit/config/cc_phone_home.py | 6 +++--- cloudinit/config/cc_rightscale_userdata.py | 4 ++-- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_scripts_per_boot.py | 2 +- cloudinit/config/cc_scripts_per_instance.py | 2 +- cloudinit/config/cc_scripts_per_once.py | 2 +- cloudinit/config/cc_scripts_user.py | 2 +- cloudinit/config/cc_scripts_vendor.py | 2 +- cloudinit/config/cc_seed_random.py | 2 +- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/config/cc_snappy.py | 4 ++-- cloudinit/config/cc_ssh.py | 6 +++--- cloudinit/config/cc_ssh_import_id.py | 4 ++-- cloudinit/config/cc_ubuntu_init_switch.py | 4 ++-- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/debian.py | 4 ++-- cloudinit/handlers/__init__.py | 6 +++--- cloudinit/handlers/cloud_config.py | 2 +- cloudinit/handlers/upstart_job.py | 2 +- cloudinit/helpers.py | 10 ++++----- cloudinit/net/__init__.py | 18 +++++++--------- cloudinit/net/network_state.py | 7 +++--- cloudinit/net/udev.py | 2 +- cloudinit/netinfo.py | 7 +++--- cloudinit/reporting/handlers.py | 10 ++++----- cloudinit/sources/DataSourceAzure.py | 12 ++++++----- cloudinit/sources/DataSourceBigstep.py | 4 ++-- cloudinit/sources/DataSourceCloudSigma.py | 5 +++-- cloudinit/sources/DataSourceCloudStack.py | 5 +++-- cloudinit/sources/DataSourceDigitalOcean.py | 4 ++-- cloudinit/sources/DataSourceGCE.py | 4 ++-- cloudinit/sources/DataSourceNoCloud.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 25 ++++++++++++++-------- cloudinit/sources/__init__.py | 3 +-- cloudinit/sources/helpers/azure.py | 1 + cloudinit/sources/helpers/openstack.py | 4 ++-- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 2 +- cloudinit/sources/helpers/vmware/imc/config.py | 2 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 2 +- .../sources/helpers/vmware/imc/config_source.py | 2 +- .../sources/helpers/vmware/imc/guestcust_error.py | 2 +- .../sources/helpers/vmware/imc/guestcust_event.py | 2 +- .../sources/helpers/vmware/imc/guestcust_state.py | 2 +- .../sources/helpers/vmware/imc/guestcust_util.py | 2 +- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 2 +- cloudinit/sources/helpers/vmware/imc/nic_base.py | 6 +++--- cloudinit/stages.py | 4 ++-- cloudinit/url_helper.py | 5 +++-- cloudinit/user_data.py | 2 +- cloudinit/util.py | 18 ++++++++-------- test-requirements.txt | 3 ++- tox.ini | 5 +++++ 62 files changed, 150 insertions(+), 132 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..e3fadc12 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -207,20 +207,20 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ks = ent['keyserver'] try: ent['key'] = getkeybyid(ent['keyid'], ks) - except: + except Exception: errorlist.append([source, "failed to get key from %s" % ks]) continue if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) - except: + except Exception: errorlist.append([source, "failed add key"]) try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") - except: + except Exception: errorlist.append([source, "failed write to file %s" % ent['filename']]) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index a295cc4e..b763a3c3 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -38,7 +38,7 @@ def handle(name, cfg, cloud, log, _args): content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except: + except Exception: util.logexc(log, "Failed to shellify bootcmd") raise @@ -49,6 +49,6 @@ def handle(name, cfg, cloud, log, _args): env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) - except: + except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index bbaf9646..b642f1f8 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -198,7 +198,7 @@ def is_device_valid(name, partition=False): d_type = "" try: d_type = device_type(name) - except: + except Exception: LOG.warn("Query against device %s failed" % name) return False diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 86ae97ab..06c53272 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -20,8 +20,8 @@ import os -from cloudinit.settings import PER_ALWAYS from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS from cloudinit import util frequency = PER_ALWAYS diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 39e3850e..545fee22 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -37,8 +37,8 @@ If cloud-init sees a 'fan' entry in cloud-config it will """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 4a51476f..c9021eb1 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -66,7 +66,7 @@ def handle(_name, cfg, cloud, log, args): try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) util.write_file(boot_fin_fn, contents) - except: + except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) if cloud.datasource.is_disconnected: diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 3c2d9985..156722d9 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -69,5 +69,5 @@ def handle(name, cfg, _cloud, log, _args): try: util.subp(['debconf-set-selections'], dconf_sel) - except: + except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index aa844ee9..9a02f056 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -57,6 +57,6 @@ def handle(name, cfg, cloud, log, _args): (stdout, _stderr) = util.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) - except: + except Exception: log.warn("Writing keys to the system console failed!") raise diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index bf735648..b1de8f84 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -111,7 +111,7 @@ def handle(name, cfg, cloud, log, args): data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" util.subp(['debconf-communicate'], data) - except: + except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) # Remove the existing configuration file (forces re-generation) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 4fe3ee21..2b981935 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -244,7 +244,7 @@ def handle_swapcfg(swapcfg): LOG.debug("swap file %s already in use.", fname) return fname LOG.debug("swap file %s existed, but not in /proc/swaps", fname) - except: + except Exception: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) return fname @@ -379,7 +379,7 @@ def handle(_name, cfg, cloud, log, _args): toks = WS.split(line) if toks[3].find(comment) != -1: continue - except: + except Exception: pass fstab_lines.append(line) @@ -390,16 +390,16 @@ def handle(_name, cfg, cloud, log, _args): if needswap: try: util.subp(("swapon", "-a")) - except: + except Exception: util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: try: util.ensure_dir(d) - except: + except Exception: util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) - except: + except Exception: util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 3dcc9459..72176d42 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, args): tries = ph_cfg.get('tries') try: tries = int(tries) - except: + except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) @@ -87,7 +87,7 @@ def handle(name, cfg, cloud, log, args): for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) - except: + except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) @@ -117,6 +117,6 @@ def handle(name, cfg, cloud, log, args): util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) - except: + except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 0ecf3a4d..8118fac4 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -52,7 +52,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() - except: + except Exception: log.debug("Failed to get raw userdata in module %s", name) return @@ -63,7 +63,7 @@ def handle(name, _cfg, cloud, log, _args): "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) return - except: + except Exception: util.logexc(log, "Failed to parse query string %s into a dictionary", ud) raise diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 66dc3363..bc09d38c 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -34,5 +34,5 @@ def handle(name, cfg, cloud, log, _args): try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) - except: + except Exception: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 42b987eb..ee3b6c9f 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index b5d71c13..c0d62b12 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index d77d36d5..ecb527f6 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 5c53014f..699857d1 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -36,7 +36,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index 0c9e504e..80bf10ff 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -37,7 +37,7 @@ def handle(name, cfg, cloud, log, _args): try: util.runparts(runparts_path, exe_prefix=prefix) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 1b011216..5085c23a 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -24,8 +24,8 @@ import os from six import BytesIO -from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 58e1b713..5c8c23b8 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -155,7 +155,7 @@ def handle(_name, cfg, cloud, log, args): cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") - except: + except Exception: util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index fa9d54a0..1a485ee6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,12 +47,12 @@ Example config: """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util import glob -import tempfile import os +import tempfile LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index d24e43c0..cb9b70aa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -57,7 +57,7 @@ def handle(_name, cfg, cloud, log, _args): for f in glob.glob(key_pth): try: util.del_file(f) - except: + except Exception: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) - except: + except Exception: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: @@ -122,7 +122,7 @@ def handle(_name, cfg, cloud, log, _args): keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) - except: + except Exception: util.logexc(log, "Applying ssh credentials failed!") diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 2d480d7e..28c4585b 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -52,14 +52,14 @@ def handle(_name, cfg, cloud, log, args): else: try: import_ids = user_cfg['ssh_import_id'] - except: + except Exception: log.debug("User %s is not configured for ssh_import_id", user) continue try: import_ids = util.uniq_merge(import_ids) import_ids = [str(i) for i in import_ids] - except: + except Exception: log.debug("User %s is not correctly configured for ssh_import_id", user) continue diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 7e88ed85..884d79f1 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -40,10 +40,10 @@ It can be configured with the following option structure:: mechanism you've used to switch the init system. """ -from cloudinit.settings import PER_INSTANCE +from cloudinit.distros import ubuntu from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util -from cloudinit.distros import ubuntu import os import time diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5879dabf..0f222c8c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,8 +50,8 @@ OSFAMILIES = { LOG = logging.getLogger(__name__) +@six.add_metaclass(abc.ABCMeta) class Distro(object): - __metaclass__ = abc.ABCMeta usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -97,7 +97,7 @@ class Distro(object): try: res = os.lstat('/run/systemd/system') return stat.S_ISDIR(res.st_mode) - except: + except Exception: return False @abc.abstractmethod diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 75ab340f..32bef1cd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,8 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util from cloudinit import net +from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -221,7 +221,7 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): msg = "removed %s with known contents" % path else: msg = (bmsg + " '%s' exists with user configured content." % path) - except: + except Exception: msg = bmsg + " %s exists, but could not be read." % path LOG.warn(msg) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 53d5604a..b6c43ce8 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -71,8 +71,8 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), key=(lambda e: 0 - len(e))) +@six.add_metaclass(abc.ABCMeta) class Handler(object): - __metaclass__ = abc.ABCMeta def __init__(self, frequency, version=2): self.handler_version = version @@ -118,7 +118,7 @@ def run_part(mod, data, filename, payload, frequency, headers): mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) - except: + except Exception: util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " "frequency %s", mod, content_type, filename, mod_ver, frequency) @@ -157,7 +157,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): # register if it fails starting. handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 - except: + except Exception: util.logexc(LOG, "Failed at registering python file: %s (part " "handler %s)", modfname, curcount) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 07b6d0e0..cad4dc0f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -158,6 +158,6 @@ class CloudConfigPartHandler(handlers.Handler): for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) - except: + except Exception: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index c5bea711..ab381e00 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -80,7 +80,7 @@ def _has_suitable_upstart(): return False try: (version_out, _err) = util.subp(["initctl", "version"]) - except: + except Exception: util.logexc(LOG, "initctl version failed") return False diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 7f00bf1f..09d75e65 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -86,7 +86,7 @@ class FileSemaphores(object): name = canon_sem_name(name) try: yield self._acquire(name, freq) - except: + except Exception: if clear_on_fail: self.clear(name, freq) raise @@ -219,7 +219,7 @@ class ConfigMerger(object): ds_cfg = self._ds.get_config_obj() if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) - except: + except Exception: util.logexc(LOG, "Failed loading of datasource config object " "from %s", self._ds) return d_cfgs @@ -230,7 +230,7 @@ class ConfigMerger(object): e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of env. config from %s', e_fn) return e_cfgs @@ -251,7 +251,7 @@ class ConfigMerger(object): if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs @@ -268,7 +268,7 @@ class ConfigMerger(object): for c_fn in self._fns: try: cfgs.append(util.read_conf(c_fn)) - except: + except Exception: util.logexc(LOG, "Failed loading of configuration from %s", c_fn) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..91e36aca 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -26,9 +26,9 @@ import re import shlex from cloudinit import log as logging +from cloudinit.net import network_state +from cloudinit.net.udev import generate_udev_rule from cloudinit import util -from .udev import generate_udev_rule -from . import network_state LOG = logging.getLogger(__name__) @@ -40,16 +40,16 @@ NET_CONFIG_OPTIONS = [ "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", "netnum", "endpoint", "local", "ttl", - ] +] NET_CONFIG_COMMANDS = [ "pre-up", "up", "post-up", "down", "pre-down", "post-down", - ] +] NET_CONFIG_BRIDGE_OPTIONS = [ "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", - ] +] DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -399,9 +399,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): def render_persistent_net(network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' + '''Given state, emit udev rules to map mac to ifname.''' content = "" interfaces = network_state.get('interfaces') for iface in interfaces.values(): @@ -465,7 +463,7 @@ def iface_add_attrs(iface): def render_route(route, indent=""): - """ When rendering routes for an iface, in some cases applying a route + """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To that end, we will optionally include an '|| true' postfix to each @@ -530,7 +528,7 @@ def iface_start_entry(iface, index): def render_interfaces(network_state): - ''' Given state, emit etc/network/interfaces content ''' + '''Given state, emit etc/network/interfaces content.''' content = "" interfaces = network_state.get('interfaces') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..4c726ab4 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -36,7 +36,7 @@ def from_state_file(state_file): return network_state -class NetworkState: +class NetworkState(object): def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config @@ -53,7 +53,7 @@ class NetworkState: def get_command_handlers(self): METHOD_PREFIX = 'handle_' methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) + x.startswith(METHOD_PREFIX), dir(self)) handlers = {} for m in methods: key = m.replace(METHOD_PREFIX, '') @@ -379,8 +379,9 @@ def mask2cidr(mask): if __name__ == '__main__': - import sys import random + import sys + from cloudinit import net def load_config(nc): diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py index 6435ace0..09188295 100644 --- a/cloudinit/net/udev.py +++ b/cloudinit/net/udev.py @@ -48,7 +48,7 @@ def generate_udev_rule(interface, mac): compose_udev_equality('DRIVERS', '?*'), compose_udev_attr_equality('address', mac), compose_udev_setting('NAME', interface), - ]) + ]) return '%s\n' % rule # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index e30d6fb5..5d24062f 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -20,10 +20,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.log import logging import re +from cloudinit import logging +from cloudinit import util + from prettytable import PrettyTable LOG = logging.getLogger() @@ -163,7 +164,7 @@ def route_info(): def getgateway(): try: routes = route_info() - except: + except Exception: pass else: for r in routes.get('ipv4', []): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 3212d173..dff20ecb 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -4,9 +4,9 @@ import abc import json import six -from ..registry import DictRegistry -from .. import (url_helper, util) -from .. import log as logging +from cloudinit import log as logging +from cloudinit.registry import DictRegistry +from cloudinit import (url_helper, util) LOG = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class LogHandler(ReportingHandler): input_level = level try: level = getattr(logging, level.upper()) - except: + except Exception: LOG.warn("invalid level '%s', using WARN", input_level) level = logging.WARN self.level = level @@ -81,7 +81,7 @@ class WebHookHandler(ReportingHandler): self.endpoint, data=json.dumps(event.as_dict()), timeout=self.timeout, retries=self.retries, ssl_details=self.ssl_details) - except: + except Exception: LOG.warn("failed posting event: %s" % event.as_string()) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..2d046600 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -27,11 +27,12 @@ import xml.etree.ElementTree as ET from xml.dom import minidom +from cloudinit.sources.helpers.azure import get_metadata_from_fabric + from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -40,7 +41,8 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = [ 'sh', '-xc', - "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] + "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x" +] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -51,7 +53,7 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - }, + }, 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } @@ -60,7 +62,7 @@ BUILTIN_CLOUD_CONFIG = { 'ephemeral0': {'table_type': 'gpt', 'layout': [100], 'overwrite': True}, - }, + }, 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0.1', 'replace_fs': 'ntfs'}], @@ -312,7 +314,7 @@ def support_new_ephemeral(cfg): file_count = 0 try: file_count = util.mount_cb(device, count_files) - except: + except Exception: return None LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index b5ee4129..f80956a5 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -4,13 +4,13 @@ # Author: Alexandru Sirbu # -import json import errno +import json from cloudinit import log as logging from cloudinit import sources -from cloudinit import util from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index d7d4e844..33fe78b9 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -19,10 +19,11 @@ from base64 import b64decode import os import re +from cloudinit.cs_utils import Cepko + from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.cs_utils import Cepko LOG = logging.getLogger(__name__) @@ -77,7 +78,7 @@ class DataSourceCloudSigma(sources.DataSource): try: server_context = self.cepko.all().result server_meta = server_context['meta'] - except: + except Exception: # TODO: check for explicit "config on", and then warn # but since no explicit config is available now, just debug. LOG.debug("CloudSigma: Unable to read from serial port") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 4ba019df..4de1f563 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,14 +25,15 @@ # along with this program. If not, see . import os -import time from socket import inet_ntoa from struct import pack +import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit import sources from cloudinit import url_helper as uhelp -from cloudinit import sources, util +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 12e863d2..44a17a00 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import ec2_utils from cloudinit import log as logging -from cloudinit import util from cloudinit import sources -from cloudinit import ec2_utils +from cloudinit import util import functools diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 7e7fc033..9234d1f8 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -18,9 +18,9 @@ from base64 import b64decode from cloudinit import log as logging -from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class DataSourceGCE(sources.DataSource): index = public_key.index(':') if index > 0: return public_key[(index + 1):] - except: + except Exception: return public_key def get_data(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 74d0e5ec..48c61a90 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -61,7 +61,7 @@ class DataSourceNoCloud(sources.DataSource): if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) - except: + except Exception: util.logexc(LOG, "Unable to parse command line data") return False @@ -288,7 +288,7 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): continue try: (k, v) = item.split("=", 1) - except: + except Exception: k = item v = None if k in s2l: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2a6cd050..f2bb9366 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -30,13 +30,20 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from .helpers.vmware.imc.config import Config -from .helpers.vmware.imc.config_file import ConfigFile -from .helpers.vmware.imc.config_nic import NicConfigurator -from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum -from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum -from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum -from .helpers.vmware.imc.guestcust_util import ( + +from cloudinit.sources.helpers.vmware.imc.config \ + import Config +from cloudinit.sources.helpers.vmware.imc.config_file \ + import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic \ + import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_event \ + import GuestCustEventEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_state \ + import GuestCustStateEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( set_customization_status, get_nics_to_enable, enable_nics @@ -262,7 +269,7 @@ def read_ovf_environment(contents): elif prop == "user-data": try: ud = base64.decodestring(val) - except: + except Exception: ud = val return (md, ud, cfg) @@ -277,7 +284,7 @@ def get_ovf_env(dirname): try: contents = util.load_file(full_fn) return (fname, contents) - except: + except Exception: util.logexc(LOG, "Failed loading ovf file %s", full_fn) return (None, False) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 6bf2c33b..43e4fd57 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -45,10 +45,9 @@ class DataSourceNotFoundException(Exception): pass +@six.add_metaclass(abc.ABCMeta) class DataSource(object): - __metaclass__ = abc.ABCMeta - def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 018cac6d..bbfc283c 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -5,6 +5,7 @@ import socket import struct import tempfile import time + from contextlib import contextmanager from xml.etree import ElementTree diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1aa6bbae..156aba6c 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -145,8 +145,8 @@ class SourceMixin(object): return device +@six.add_metaclass(abc.ABCMeta) class BaseReader(object): - __metaclass__ = abc.ABCMeta def __init__(self, base_path): self.base_path = base_path @@ -475,7 +475,7 @@ class MetadataReader(BaseReader): def convert_vendordata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). + """data: a loaded json *object* (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index faba5887..204fbcc4 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class BootProtoEnum: +class BootProtoEnum(object): """Specifies the NIC Boot Settings.""" DHCP = 'dhcp' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index aebc12a0..1dcd053a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -20,7 +20,7 @@ from .nic import Nic -class Config: +class Config(object): """ Stores the Contents specified in the Customization Specification file. diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77098a05..511cc918 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,7 +26,7 @@ from cloudinit import util logger = logging.getLogger(__name__) -class NicConfigurator: +class NicConfigurator(object): def __init__(self, nics): """ Initialize the Nic Configurator diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index a367e476..8a650871 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -18,6 +18,6 @@ # along with this program. If not, see . -class ConfigSource: +class ConfigSource(object): """Specifies a source for the Config Content.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 1b04161f..750be1e3 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustErrorEnum: +class GuestCustErrorEnum(object): """Specifies different errors of Guest Customization engine""" GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py index fc22568f..e13b791d 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustEventEnum: +class GuestCustEventEnum(object): """Specifies different types of Guest Customization Events""" GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py index f255be5f..b9ddf513 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustStateEnum: +class GuestCustStateEnum(object): """Specifies different states of Guest Customization engine""" GUESTCUST_STATE_RUNNING = 4 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index d39f0a65..020ab613 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -23,8 +23,8 @@ import time from cloudinit import util -from .guestcust_state import GuestCustStateEnum from .guestcust_event import GuestCustEventEnum +from .guestcust_state import GuestCustStateEnum logger = logging.getLogger(__name__) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 33f88726..873ddc3b 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class Ipv4ModeEnum: +class Ipv4ModeEnum(object): """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py index 030ba311..3c892db0 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic_base.py +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class NicBase: +class NicBase(object): """ Define what are expected of each nic. The following properties should be provided in an implementation class. @@ -93,7 +93,7 @@ class NicBase: raise NotImplementedError('Check constraints on properties') -class StaticIpv4Base: +class StaticIpv4Base(object): """ Define what are expected of a static IPv4 setting The following properties should be provided in an implementation class. @@ -124,7 +124,7 @@ class StaticIpv4Base: raise NotImplementedError('Ipv4 GATEWAY') -class StaticIpv6Base: +class StaticIpv6Base(object): """Define what are expected of a static IPv6 setting The following properties should be provided in an implementation class. """ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..62d066de 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,10 +44,10 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util -from cloudinit.reporting import events LOG = logging.getLogger(__name__) @@ -483,7 +483,7 @@ class Init(object): c_handlers.initialized.remove(mod) try: handlers.call_end(mod, data, frequency) - except: + except Exception: util.logexc(LOG, "Failed to finalize handler: %s", mod) try: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 936f7da5..c05e9d90 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -28,8 +28,9 @@ import time from email.utils import parsedate from functools import partial -from requests import exceptions + import oauthlib.oauth1 as oauth1 +from requests import exceptions from six.moves.urllib.parse import ( urlparse, urlunparse, @@ -61,7 +62,7 @@ try: SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): CONFIG_ENABLED = True -except: +except ImportError: pass diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f7c5787c..f0631906 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -178,7 +178,7 @@ class UserDataProcessor(object): payload = util.load_yaml(msg.get_payload(decode=True)) if payload: payload_idx = payload.get('launch-index') - except: + except Exception: pass # Header overrides contents, for now (?) or the other way around? if header_idx is not None: diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..8d6cbb4b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -288,7 +288,7 @@ def fork_cb(child_cb, *args, **kwargs): try: child_cb(*args, **kwargs) os._exit(0) - except: + except Exception: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) @@ -472,7 +472,7 @@ def is_ipv4(instr): try: toks = [x for x in toks if int(x) < 256 and int(x) >= 0] - except: + except Exception: return False return len(toks) == 4 @@ -1210,7 +1210,7 @@ def get_cmdline(): else: try: cmdline = load_file("/proc/cmdline").strip() - except: + except Exception: cmdline = "" PROC_CMDLINE = cmdline @@ -1380,7 +1380,7 @@ def read_write_cmdline_url(target_fn): if not os.path.exists(target_fn): try: (key, url, content) = get_cmdline_url() - except: + except Exception: logexc(LOG, "Failed fetching command line url") return try: @@ -1391,7 +1391,7 @@ def read_write_cmdline_url(target_fn): elif key and not content: LOG.debug(("Command line key %s with url" " %s had no contents"), key, url) - except: + except Exception: logexc(LOG, "Failed writing url content to %s", target_fn) @@ -1449,7 +1449,7 @@ def mounts(): mp = m.group(2) fstype = m.group(3) opts = m.group(4) - except: + except Exception: continue # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. @@ -1575,7 +1575,7 @@ def copy(src, dest): def time_rfc2822(): try: ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: + except Exception: ts = "??" return ts @@ -1601,7 +1601,7 @@ def uptime(): bootup = buf.value uptime_str = now - bootup - except: + except Exception: logexc(LOG, "Unable to read uptime using method: %s" % method) return uptime_str @@ -2055,7 +2055,7 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): tmsg += " (N/A)" try: logfunc(msg + tmsg) - except: + except Exception: pass return ret diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..170856ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,8 @@ httpretty>=0.7.1 mock nose -pep8==1.5.7 +pep8>=1.5.7 pyflakes contextlib2 setuptools +flake8>=2.5.4 diff --git a/tox.ini b/tox.ini index bd7c27dd..7a144efe 100644 --- a/tox.ini +++ b/tox.ini @@ -30,3 +30,8 @@ deps = pyflakes setenv = LC_ALL = C + +[flake8] + +ignore=H404,H405,H105,H301,H104,H403,H101 +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,tests -- cgit v1.2.3 From 880d9fc2f9c62abf19b1506595aa81e5417dea45 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 19 May 2016 14:13:07 -0700 Subject: Adjust net module to be more isolated This allows it to be used outside of cloudinit more easily in the future. --- cloudinit/net/__init__.py | 108 ++++++++++++++++++--------------- cloudinit/net/cmdline.py | 15 ++++- cloudinit/net/eni.py | 35 +++++------ cloudinit/net/network_state.py | 38 +++++++++--- cloudinit/sources/helpers/openstack.py | 4 ++ cloudinit/util.py | 9 --- tests/unittests/test_net.py | 7 ++- 7 files changed, 129 insertions(+), 87 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 0202cbd8..07e7307e 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -17,44 +17,81 @@ # along with Curtin. If not, see . import errno +import logging import os - -from cloudinit import log as logging -from cloudinit.net import network_state -from cloudinit import util - +import six +import yaml LOG = logging.getLogger(__name__) SYS_CLASS_NET = "/sys/class/net/" LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-" DEFAULT_PRIMARY_INTERFACE = 'eth0' +# NOTE(harlowja): some of these are similar to what is in cloudinit main +# source or utils tree/module but the reason that is done is so that this +# whole module can be easily extracted and placed into other +# code-bases (curtin for example). -def sys_dev_path(devname, path=""): - return SYS_CLASS_NET + devname + "/" + path +def write_file(path, content): + """Simple writing a file helper.""" + base_path = os.path.dirname(path) + if not os.path.isdir(base_path): + os.makedirs(base_path) + with open(path, "wb+") as fh: + if isinstance(content, six.text_type): + content = content.encode("utf8") + fh.write(content) -def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): +def read_file(path, decode='utf8', enoent=None): try: - contents = "" - with open(sys_dev_path(devname, path), "r") as fp: - contents = fp.read().strip() - if translate is None: - return contents - - try: - return translate.get(contents) - except KeyError: - LOG.debug("found unexpected value '%s' in '%s/%s'", contents, - devname, path) - if keyerror is not None: - return keyerror - raise + with open(path, "rb") as fh: + contents = fh.load() except OSError as e: if e.errno == errno.ENOENT and enoent is not None: return enoent raise + if decode: + return contents.decode(decode) + return contents + + +def dump_yaml(obj): + return yaml.safe_dump(obj, + line_break="\n", + indent=4, + explicit_start=True, + explicit_end=True, + default_flow_style=False) + + +def read_yaml_file(path): + val = yaml.safe_load(read_file(path)) + if not isinstance(val, dict): + gotten_type_name = type(val).__name__ + raise TypeError("Expected dict to be loaded from %s, got" + " '%s' instead" % (path, gotten_type_name)) + return val + + +def sys_dev_path(devname, path=""): + return SYS_CLASS_NET + devname + "/" + path + + +def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None): + contents = read_file(sys_dev_path(devname, path), enoent=enoent) + contents = contents.strip() + if translate is None: + return contents + try: + return translate.get(contents) + except KeyError: + LOG.debug("found unexpected value '%s' in '%s/%s'", contents, + devname, path) + if keyerror is not None: + return keyerror + raise def is_up(devname): @@ -107,31 +144,6 @@ class ParserError(Exception): """Raised when parser has issue parsing the interfaces file.""" -def parse_net_config_data(net_config, skip_broken=True): - """Parses the config, returns NetworkState object - - :param net_config: curtin network config dict - """ - state = None - if 'version' in net_config and 'config' in net_config: - ns = network_state.NetworkState(version=net_config.get('version'), - config=net_config.get('config')) - ns.parse_config(skip_broken=skip_broken) - state = ns.network_state - return state - - -def parse_net_config(path, skip_broken=True): - """Parses a curtin network configuration file and - return network state""" - ns = None - net_config = util.read_conf(path) - if 'network' in net_config: - ns = parse_net_config_data(net_config.get('network'), - skip_broken=skip_broken) - return ns - - def is_disabled_cfg(cfg): if not cfg or not isinstance(cfg, dict): return False @@ -146,7 +158,7 @@ def sys_netdev_info(name, field): fname = os.path.join(SYS_CLASS_NET, name, field) if not os.path.exists(fname): raise OSError("%s: could not find sysfs entry: %s" % (name, fname)) - data = util.load_file(fname) + data = read_file(fname) if data[-1] == '\n': data = data[:-1] return data diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index f5712533..b85d4b0a 100644 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -20,12 +20,25 @@ import base64 import glob import gzip import io +import shlex +import sys + +import six from cloudinit.net import get_devicelist from cloudinit.net import sys_netdev_info from cloudinit import util +PY26 = sys.version_info[0:2] == (2, 6) + + +def _shlex_split(blob): + if PY26 and isinstance(blob, six.text_type): + # Older versions don't support unicode input + blob = blob.encode("utf8") + return shlex.split(blob) + def _load_shell_content(content, add_empty=False, empty_val=None): """Given shell like syntax (key=value\nkey2=value2\n) in content @@ -33,7 +46,7 @@ def _load_shell_content(content, add_empty=False, empty_val=None): then add entries in to the returned dictionary for 'VAR=' variables. Set their value to empty_val.""" data = {} - for line in util.shlex_split(content): + for line in _shlex_split(content): try: key, value = line.split("=", 1) except ValueError: diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py index b1bdac24..adb31c22 100644 --- a/cloudinit/net/eni.py +++ b/cloudinit/net/eni.py @@ -16,10 +16,11 @@ import glob import os import re +from cloudinit import net + from cloudinit.net import LINKS_FNAME_PREFIX from cloudinit.net import ParserError from cloudinit.net.udev import generate_udev_rule -from cloudinit import util NET_CONFIG_COMMANDS = [ @@ -363,16 +364,13 @@ class Renderer(object): links_prefix=LINKS_FNAME_PREFIX, netrules='etc/udev/rules.d/70-persistent-net.rules'): - fpeni = os.path.sep.join((target, eni,)) - util.ensure_dir(os.path.dirname(fpeni)) - with open(fpeni, 'w+') as f: - f.write(self._render_interfaces(network_state)) + fpeni = os.path.join(target, eni) + net.write_file(fpeni, self._render_interfaces(network_state)) if netrules: - netrules = os.path.sep.join((target, netrules,)) - util.ensure_dir(os.path.dirname(netrules)) - with open(netrules, 'w+') as f: - f.write(self._render_persistent_net(network_state)) + netrules = os.path.join(target, netrules) + net.write_file(netrules, + self._render_persistent_net(network_state)) if links_prefix: self._render_systemd_links(target, network_state, links_prefix) @@ -382,18 +380,17 @@ class Renderer(object): fp_prefix = os.path.sep.join((target, links_prefix)) for f in glob.glob(fp_prefix + "*"): os.unlink(f) - interfaces = network_state.get('interfaces') for iface in interfaces.values(): if (iface['type'] == 'physical' and 'name' in iface and iface.get('mac_address')): fname = fp_prefix + iface['name'] + ".link" - with open(fname, "w") as fp: - fp.write("\n".join([ - "[Match]", - "MACAddress=" + iface['mac_address'], - "", - "[Link]", - "Name=" + iface['name'], - "" - ])) + content = "\n".join([ + "[Match]", + "MACAddress=" + iface['mac_address'], + "", + "[Link]", + "Name=" + iface['name'], + "" + ]) + net.write_file(fname, content) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 2feffa71..c5aeadb5 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -16,12 +16,11 @@ # along with Curtin. If not, see . import copy +import logging import six -from cloudinit import log as logging -from cloudinit import util -from cloudinit.util import yaml_dumps as dump_config +from cloudinit import net LOG = logging.getLogger(__name__) @@ -31,9 +30,34 @@ NETWORK_STATE_REQUIRED_KEYS = { } +def parse_net_config_data(net_config, skip_broken=True): + """Parses the config, returns NetworkState object + + :param net_config: curtin network config dict + """ + state = None + if 'version' in net_config and 'config' in net_config: + ns = NetworkState(version=net_config.get('version'), + config=net_config.get('config')) + ns.parse_config(skip_broken=skip_broken) + state = ns.network_state + return state + + +def parse_net_config(path, skip_broken=True): + """Parses a curtin network configuration file and + return network state""" + ns = None + net_config = net.read_yaml_file(path) + if 'network' in net_config: + ns = parse_net_config_data(net_config.get('network'), + skip_broken=skip_broken) + return ns + + def from_state_file(state_file): network_state = None - state = util.read_conf(state_file) + state = net.read_yaml_file(state_file) network_state = NetworkState() network_state.load(state) return network_state @@ -111,7 +135,7 @@ class NetworkState(object): 'config': self.config, 'network_state': self.network_state, } - return dump_config(state) + return net.dump_yaml(state) def load(self, state): if 'version' not in state: @@ -121,7 +145,7 @@ class NetworkState(object): required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']] missing_keys = diff_keys(required_keys, state) if missing_keys: - msg = 'Invalid state, missing keys: %s'.format(missing_keys) + msg = 'Invalid state, missing keys: %s' % (missing_keys) LOG.error(msg) raise ValueError(msg) @@ -130,7 +154,7 @@ class NetworkState(object): setattr(self, key, state[key]) def dump_network_state(self): - return dump_config(self.network_state) + return net.dump_yaml(self.network_state) def parse_config(self, skip_broken=True): # rebuild network state diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index b2acc648..f85fd864 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -551,6 +551,10 @@ def convert_net_json(network_json): 'type': 'static', 'address': network.get('ip_address'), }) + if network['type'] == 'ipv6': + subnet['ipv6'] = True + else: + subnet['ipv4'] = True subnets.append(subnet) cfg.update({'subnets': subnets}) if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: diff --git a/cloudinit/util.py b/cloudinit/util.py index 7562b97a..2bec476e 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -37,7 +37,6 @@ import pwd import random import re import shutil -import shlex import socket import stat import string @@ -82,7 +81,6 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], ['lxc-is-container']) PROC_CMDLINE = None -PY26 = sys.version_info[0:2] == (2, 6) def decode_binary(blob, encoding='utf-8'): @@ -1202,13 +1200,6 @@ def load_file(fname, read_cb=None, quiet=False, decode=True): return contents -def shlex_split(blob): - if PY26 and isinstance(blob, six.text_type): - # Older versions don't support unicode input - blob = blob.encode("utf8") - return shlex.split(blob) - - def get_cmdline(): if 'DEBUG_PROC_CMDLINE' in os.environ: return os.environ["DEBUG_PROC_CMDLINE"] diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index ed2c6d0f..75c433f6 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -2,6 +2,7 @@ from cloudinit import util from cloudinit import net from cloudinit.net import cmdline from cloudinit.net import eni +from cloudinit.net import network_state from .helpers import TestCase from .helpers import mock @@ -112,14 +113,14 @@ class TestEniNetRendering(TestCase): mock_sys_dev_path.side_effect = sys_dev_path network_cfg = net.generate_fallback_config() - network_state = net.parse_net_config_data(network_cfg, - skip_broken=False) + ns = network_state.parse_net_config_data(network_cfg, + skip_broken=False) render_dir = os.path.join(tmp_dir, "render") os.makedirs(render_dir) renderer = eni.Renderer() - renderer.render_network_state(render_dir, network_state, + renderer.render_network_state(render_dir, ns, eni="interfaces", links_prefix=None, netrules=None) -- cgit v1.2.3 From ee239517c5342cbd62c9fdeaf735d78d6fd1fbb8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 11:59:14 +0200 Subject: support apt_sources to be a dictionary key is the filename, and "old" input shall be handled as it was all the time. For compatibility this will (continue to) overwrite the file of multiple options that did not specify an output file (they all get the same default). Yet it will process them all - as it always did - e.g. to add the keys of all of them. Any users of the new format won't have these issues, as they will always have a key. --- cloudinit/config/cc_apt_configure.py | 34 +++++++++++++++++++++++++--------- cloudinit/util.py | 8 ++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e5a962ac..a46ebb3e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,8 +215,28 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + # convert old list format to new dict based format + if isinstance(srclist, list): + srcdict = {} + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + else: + srcdict = srclist + errorlist = [] - for ent in srclist: + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent[filename] = filename + # keys can be added without specifying a source try: add_key(ent) @@ -226,10 +246,13 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -238,13 +261,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..2931efbd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,14 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey def read_conf(fname): try: -- cgit v1.2.3 From a63a64a70def97730d2ab544b0df9f87f3484333 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:53:03 +0200 Subject: final pep8 check fixups --- cloudinit/util.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 2931efbd..0773af69 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,7 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) + def rand_dict_key(dictionary, postfix=None): if not postfix: postfix = "" @@ -344,6 +345,7 @@ def rand_dict_key(dictionary, postfix=None): break return newkey + def read_conf(fname): try: return load_yaml(load_file(fname), default={}) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 237cf14d..c19904fb 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -55,7 +55,6 @@ class TestAptSourceConfig(TestCase): self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - @staticmethod def _get_default_params(): """ get_default_params @@ -68,9 +67,9 @@ class TestAptSourceConfig(TestCase): def myjoin(self, *args, **kwargs): """ myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" - and args[1] == "cloud_config_sources.list" - and len(args) == 2): + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): return self.join(self.tmp, args[0].lstrip("/"), args[1]) else: return self.join(*args, **kwargs) @@ -137,7 +136,6 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_src_basic_tri(self): """ test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in @@ -232,7 +230,6 @@ class TestAptSourceConfig(TestCase): "universe"), contents, flags=re.IGNORECASE)) - def test_apt_src_replace_tri(self): """ test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with -- cgit v1.2.3 From 922562bccbc6b6c7f3309ecd36f1835b2ad817da Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 23:32:22 -0400 Subject: python3 OSError does not have a .message --- cloudinit/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/util.py') diff --git a/cloudinit/util.py b/cloudinit/util.py index 8873264d..e5dd61a0 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2210,7 +2210,7 @@ def _call_dmidecode(key, dmidecode_path): return "" return result except (IOError, OSError) as _err: - LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message) + LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err) return None -- cgit v1.2.3