From cca640d332eb8a6b068033a28b0b319873c7fbf6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:21:50 +0200 Subject: allow to add keys without specifying a source --- cloudinit/config/cc_apt_configure.py | 42 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -164,6 +164,29 @@ def generate_sources_list(codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key(ent, errorlist): + """ + add key to the system as defiend in entry (if any) + suppords raw keys or keyid's + The latter will as a first step fetched to get the raw key + """ + if ('keyid' in ent and 'key' not in ent): + keyserver = "keyserver.ubuntu.com" + if 'keyserver' in ent: + keyserver = ent['keyserver'] + try: + ent['key'] = getkeybyid(ent['keyid'], keyserver) + except: + errorlist.append([ent, "failed to get key from %s" % keyserver]) + return + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + errorlist.append([ent, "failed add key"]) + + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated @@ -179,6 +202,9 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: + # keys can be added without specifying a source + add_key(ent, errorlist) + if 'source' not in ent: errorlist.append(["", "missing source"]) continue @@ -201,22 +227,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], ks) - except: - errorlist.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - errorlist.append([source, "failed add key"]) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") -- cgit v1.2.3 From 6fc583a4bcb49f7dbecdac095bc63e90dd6edaf3 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:43:59 +0200 Subject: test mirror list with failing mirror --- cloudinit/config/cc_apt_configure.py | 1 + .../test_handler_apt_configure_sources_list.py | 23 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..ccbdcbc1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,6 +57,7 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) + print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index aff272a3..bac2da24 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,14 +58,20 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # TODO Later - custom template filename # TODO Later - custom template raw - def apt_source_list(self, distro, mirror): + def apt_source_list(self, distro, mirror, mirrorcheck=None): """ apt_source_list Test rendering of a source.list from template for a given distro """ self.patchOS(self.new_root) self.patchUtils(self.new_root) - cfg = {'apt_mirror': mirror} + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} mycloud = self._get_cloud(distro) with mock.patch.object(templater, 'render_to_file') as mocktmpl: @@ -81,9 +87,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): '/etc/apt/sources.list', {'codename': '', 'primary': - mirror, + mirrorcheck, 'mirror': - mirror}) + mirrorcheck}) def test_apt_source_list_ubuntu(self): @@ -100,4 +106,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('debian', 'ftp.us.debian.org') + def test_apt_srcl_ubuntu_mirrorfail(self): + """ test_apt_source_list_ubuntu_mirrorfail + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + + # vi: ts=4 expandtab -- cgit v1.2.3 From 18864f8e7331da359399decb1b080e36fa343f5a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:45:15 +0200 Subject: remove missed test print --- cloudinit/config/cc_apt_configure.py | 1 - .../test_handler/test_handler_apt_configure_sources_list.py | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ccbdcbc1..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,7 +57,6 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) - print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index d48167c9..b8fe03ae 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -106,6 +106,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + def test_apt_srcl_debian_mirrorfail(self): + """ test_apt_source_list_debian_mirrorfail + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', ['http://does.not.exist', + 'ftp.us.debian.org'], + 'ftp.us.debian.org') + + def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu -- cgit v1.2.3 From 0d8bf6f2c1464b5f4dab735841a50d02016d2caf Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:51:21 +0200 Subject: add feature to allow a custom template for source list --- cloudinit/config/cc_apt_configure.py | 18 +++++++++++++----- cloudinit/templater.py | 5 +++++ 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..2ab5e86c 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,6 +21,7 @@ import glob import os import re +import tempfile from cloudinit import templater from cloudinit import util @@ -70,7 +71,7 @@ def handle(name, cfg, cloud, log, _args): if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirrors, cloud, log) + generate_sources_list(cfg, release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', {"primary": "archive.ubuntu.com/ubuntu", "security": "security.ubuntu.com/ubuntu"}) @@ -149,7 +150,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirrors, cloud, log): +def generate_sources_list(cfg, codename, mirrors, cloud, log): + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + + custtmpl = cfg.get('apt_custom_sources_list', None) + if custtmpl is not None: + templater.render_string_to_file(custtmpl, + '/etc/apt/sources.list', params) + return + template_fn = cloud.get_template_filename('sources.list.%s' % (cloud.distro.name)) if not template_fn: @@ -158,9 +169,6 @@ def generate_sources_list(codename, mirrors, cloud, log): log.warn("No template found, not rendering /etc/apt/sources.list") return - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] templater.render_to_file(template_fn, '/etc/apt/sources.list', params) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index a9231482..8a6ad417 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -142,6 +142,11 @@ def render_to_file(fn, outfn, params, mode=0o644): util.write_file(outfn, contents, mode=mode) +def render_string_to_file(content, outfn, params, mode=0o644): + contents = render_string(content, params) + util.write_file(outfn, contents, mode=mode) + + def render_string(content, params): if not params: params = {} -- cgit v1.2.3 From 9c6b5f54ad5b83131de6d997930bd9f4031e6a83 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:01 +0200 Subject: move errorlist.append out of add_key --- cloudinit/config/cc_apt_configure.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2ab5e86c..91d02815 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -172,7 +172,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_key(ent, errorlist): +def add_key(ent): """ add key to the system as defiend in entry (if any) suppords raw keys or keyid's @@ -185,14 +185,13 @@ def add_key(ent, errorlist): try: ent['key'] = getkeybyid(ent['keyid'], keyserver) except: - errorlist.append([ent, "failed to get key from %s" % keyserver]) - return + raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) except: - errorlist.append([ent, "failed add key"]) + raise Exception('failed add key') def add_sources(srclist, template_params=None, aa_repo_match=None): @@ -211,7 +210,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: # keys can be added without specifying a source - add_key(ent, errorlist) + try: + add_key(ent) + except Exception as detail: + errorlist.append([ent, detail]) if 'source' not in ent: errorlist.append(["", "missing source"]) -- cgit v1.2.3 From e55ccfa5670e16aa7431a193d0838aa7d04db4d5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:34 +0200 Subject: remove Unnecessary parens in add_key --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 91d02815..492c3c00 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -178,7 +178,7 @@ def add_key(ent): suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ - if ('keyid' in ent and 'key' not in ent): + if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] -- cgit v1.2.3 From f11827aee59cf2678c3d2a157218a3fe2831f5c2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 10:56:26 -0700 Subject: Fix up a ton of flake8 issues --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/config/cc_bootcmd.py | 4 ++-- cloudinit/config/cc_disk_setup.py | 2 +- cloudinit/config/cc_emit_upstart.py | 2 +- cloudinit/config/cc_fan.py | 2 +- cloudinit/config/cc_final_message.py | 2 +- cloudinit/config/cc_grub_dpkg.py | 2 +- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_lxd.py | 2 +- cloudinit/config/cc_mounts.py | 10 ++++----- cloudinit/config/cc_phone_home.py | 6 +++--- cloudinit/config/cc_rightscale_userdata.py | 4 ++-- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_scripts_per_boot.py | 2 +- cloudinit/config/cc_scripts_per_instance.py | 2 +- cloudinit/config/cc_scripts_per_once.py | 2 +- cloudinit/config/cc_scripts_user.py | 2 +- cloudinit/config/cc_scripts_vendor.py | 2 +- cloudinit/config/cc_seed_random.py | 2 +- cloudinit/config/cc_set_passwords.py | 2 +- cloudinit/config/cc_snappy.py | 4 ++-- cloudinit/config/cc_ssh.py | 6 +++--- cloudinit/config/cc_ssh_import_id.py | 4 ++-- cloudinit/config/cc_ubuntu_init_switch.py | 4 ++-- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/debian.py | 4 ++-- cloudinit/handlers/__init__.py | 6 +++--- cloudinit/handlers/cloud_config.py | 2 +- cloudinit/handlers/upstart_job.py | 2 +- cloudinit/helpers.py | 10 ++++----- cloudinit/net/__init__.py | 18 +++++++--------- cloudinit/net/network_state.py | 7 +++--- cloudinit/net/udev.py | 2 +- cloudinit/netinfo.py | 7 +++--- cloudinit/reporting/handlers.py | 10 ++++----- cloudinit/sources/DataSourceAzure.py | 12 ++++++----- cloudinit/sources/DataSourceBigstep.py | 4 ++-- cloudinit/sources/DataSourceCloudSigma.py | 5 +++-- cloudinit/sources/DataSourceCloudStack.py | 5 +++-- cloudinit/sources/DataSourceDigitalOcean.py | 4 ++-- cloudinit/sources/DataSourceGCE.py | 4 ++-- cloudinit/sources/DataSourceNoCloud.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 25 ++++++++++++++-------- cloudinit/sources/__init__.py | 3 +-- cloudinit/sources/helpers/azure.py | 1 + cloudinit/sources/helpers/openstack.py | 4 ++-- cloudinit/sources/helpers/vmware/imc/boot_proto.py | 2 +- cloudinit/sources/helpers/vmware/imc/config.py | 2 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 2 +- .../sources/helpers/vmware/imc/config_source.py | 2 +- .../sources/helpers/vmware/imc/guestcust_error.py | 2 +- .../sources/helpers/vmware/imc/guestcust_event.py | 2 +- .../sources/helpers/vmware/imc/guestcust_state.py | 2 +- .../sources/helpers/vmware/imc/guestcust_util.py | 2 +- cloudinit/sources/helpers/vmware/imc/ipv4_mode.py | 2 +- cloudinit/sources/helpers/vmware/imc/nic_base.py | 6 +++--- cloudinit/stages.py | 4 ++-- cloudinit/url_helper.py | 5 +++-- cloudinit/user_data.py | 2 +- cloudinit/util.py | 18 ++++++++-------- test-requirements.txt | 3 ++- tox.ini | 5 +++++ 62 files changed, 150 insertions(+), 132 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..e3fadc12 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -207,20 +207,20 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ks = ent['keyserver'] try: ent['key'] = getkeybyid(ent['keyid'], ks) - except: + except Exception: errorlist.append([source, "failed to get key from %s" % ks]) continue if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) - except: + except Exception: errorlist.append([source, "failed add key"]) try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") - except: + except Exception: errorlist.append([source, "failed write to file %s" % ent['filename']]) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index a295cc4e..b763a3c3 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -38,7 +38,7 @@ def handle(name, cfg, cloud, log, _args): content = util.shellify(cfg["bootcmd"]) tmpf.write(util.encode_text(content)) tmpf.flush() - except: + except Exception: util.logexc(log, "Failed to shellify bootcmd") raise @@ -49,6 +49,6 @@ def handle(name, cfg, cloud, log, _args): env['INSTANCE_ID'] = str(iid) cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) - except: + except Exception: util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index bbaf9646..b642f1f8 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -198,7 +198,7 @@ def is_device_valid(name, partition=False): d_type = "" try: d_type = device_type(name) - except: + except Exception: LOG.warn("Query against device %s failed" % name) return False diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 86ae97ab..06c53272 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -20,8 +20,8 @@ import os -from cloudinit.settings import PER_ALWAYS from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS from cloudinit import util frequency = PER_ALWAYS diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py index 39e3850e..545fee22 100644 --- a/cloudinit/config/cc_fan.py +++ b/cloudinit/config/cc_fan.py @@ -37,8 +37,8 @@ If cloud-init sees a 'fan' entry in cloud-config it will """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index 4a51476f..c9021eb1 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -66,7 +66,7 @@ def handle(_name, cfg, cloud, log, args): try: contents = "%s - %s - v. %s\n" % (uptime, ts, cver) util.write_file(boot_fin_fn, contents) - except: + except Exception: util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn) if cloud.datasource.is_disconnected: diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py index 3c2d9985..156722d9 100644 --- a/cloudinit/config/cc_grub_dpkg.py +++ b/cloudinit/config/cc_grub_dpkg.py @@ -69,5 +69,5 @@ def handle(name, cfg, _cloud, log, _args): try: util.subp(['debconf-set-selections'], dconf_sel) - except: + except Exception: util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg") diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index aa844ee9..9a02f056 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -57,6 +57,6 @@ def handle(name, cfg, cloud, log, _args): (stdout, _stderr) = util.subp(cmd) util.multi_log("%s\n" % (stdout.strip()), stderr=False, console=True) - except: + except Exception: log.warn("Writing keys to the system console failed!") raise diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index bf735648..b1de8f84 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -111,7 +111,7 @@ def handle(name, cfg, cloud, log, args): data = "\n".join(["set %s %s" % (k, v) for k, v in debconf.items()]) + "\n" util.subp(['debconf-communicate'], data) - except: + except Exception: util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm) # Remove the existing configuration file (forces re-generation) diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 4fe3ee21..2b981935 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -244,7 +244,7 @@ def handle_swapcfg(swapcfg): LOG.debug("swap file %s already in use.", fname) return fname LOG.debug("swap file %s existed, but not in /proc/swaps", fname) - except: + except Exception: LOG.warn("swap file %s existed. Error reading /proc/swaps", fname) return fname @@ -379,7 +379,7 @@ def handle(_name, cfg, cloud, log, _args): toks = WS.split(line) if toks[3].find(comment) != -1: continue - except: + except Exception: pass fstab_lines.append(line) @@ -390,16 +390,16 @@ def handle(_name, cfg, cloud, log, _args): if needswap: try: util.subp(("swapon", "-a")) - except: + except Exception: util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: try: util.ensure_dir(d) - except: + except Exception: util.logexc(log, "Failed to make '%s' config-mount", d) try: util.subp(("mount", "-a")) - except: + except Exception: util.logexc(log, "Activating mounts via 'mount -a' failed") diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 3dcc9459..72176d42 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, args): tries = ph_cfg.get('tries') try: tries = int(tries) - except: + except Exception: tries = 10 util.logexc(log, "Configuration entry 'tries' is not an integer, " "using %s instead", tries) @@ -87,7 +87,7 @@ def handle(name, cfg, cloud, log, args): for (n, path) in pubkeys.items(): try: all_keys[n] = util.load_file(path) - except: + except Exception: util.logexc(log, "%s: failed to open, can not phone home that " "data!", path) @@ -117,6 +117,6 @@ def handle(name, cfg, cloud, log, args): util.read_file_or_url(url, data=real_submit_keys, retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) - except: + except Exception: util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 0ecf3a4d..8118fac4 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -52,7 +52,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK' def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() - except: + except Exception: log.debug("Failed to get raw userdata in module %s", name) return @@ -63,7 +63,7 @@ def handle(name, _cfg, cloud, log, _args): "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) return - except: + except Exception: util.logexc(log, "Failed to parse query string %s into a dictionary", ud) raise diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 66dc3363..bc09d38c 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -34,5 +34,5 @@ def handle(name, cfg, cloud, log, _args): try: content = util.shellify(cmd) util.write_file(out_fn, content, 0o700) - except: + except Exception: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 42b987eb..ee3b6c9f 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index b5d71c13..c0d62b12 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index d77d36d5..ecb527f6 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 5c53014f..699857d1 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -36,7 +36,7 @@ def handle(name, _cfg, cloud, log, _args): runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py index 0c9e504e..80bf10ff 100644 --- a/cloudinit/config/cc_scripts_vendor.py +++ b/cloudinit/config/cc_scripts_vendor.py @@ -37,7 +37,7 @@ def handle(name, cfg, cloud, log, _args): try: util.runparts(runparts_path, exe_prefix=prefix) - except: + except Exception: log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py index 1b011216..5085c23a 100644 --- a/cloudinit/config/cc_seed_random.py +++ b/cloudinit/config/cc_seed_random.py @@ -24,8 +24,8 @@ import os from six import BytesIO -from cloudinit.settings import PER_INSTANCE from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 58e1b713..5c8c23b8 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -155,7 +155,7 @@ def handle(_name, cfg, cloud, log, args): cmd = filter(None, cmd) # Remove empty arguments util.subp(cmd) log.debug("Restarted the ssh daemon") - except: + except Exception: util.logexc(log, "Restarting of the ssh daemon failed") if len(errors): diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index fa9d54a0..1a485ee6 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -47,12 +47,12 @@ Example config: """ from cloudinit import log as logging -from cloudinit import util from cloudinit.settings import PER_INSTANCE +from cloudinit import util import glob -import tempfile import os +import tempfile LOG = logging.getLogger(__name__) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index d24e43c0..cb9b70aa 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -57,7 +57,7 @@ def handle(_name, cfg, cloud, log, _args): for f in glob.glob(key_pth): try: util.del_file(f) - except: + except Exception: util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: @@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) - except: + except Exception: util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) else: @@ -122,7 +122,7 @@ def handle(_name, cfg, cloud, log, _args): keys.extend(cfgkeys) apply_credentials(keys, user, disable_root, disable_root_opts) - except: + except Exception: util.logexc(log, "Applying ssh credentials failed!") diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 2d480d7e..28c4585b 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -52,14 +52,14 @@ def handle(_name, cfg, cloud, log, args): else: try: import_ids = user_cfg['ssh_import_id'] - except: + except Exception: log.debug("User %s is not configured for ssh_import_id", user) continue try: import_ids = util.uniq_merge(import_ids) import_ids = [str(i) for i in import_ids] - except: + except Exception: log.debug("User %s is not correctly configured for ssh_import_id", user) continue diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py index 7e88ed85..884d79f1 100644 --- a/cloudinit/config/cc_ubuntu_init_switch.py +++ b/cloudinit/config/cc_ubuntu_init_switch.py @@ -40,10 +40,10 @@ It can be configured with the following option structure:: mechanism you've used to switch the init system. """ -from cloudinit.settings import PER_INSTANCE +from cloudinit.distros import ubuntu from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE from cloudinit import util -from cloudinit.distros import ubuntu import os import time diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5879dabf..0f222c8c 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -50,8 +50,8 @@ OSFAMILIES = { LOG = logging.getLogger(__name__) +@six.add_metaclass(abc.ABCMeta) class Distro(object): - __metaclass__ = abc.ABCMeta usr_lib_exec = "/usr/lib" hosts_fn = "/etc/hosts" @@ -97,7 +97,7 @@ class Distro(object): try: res = os.lstat('/run/systemd/system') return stat.S_ISDIR(res.st_mode) - except: + except Exception: return False @abc.abstractmethod diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 75ab340f..32bef1cd 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -25,8 +25,8 @@ import os from cloudinit import distros from cloudinit import helpers from cloudinit import log as logging -from cloudinit import util from cloudinit import net +from cloudinit import util from cloudinit.distros.parsers.hostname import HostnameConf @@ -221,7 +221,7 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"): msg = "removed %s with known contents" % path else: msg = (bmsg + " '%s' exists with user configured content." % path) - except: + except Exception: msg = bmsg + " %s exists, but could not be read." % path LOG.warn(msg) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 53d5604a..b6c43ce8 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -71,8 +71,8 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), key=(lambda e: 0 - len(e))) +@six.add_metaclass(abc.ABCMeta) class Handler(object): - __metaclass__ = abc.ABCMeta def __init__(self, frequency, version=2): self.handler_version = version @@ -118,7 +118,7 @@ def run_part(mod, data, filename, payload, frequency, headers): mod.handle_part(data, content_type, filename, payload) else: raise ValueError("Unknown module version %s" % (mod_ver)) - except: + except Exception: util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " "frequency %s", mod, content_type, filename, mod_ver, frequency) @@ -157,7 +157,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): # register if it fails starting. handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 - except: + except Exception: util.logexc(LOG, "Failed at registering python file: %s (part " "handler %s)", modfname, curcount) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 07b6d0e0..cad4dc0f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -158,6 +158,6 @@ class CloudConfigPartHandler(handlers.Handler): for i in ("\n", "\r", "\t"): filename = filename.replace(i, " ") self.file_names.append(filename.strip()) - except: + except Exception: util.logexc(LOG, "Failed at merging in cloud config part from %s", filename) diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index c5bea711..ab381e00 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -80,7 +80,7 @@ def _has_suitable_upstart(): return False try: (version_out, _err) = util.subp(["initctl", "version"]) - except: + except Exception: util.logexc(LOG, "initctl version failed") return False diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 7f00bf1f..09d75e65 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -86,7 +86,7 @@ class FileSemaphores(object): name = canon_sem_name(name) try: yield self._acquire(name, freq) - except: + except Exception: if clear_on_fail: self.clear(name, freq) raise @@ -219,7 +219,7 @@ class ConfigMerger(object): ds_cfg = self._ds.get_config_obj() if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) - except: + except Exception: util.logexc(LOG, "Failed loading of datasource config object " "from %s", self._ds) return d_cfgs @@ -230,7 +230,7 @@ class ConfigMerger(object): e_fn = os.environ[CFG_ENV_NAME] try: e_cfgs.append(util.read_conf(e_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of env. config from %s', e_fn) return e_cfgs @@ -251,7 +251,7 @@ class ConfigMerger(object): if cc_fn and os.path.isfile(cc_fn): try: i_cfgs.append(util.read_conf(cc_fn)) - except: + except Exception: util.logexc(LOG, 'Failed loading of cloud-config from %s', cc_fn) return i_cfgs @@ -268,7 +268,7 @@ class ConfigMerger(object): for c_fn in self._fns: try: cfgs.append(util.read_conf(c_fn)) - except: + except Exception: util.logexc(LOG, "Failed loading of configuration from %s", c_fn) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 31544fd8..91e36aca 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -26,9 +26,9 @@ import re import shlex from cloudinit import log as logging +from cloudinit.net import network_state +from cloudinit.net.udev import generate_udev_rule from cloudinit import util -from .udev import generate_udev_rule -from . import network_state LOG = logging.getLogger(__name__) @@ -40,16 +40,16 @@ NET_CONFIG_OPTIONS = [ "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime", "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame", "netnum", "endpoint", "local", "ttl", - ] +] NET_CONFIG_COMMANDS = [ "pre-up", "up", "post-up", "down", "pre-down", "post-down", - ] +] NET_CONFIG_BRIDGE_OPTIONS = [ "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit", "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp", - ] +] DEFAULT_PRIMARY_INTERFACE = 'eth0' @@ -399,9 +399,7 @@ def config_from_klibc_net_cfg(files=None, mac_addrs=None): def render_persistent_net(network_state): - ''' Given state, emit udev rules to map - mac to ifname - ''' + '''Given state, emit udev rules to map mac to ifname.''' content = "" interfaces = network_state.get('interfaces') for iface in interfaces.values(): @@ -465,7 +463,7 @@ def iface_add_attrs(iface): def render_route(route, indent=""): - """ When rendering routes for an iface, in some cases applying a route + """When rendering routes for an iface, in some cases applying a route may result in the route command returning non-zero which produces some confusing output for users manually using ifup/ifdown[1]. To that end, we will optionally include an '|| true' postfix to each @@ -530,7 +528,7 @@ def iface_start_entry(iface, index): def render_interfaces(network_state): - ''' Given state, emit etc/network/interfaces content ''' + '''Given state, emit etc/network/interfaces content.''' content = "" interfaces = network_state.get('interfaces') diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index e32d2cdf..4c726ab4 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -36,7 +36,7 @@ def from_state_file(state_file): return network_state -class NetworkState: +class NetworkState(object): def __init__(self, version=NETWORK_STATE_VERSION, config=None): self.version = version self.config = config @@ -53,7 +53,7 @@ class NetworkState: def get_command_handlers(self): METHOD_PREFIX = 'handle_' methods = filter(lambda x: callable(getattr(self, x)) and - x.startswith(METHOD_PREFIX), dir(self)) + x.startswith(METHOD_PREFIX), dir(self)) handlers = {} for m in methods: key = m.replace(METHOD_PREFIX, '') @@ -379,8 +379,9 @@ def mask2cidr(mask): if __name__ == '__main__': - import sys import random + import sys + from cloudinit import net def load_config(nc): diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py index 6435ace0..09188295 100644 --- a/cloudinit/net/udev.py +++ b/cloudinit/net/udev.py @@ -48,7 +48,7 @@ def generate_udev_rule(interface, mac): compose_udev_equality('DRIVERS', '?*'), compose_udev_attr_equality('address', mac), compose_udev_setting('NAME', interface), - ]) + ]) return '%s\n' % rule # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py index e30d6fb5..5d24062f 100644 --- a/cloudinit/netinfo.py +++ b/cloudinit/netinfo.py @@ -20,10 +20,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import cloudinit.util as util -from cloudinit.log import logging import re +from cloudinit import logging +from cloudinit import util + from prettytable import PrettyTable LOG = logging.getLogger() @@ -163,7 +164,7 @@ def route_info(): def getgateway(): try: routes = route_info() - except: + except Exception: pass else: for r in routes.get('ipv4', []): diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py index 3212d173..dff20ecb 100644 --- a/cloudinit/reporting/handlers.py +++ b/cloudinit/reporting/handlers.py @@ -4,9 +4,9 @@ import abc import json import six -from ..registry import DictRegistry -from .. import (url_helper, util) -from .. import log as logging +from cloudinit import log as logging +from cloudinit.registry import DictRegistry +from cloudinit import (url_helper, util) LOG = logging.getLogger(__name__) @@ -36,7 +36,7 @@ class LogHandler(ReportingHandler): input_level = level try: level = getattr(logging, level.upper()) - except: + except Exception: LOG.warn("invalid level '%s', using WARN", input_level) level = logging.WARN self.level = level @@ -81,7 +81,7 @@ class WebHookHandler(ReportingHandler): self.endpoint, data=json.dumps(event.as_dict()), timeout=self.timeout, retries=self.retries, ssl_details=self.ssl_details) - except: + except Exception: LOG.warn("failed posting event: %s" % event.as_string()) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 698f4cac..2d046600 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -27,11 +27,12 @@ import xml.etree.ElementTree as ET from xml.dom import minidom +from cloudinit.sources.helpers.azure import get_metadata_from_fabric + from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.azure import get_metadata_from_fabric LOG = logging.getLogger(__name__) @@ -40,7 +41,8 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] BOUNCE_COMMAND = [ 'sh', '-xc', - "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] + "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x" +] BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, @@ -51,7 +53,7 @@ BUILTIN_DS_CONFIG = { 'policy': True, 'command': BOUNCE_COMMAND, 'hostname_command': 'hostname', - }, + }, 'disk_aliases': {'ephemeral0': '/dev/sdb'}, } @@ -60,7 +62,7 @@ BUILTIN_CLOUD_CONFIG = { 'ephemeral0': {'table_type': 'gpt', 'layout': [100], 'overwrite': True}, - }, + }, 'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0.1', 'replace_fs': 'ntfs'}], @@ -312,7 +314,7 @@ def support_new_ephemeral(cfg): file_count = 0 try: file_count = util.mount_cb(device, count_files) - except: + except Exception: return None LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count) diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py index b5ee4129..f80956a5 100644 --- a/cloudinit/sources/DataSourceBigstep.py +++ b/cloudinit/sources/DataSourceBigstep.py @@ -4,13 +4,13 @@ # Author: Alexandru Sirbu # -import json import errno +import json from cloudinit import log as logging from cloudinit import sources -from cloudinit import util from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index d7d4e844..33fe78b9 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -19,10 +19,11 @@ from base64 import b64decode import os import re +from cloudinit.cs_utils import Cepko + from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.cs_utils import Cepko LOG = logging.getLogger(__name__) @@ -77,7 +78,7 @@ class DataSourceCloudSigma(sources.DataSource): try: server_context = self.cepko.all().result server_meta = server_context['meta'] - except: + except Exception: # TODO: check for explicit "config on", and then warn # but since no explicit config is available now, just debug. LOG.debug("CloudSigma: Unable to read from serial port") diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 4ba019df..4de1f563 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,14 +25,15 @@ # along with this program. If not, see . import os -import time from socket import inet_ntoa from struct import pack +import time from cloudinit import ec2_utils as ec2 from cloudinit import log as logging +from cloudinit import sources from cloudinit import url_helper as uhelp -from cloudinit import sources, util +from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py index 12e863d2..44a17a00 100644 --- a/cloudinit/sources/DataSourceDigitalOcean.py +++ b/cloudinit/sources/DataSourceDigitalOcean.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import ec2_utils from cloudinit import log as logging -from cloudinit import util from cloudinit import sources -from cloudinit import ec2_utils +from cloudinit import util import functools diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 7e7fc033..9234d1f8 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -18,9 +18,9 @@ from base64 import b64decode from cloudinit import log as logging -from cloudinit import util from cloudinit import sources from cloudinit import url_helper +from cloudinit import util LOG = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class DataSourceGCE(sources.DataSource): index = public_key.index(':') if index > 0: return public_key[(index + 1):] - except: + except Exception: return public_key def get_data(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 74d0e5ec..48c61a90 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -61,7 +61,7 @@ class DataSourceNoCloud(sources.DataSource): if parse_cmdline_data(self.cmdline_id, md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) - except: + except Exception: util.logexc(LOG, "Unable to parse command line data") return False @@ -288,7 +288,7 @@ def parse_cmdline_data(ds_id, fill, cmdline=None): continue try: (k, v) = item.split("=", 1) - except: + except Exception: k = item v = None if k in s2l: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 2a6cd050..f2bb9366 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -30,13 +30,20 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from .helpers.vmware.imc.config import Config -from .helpers.vmware.imc.config_file import ConfigFile -from .helpers.vmware.imc.config_nic import NicConfigurator -from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum -from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum -from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum -from .helpers.vmware.imc.guestcust_util import ( + +from cloudinit.sources.helpers.vmware.imc.config \ + import Config +from cloudinit.sources.helpers.vmware.imc.config_file \ + import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic \ + import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_event \ + import GuestCustEventEnum +from cloudinit.sources.helpers.vmware.imc.guestcust_state \ + import GuestCustStateEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum +from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( set_customization_status, get_nics_to_enable, enable_nics @@ -262,7 +269,7 @@ def read_ovf_environment(contents): elif prop == "user-data": try: ud = base64.decodestring(val) - except: + except Exception: ud = val return (md, ud, cfg) @@ -277,7 +284,7 @@ def get_ovf_env(dirname): try: contents = util.load_file(full_fn) return (fname, contents) - except: + except Exception: util.logexc(LOG, "Failed loading ovf file %s", full_fn) return (None, False) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 6bf2c33b..43e4fd57 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -45,10 +45,9 @@ class DataSourceNotFoundException(Exception): pass +@six.add_metaclass(abc.ABCMeta) class DataSource(object): - __metaclass__ = abc.ABCMeta - def __init__(self, sys_cfg, distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 018cac6d..bbfc283c 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -5,6 +5,7 @@ import socket import struct import tempfile import time + from contextlib import contextmanager from xml.etree import ElementTree diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 1aa6bbae..156aba6c 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -145,8 +145,8 @@ class SourceMixin(object): return device +@six.add_metaclass(abc.ABCMeta) class BaseReader(object): - __metaclass__ = abc.ABCMeta def __init__(self, base_path): self.base_path = base_path @@ -475,7 +475,7 @@ class MetadataReader(BaseReader): def convert_vendordata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). + """data: a loaded json *object* (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py index faba5887..204fbcc4 100644 --- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py +++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class BootProtoEnum: +class BootProtoEnum(object): """Specifies the NIC Boot Settings.""" DHCP = 'dhcp' diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index aebc12a0..1dcd053a 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -20,7 +20,7 @@ from .nic import Nic -class Config: +class Config(object): """ Stores the Contents specified in the Customization Specification file. diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 77098a05..511cc918 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -26,7 +26,7 @@ from cloudinit import util logger = logging.getLogger(__name__) -class NicConfigurator: +class NicConfigurator(object): def __init__(self, nics): """ Initialize the Nic Configurator diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py index a367e476..8a650871 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_source.py +++ b/cloudinit/sources/helpers/vmware/imc/config_source.py @@ -18,6 +18,6 @@ # along with this program. If not, see . -class ConfigSource: +class ConfigSource(object): """Specifies a source for the Config Content.""" pass diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 1b04161f..750be1e3 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustErrorEnum: +class GuestCustErrorEnum(object): """Specifies different errors of Guest Customization engine""" GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py index fc22568f..e13b791d 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustEventEnum: +class GuestCustEventEnum(object): """Specifies different types of Guest Customization Events""" GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py index f255be5f..b9ddf513 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class GuestCustStateEnum: +class GuestCustStateEnum(object): """Specifies different states of Guest Customization engine""" GUESTCUST_STATE_RUNNING = 4 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index d39f0a65..020ab613 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -23,8 +23,8 @@ import time from cloudinit import util -from .guestcust_state import GuestCustStateEnum from .guestcust_event import GuestCustEventEnum +from .guestcust_state import GuestCustStateEnum logger = logging.getLogger(__name__) diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py index 33f88726..873ddc3b 100644 --- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py +++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class Ipv4ModeEnum: +class Ipv4ModeEnum(object): """ The IPv4 configuration mode which directly represents the user's goal. diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py index 030ba311..3c892db0 100644 --- a/cloudinit/sources/helpers/vmware/imc/nic_base.py +++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py @@ -18,7 +18,7 @@ # along with this program. If not, see . -class NicBase: +class NicBase(object): """ Define what are expected of each nic. The following properties should be provided in an implementation class. @@ -93,7 +93,7 @@ class NicBase: raise NotImplementedError('Check constraints on properties') -class StaticIpv4Base: +class StaticIpv4Base(object): """ Define what are expected of a static IPv4 setting The following properties should be provided in an implementation class. @@ -124,7 +124,7 @@ class StaticIpv4Base: raise NotImplementedError('Ipv4 GATEWAY') -class StaticIpv6Base: +class StaticIpv6Base(object): """Define what are expected of a static IPv6 setting The following properties should be provided in an implementation class. """ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ffb15165..62d066de 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -44,10 +44,10 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import net +from cloudinit.reporting import events from cloudinit import sources from cloudinit import type_utils from cloudinit import util -from cloudinit.reporting import events LOG = logging.getLogger(__name__) @@ -483,7 +483,7 @@ class Init(object): c_handlers.initialized.remove(mod) try: handlers.call_end(mod, data, frequency) - except: + except Exception: util.logexc(LOG, "Failed to finalize handler: %s", mod) try: diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 936f7da5..c05e9d90 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -28,8 +28,9 @@ import time from email.utils import parsedate from functools import partial -from requests import exceptions + import oauthlib.oauth1 as oauth1 +from requests import exceptions from six.moves.urllib.parse import ( urlparse, urlunparse, @@ -61,7 +62,7 @@ try: SSL_ENABLED = True if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'): CONFIG_ENABLED = True -except: +except ImportError: pass diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index f7c5787c..f0631906 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -178,7 +178,7 @@ class UserDataProcessor(object): payload = util.load_yaml(msg.get_payload(decode=True)) if payload: payload_idx = payload.get('launch-index') - except: + except Exception: pass # Header overrides contents, for now (?) or the other way around? if header_idx is not None: diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..8d6cbb4b 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -288,7 +288,7 @@ def fork_cb(child_cb, *args, **kwargs): try: child_cb(*args, **kwargs) os._exit(0) - except: + except Exception: logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) @@ -472,7 +472,7 @@ def is_ipv4(instr): try: toks = [x for x in toks if int(x) < 256 and int(x) >= 0] - except: + except Exception: return False return len(toks) == 4 @@ -1210,7 +1210,7 @@ def get_cmdline(): else: try: cmdline = load_file("/proc/cmdline").strip() - except: + except Exception: cmdline = "" PROC_CMDLINE = cmdline @@ -1380,7 +1380,7 @@ def read_write_cmdline_url(target_fn): if not os.path.exists(target_fn): try: (key, url, content) = get_cmdline_url() - except: + except Exception: logexc(LOG, "Failed fetching command line url") return try: @@ -1391,7 +1391,7 @@ def read_write_cmdline_url(target_fn): elif key and not content: LOG.debug(("Command line key %s with url" " %s had no contents"), key, url) - except: + except Exception: logexc(LOG, "Failed writing url content to %s", target_fn) @@ -1449,7 +1449,7 @@ def mounts(): mp = m.group(2) fstype = m.group(3) opts = m.group(4) - except: + except Exception: continue # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. @@ -1575,7 +1575,7 @@ def copy(src, dest): def time_rfc2822(): try: ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime()) - except: + except Exception: ts = "??" return ts @@ -1601,7 +1601,7 @@ def uptime(): bootup = buf.value uptime_str = now - bootup - except: + except Exception: logexc(LOG, "Unable to read uptime using method: %s" % method) return uptime_str @@ -2055,7 +2055,7 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): tmsg += " (N/A)" try: logfunc(msg + tmsg) - except: + except Exception: pass return ret diff --git a/test-requirements.txt b/test-requirements.txt index 9b3d07c5..170856ad 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,8 @@ httpretty>=0.7.1 mock nose -pep8==1.5.7 +pep8>=1.5.7 pyflakes contextlib2 setuptools +flake8>=2.5.4 diff --git a/tox.ini b/tox.ini index bd7c27dd..7a144efe 100644 --- a/tox.ini +++ b/tox.ini @@ -30,3 +30,8 @@ deps = pyflakes setenv = LC_ALL = C + +[flake8] + +ignore=H404,H405,H105,H301,H104,H403,H101 +exclude = .venv,.tox,dist,doc,*egg,.git,build,tools,tests -- cgit v1.2.3 From 53834934e4c520b2fb8b5acffca641213ddd688a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:29:33 +0200 Subject: fix EXPORT_GPG_KEYID for long key fingerprints --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 492c3c00..28f20939 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -43,7 +43,7 @@ EXPORT_GPG_KEYID = """ [ -n "$k" ] || exit 1; armour=$(gpg --list-keys --armour "${k}") if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && + gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && gpg --batch --yes --delete-keys "${k}" fi -- cgit v1.2.3 From 2202494b72cae19cbf9d34a8f3176d7021becb13 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:53:12 +0200 Subject: split add_key and add_key_raw fior better testability --- cloudinit/config/cc_apt_configure.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 28f20939..e7b8a9b3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,10 +171,19 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key_raw(key): + """ + actual adding of a key as defined in key argument + to the system + """ + try: + util.subp(('apt-key', 'add', '-'), key) + except: + raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in entry (if any) + add key to the system as defiend in ent (if any) suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ @@ -188,10 +197,7 @@ def add_key(ent): raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - raise Exception('failed add key') + add_key_raw(ent['key']) def add_sources(srclist, template_params=None, aa_repo_match=None): -- cgit v1.2.3 From 454de24c7d457b980c91849b128efe4faee62032 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:21:36 +0200 Subject: make pep8 happy with a few spaces --- cloudinit/config/cc_apt_configure.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 1 + 2 files changed, 3 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e7b8a9b3..e5a962ac 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,6 +171,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + def add_key_raw(key): """ actual adding of a key as defined in key argument @@ -181,6 +182,7 @@ def add_key_raw(key): except: raise Exception('failed add key') + def add_key(ent): """ add key to the system as defiend in ent (if any) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 439bd038..e130392c 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -30,6 +30,7 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== =ACB2 -----END PGP PUBLIC KEY BLOCK-----""" + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding -- cgit v1.2.3 From ee239517c5342cbd62c9fdeaf735d78d6fd1fbb8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 11:59:14 +0200 Subject: support apt_sources to be a dictionary key is the filename, and "old" input shall be handled as it was all the time. For compatibility this will (continue to) overwrite the file of multiple options that did not specify an output file (they all get the same default). Yet it will process them all - as it always did - e.g. to add the keys of all of them. Any users of the new format won't have these issues, as they will always have a key. --- cloudinit/config/cc_apt_configure.py | 34 +++++++++++++++++++++++++--------- cloudinit/util.py | 8 ++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e5a962ac..a46ebb3e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,8 +215,28 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + # convert old list format to new dict based format + if isinstance(srclist, list): + srcdict = {} + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + else: + srcdict = srclist + errorlist = [] - for ent in srclist: + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent[filename] = filename + # keys can be added without specifying a source try: add_key(ent) @@ -226,10 +246,13 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -238,13 +261,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..2931efbd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,14 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey def read_conf(fname): try: -- cgit v1.2.3 From a33f8c09863381006f708a1e9d49997ed9f7befa Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 12:12:26 +0200 Subject: warn about multiple colliding apt_source without filenames --- cloudinit/config/cc_apt_configure.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a46ebb3e..327b543e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,15 +215,22 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + errorlist = [] # convert old list format to new dict based format if isinstance(srclist, list): srcdict = {} + fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + if fnfallbackused is not None: + errorlist.append(["multiple apt_source entries without", + "filename will conflict: %s vs %s" % + (srcent, fnfallbackused)]) + fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -231,7 +238,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: srcdict = srclist - errorlist = [] for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: -- cgit v1.2.3 From 4ed5251d17ee7a44ce12d38d9b3d4fa554279419 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:06:48 +0200 Subject: fix issue with dictionary style apt_sources handling filenames --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 327b543e..a25d6af1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -241,7 +241,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: - ent[filename] = filename + ent['filename'] = filename # keys can be added without specifying a source try: -- cgit v1.2.3 From 2945e028477ddb031d9a51ada16d5b992380242a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:12:18 +0200 Subject: make sure we only handle list or dict apt_sources and bail out for others --- cloudinit/config/cc_apt_configure.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a25d6af1..dd199471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -217,8 +217,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] # convert old list format to new dict based format + srcdict = {} if isinstance(srclist, list): - srcdict = {} fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: @@ -235,8 +235,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # all with filename use that as key (matching new format) key = srcent['filename'] srcdict[key] = srcent - else: + elif isinstance(srclist, dict): srcdict = srclist + else: + errorlist.append(["srclist", "unknown apt_sources format"]) for filename in srcdict: ent = srcdict[filename] -- cgit v1.2.3 From 2a20382249c35f368f931dc2bcd8225ea1fc4e84 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 18:55:35 +0200 Subject: integrate further smaller review feedback --- cloudinit/config/cc_apt_configure.py | 52 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 25 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 8f22c446..02b16336 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,7 +21,6 @@ import glob import os import re -import tempfile from cloudinit import templater from cloudinit import util @@ -179,44 +178,29 @@ def add_key_raw(key): """ try: util.subp(('apt-key', 'add', '-'), key) - except: + except util.ProcessExecutionError: raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in ent (if any) - suppords raw keys or keyid's - The latter will as a first step fetched to get the raw key + add key to the system as defined in ent (if any) + supports raw keys or keyid's + The latter will as a first step fetch the raw key from a keyserver """ if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], keyserver) - except: - raise Exception('failed to get key from %s' % keyserver) + ent['key'] = getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_key_raw(ent['key']) - -def add_sources(srclist, template_params=None, aa_repo_match=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList +def convert_to_new_format(srclist, errorlist): + """ convert_to_new_format + convert the old list based format to the new dict based one """ - if template_params is None: - template_params = {} - - if aa_repo_match is None: - def aa_repo_match(x): - return False - - errorlist = [] - # convert old list format to new dict based format srcdict = {} if isinstance(srclist, list): fnfallbackused = None @@ -240,6 +224,24 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: errorlist.append(["srclist", "unknown apt_sources format"]) + return srcdict + +def add_sources(srclist, template_params=None, aa_repo_match=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + if aa_repo_match is None: + def aa_repo_match(x): + return False + + errorlist = [] + srcdict = convert_to_new_format(srclist, errorlist) + for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: @@ -257,7 +259,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): source = ent['source'] source = templater.render_string(source, template_params) - if not ent['filename'].startswith("/"): + if not ent['filename'].startswith(os.path.sep): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) -- cgit v1.2.3 From c2cfbd6831912e6dfd6b16ed21afb929592ce51a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 19:06:55 +0200 Subject: pacify pep8 regarding the new changes --- cloudinit/config/cc_apt_configure.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 02b16336..ffbf7513 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -197,6 +197,7 @@ def add_key(ent): if 'key' in ent: add_key_raw(ent['key']) + def convert_to_new_format(srclist, errorlist): """ convert_to_new_format convert the old list based format to the new dict based one @@ -226,6 +227,7 @@ def convert_to_new_format(srclist, errorlist): return srcdict + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated -- cgit v1.2.3 From aa236033b159b691f5ec31885750a8167c63b2a1 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:27:40 +0200 Subject: drop errorlist from convert_to_new_format --- cloudinit/config/cc_apt_configure.py | 12 +++--------- tests/unittests/test_handler/test_handler_apt_source.py | 6 ++---- 2 files changed, 5 insertions(+), 13 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ffbf7513..2dd48844 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -198,24 +198,18 @@ def add_key(ent): add_key_raw(ent['key']) -def convert_to_new_format(srclist, errorlist): +def convert_to_new_format(srclist): """ convert_to_new_format convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): - fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - if fnfallbackused is not None: - errorlist.append(["multiple apt_source entries without", - "filename will conflict: %s vs %s" % - (srcent, fnfallbackused)]) - fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -223,7 +217,7 @@ def convert_to_new_format(srclist, errorlist): elif isinstance(srclist, dict): srcdict = srclist else: - errorlist.append(["srclist", "unknown apt_sources format"]) + raise ValueError("unknown apt_sources format") return srcdict @@ -242,7 +236,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): return False errorlist = [] - srcdict = convert_to_new_format(srclist, errorlist) + srcdict = convert_to_new_format(srclist) for filename in srcdict: ent = srcdict[filename] diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 753e86a8..4536c5b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -529,7 +529,6 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile2} cfg3 = {'source': 'deb $MIRROR $RELEASE universe', 'filename': self.aptlistfile3} - errorlist = [] checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, 'source': 'deb $MIRROR $RELEASE ' 'multiverse'}, @@ -539,11 +538,10 @@ class TestAptSourceConfig(TestCase): 'source': 'deb $MIRROR $RELEASE ' 'universe'}} - newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3], - errorlist) + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) self.assertEqual(newcfg, checkcfg) - newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist) + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) self.assertEqual(newcfg2, checkcfg) -- cgit v1.2.3 From 3c85315373306729443ef79fd8e54af46a7bc849 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 13:07:22 +0200 Subject: fix EXPORT_GPG_KEYID for existing keys This was broken for keys already existing in the local keyring. There instead of the keycontent it reported the header like: pub 1024R/03683F77 2009-10-27 uid Launchpad PPA for Scott Moser --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2dd48844..d603f417 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -40,7 +40,7 @@ EXPORT_GPG_KEYID = """ k=${1} ks=${2}; exec 2>/dev/null [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") + armour=$(gpg --export --armour "${k}") if [ -z "${armour}" ]; then gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && -- cgit v1.2.3 From 7c27cb1fcae01d34b32895630d57b2c9bc624caf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 09:47:29 -0400 Subject: fix log message in emit_upstart --- cloudinit/config/cc_emit_upstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 06c53272..98828b9e 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -56,7 +56,7 @@ def handle(name, _cfg, cloud, log, args): event_names = ['cloud-config'] if not is_upstart_system(): - log.debug("not upstart system, '%s' disabled") + log.debug("not upstart system, '%s' disabled", name) return cfgpath = cloud.paths.get_ipath_cur("cloud_config") -- cgit v1.2.3 From e2c2d70cde19211b18e5ec333e1cb0382d93f14d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 00:42:22 -0400 Subject: lxd: fix log messsage --- cloudinit/config/cc_lxd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index b1de8f84..70d4e7c3 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -52,7 +52,8 @@ def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') if not lxd_cfg: - log.debug("Skipping module named %s, not present or disabled by cfg") + log.debug("Skipping module named %s, not present or disabled by cfg", + name) return if not isinstance(lxd_cfg, dict): log.warn("lxd config must be a dictionary. found a '%s'", -- cgit v1.2.3 From 80931f7008971c9a7705c054fabc29fec7a133e2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:27:32 -0400 Subject: fix tox -e flake8 --- cloudinit/config/cc_apt_configure.py | 4 +- .../test_handler_apt_configure_sources_list.py | 17 +++--- .../test_handler/test_handler_apt_source.py | 66 +++++++++++----------- 3 files changed, 43 insertions(+), 44 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index d603f417..7a9777c0 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -199,8 +199,8 @@ def add_key(ent): def convert_to_new_format(srclist): - """ convert_to_new_format - convert the old list based format to the new dict based one + """convert_to_new_format + convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 5cf386f8..5d0417a2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -3,7 +3,6 @@ Test templating of sources list """ import logging import os -import re import shutil import tempfile @@ -62,14 +61,14 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): - """ TestAptSourceConfigSourceList + """TestAptSourceConfigSourceList Main Class to test sources list rendering """ def setUp(self): @@ -89,7 +88,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): return cloud.Cloud(myds, paths, {}, mydist, None) def apt_source_list(self, distro, mirror, mirrorcheck=None): - """ apt_source_list + """apt_source_list Test rendering of a source.list from template for a given distro """ if mirrorcheck is None: @@ -115,19 +114,19 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) def test_apt_source_list_debian(self): - """ test_apt_source_list_debian + """test_apt_source_list_debian Test rendering of a source.list from template for debian """ self.apt_source_list('debian', 'http://httpredir.debian.org/debian') def test_apt_source_list_ubuntu(self): - """ test_apt_source_list_ubuntu + """test_apt_source_list_ubuntu Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_debian_mirrorfail(self): - """ test_apt_source_list_debian_mirrorfail + """test_apt_source_list_debian_mirrorfail Test rendering of a source.list from template for debian """ self.apt_source_list('debian', ['http://does.not.exist', @@ -135,7 +134,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://httpredir.debian.org/debian') def test_apt_srcl_ubuntu_mirrorfail(self): - """ test_apt_source_list_ubuntu_mirrorfail + """test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', ['http://does.not.exist', @@ -143,7 +142,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_custom(self): - """ test_apt_srcl_custom + """test_apt_srcl_custom Test rendering from a custom source.list template """ cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index fe2ffae5..4dbe69f0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -33,14 +33,14 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfig(TestCase): - """ TestAptSourceConfig + """TestAptSourceConfig Main Class to test apt_source configs """ def setUp(self): @@ -57,7 +57,7 @@ class TestAptSourceConfig(TestCase): @staticmethod def _get_default_params(): - """ get_default_params + """get_default_params Get the most basic default mrror and release info to be used in tests """ params = {} @@ -66,7 +66,7 @@ class TestAptSourceConfig(TestCase): return params def myjoin(self, *args, **kwargs): - """ myjoin - redir into writable tmpdir""" + """myjoin - redir into writable tmpdir""" if (args[0] == "/etc/apt/sources.list.d/" and args[1] == "cloud_config_sources.list" and len(args) == 2): @@ -75,7 +75,7 @@ class TestAptSourceConfig(TestCase): return self.join(*args, **kwargs) def apt_src_basic(self, filename, cfg): - """ apt_src_basic + """apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() @@ -92,7 +92,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic(self): - """ test_apt_src_basic + """test_apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. """ @@ -103,7 +103,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, [cfg]) def test_apt_src_basic_dict(self): - """ test_apt_src_basic_dict + """test_apt_src_basic_dict Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. Provided in a dictionary with filename being the key (new format) @@ -115,7 +115,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, cfg) def apt_src_basic_tri(self, cfg): - """ apt_src_basic_tri + """apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. generic part to check three files with different content @@ -137,7 +137,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic_tri(self): - """ test_apt_src_basic_tri + """test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. """ @@ -156,7 +156,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri([cfg1, cfg2, cfg3]) def test_apt_src_basic_dict_tri(self): - """ test_apt_src_basic_dict_tri + """test_apt_src_basic_dict_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. Provided in a dictionary with filename being the key (new format) @@ -176,7 +176,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri(cfg) def test_apt_src_basic_nofn(self): - """ test_apt_src_basic_nofn + """test_apt_src_basic_nofn Test Fix deb source string, has to overwrite mirror conf in params. Test without a filename provided in config and test for known fallback. """ @@ -187,7 +187,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.fallbackfn, [cfg]) def apt_src_replacement(self, filename, cfg): - """ apt_src_replace + """apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -202,7 +202,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace(self): - """ test_apt_src_replace + """test_apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs with Filename being set """ @@ -211,7 +211,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.aptlistfile, [cfg]) def apt_src_replace_tri(self, cfg): - """ apt_src_replace_tri + """apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with generic part """ @@ -231,7 +231,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace_tri(self): - """ test_apt_src_replace_tri + """test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set """ @@ -244,7 +244,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replace_tri([cfg1, cfg2, cfg3]) def test_apt_src_replace_dict_tri(self): - """ test_apt_src_replace_dict_tri + """test_apt_src_replace_dict_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set Provided in a dictionary with filename being the key (new format) @@ -252,13 +252,13 @@ class TestAptSourceConfig(TestCase): filenames to be overwritten inside the directory entry. """ cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, - 'notused': {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} self.apt_src_replace_tri(cfg) def test_apt_src_replace_nofn(self): - """ test_apt_src_replace_nofn + """test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with No filename being set """ @@ -267,7 +267,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.fallbackfn, [cfg]) def apt_src_keyid(self, filename, cfg, keynum): - """ apt_src_keyid + """apt_src_keyid Test specification of a source + keyid """ params = self._get_default_params() @@ -293,7 +293,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid(self): - """ test_apt_src_keyid + """test_apt_src_keyid Test specification of a source + keyid with filename being set """ cfg = {'source': ('deb ' @@ -305,7 +305,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.aptlistfile, [cfg], 1) def test_apt_src_keyid_tri(self): - """ test_apt_src_keyid_tri + """test_apt_src_keyid_tri Test specification of a source + keyid with filename being set Setting three of such, check for content and keys """ @@ -345,7 +345,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid_nofn(self): - """ test_apt_src_keyid_nofn + """test_apt_src_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -357,7 +357,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.fallbackfn, [cfg], 1) def apt_src_key(self, filename, cfg): - """ apt_src_key + """apt_src_key Test specification of a source + key """ params = self._get_default_params() @@ -378,7 +378,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_key(self): - """ test_apt_src_key + """test_apt_src_key Test specification of a source + key with filename being set """ cfg = {'source': ('deb ' @@ -390,7 +390,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.aptlistfile, cfg) def test_apt_src_key_nofn(self): - """ test_apt_src_key_nofn + """test_apt_src_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' @@ -402,7 +402,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.fallbackfn, cfg) def test_apt_src_keyonly(self): - """ test_apt_src_keyonly + """test_apt_src_keyonly Test specification key without source """ params = self._get_default_params() @@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyidonly(self): - """ test_apt_src_keyidonly + """test_apt_src_keyidonly Test specification of a keyid without source """ params = self._get_default_params() @@ -436,7 +436,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyid_real(self): - """ test_apt_src_keyid_real + """test_apt_src_keyid_real Test specification of a keyid without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_longkeyid_real(self): - """ test_apt_src_longkeyid_real + """test_apt_src_longkeyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -472,7 +472,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa(self): - """ test_apt_src_ppa + """test_apt_src_ppa Test specification of a ppa """ params = self._get_default_params() @@ -491,7 +491,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa_tri(self): - """ test_apt_src_ppa_tri + """test_apt_src_ppa_tri Test specification of a ppa """ params = self._get_default_params() @@ -519,7 +519,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile3)) def test_convert_to_new_format(self): - """ test_convert_to_new_format + """test_convert_to_new_format Test the conversion of old to new format And the noop conversion of new to new format as well """ -- cgit v1.2.3 From dca09871e61186deb216ce5cb1fc5db3b69c9fc2 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 09:18:17 +0200 Subject: pythonify getkeybyid and move it to uitl.py --- cloudinit/config/cc_apt_configure.py | 27 +------------------------ cloudinit/util.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 26 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 7a9777c0..2f270662 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -34,21 +34,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" -# A temporary shell program to get a given gpg key -# from a given keyserver -EXPORT_GPG_KEYID = """ - k=${1} ks=${2}; - exec 2>/dev/null - [ -n "$k" ] || exit 1; - armour=$(gpg --export --armour "${k}") - if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv "${k}" >/dev/null && - armour=$(gpg --export --armour "${k}") && - gpg --batch --yes --delete-keys "${k}" - fi - [ -n "${armour}" ] && echo "${armour}" -""" - def handle(name, cfg, cloud, log, _args): if util.is_false(cfg.get('apt_configure_enabled', True)): @@ -108,16 +93,6 @@ def handle(name, cfg, cloud, log, _args): util.logexc(log, "Failed to run debconf-set-selections") -# get gpg keyid from keyserver -def getkeybyid(keyid, keyserver): - with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh: - fh.write(EXPORT_GPG_KEYID) - fh.flush() - cmd = ['/bin/sh', fh.name, keyid, keyserver] - (stdout, _stderr) = util.subp(cmd) - return stdout.strip() - - def mirror2lists_fileprefix(mirror): string = mirror # take off http:// or ftp:// @@ -192,7 +167,7 @@ def add_key(ent): keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = getkeybyid(ent['keyid'], keyserver) + ent['key'] = util.getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_key_raw(ent['key']) diff --git a/cloudinit/util.py b/cloudinit/util.py index d6b80dbe..d3b14f72 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2234,3 +2234,41 @@ def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(six.StringIO(string)) return email.message_from_string(string) + + +def gpg_export_armour(key): + """Export gpg key, armoured key gets returned""" + (armour, _) = subp(["gpg", "--export", "--armour", key], capture=True) + return armour + + +def gpg_recv_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + try: + subp(["gpg", "--keyserver", keyserver, "--recv", key], + capture=True) + except ProcessExecutionError as error: + raise ValueError('Failed to import key %s from server %s - error %s' % + (key, keyserver, error)) + + +def gpg_delete_key(key): + """Delete the specified key from the local gpg ring""" + subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=False) + + +def getkeybyid(keyid, keyserver): + """get gpg keyid from keyserver""" + armour = gpg_export_armour(keyid) + if not armour: + try: + gpg_recv_key(keyid, keyserver=keyserver) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + + armour = gpg_export_armour(keyid) + # delete just imported key to leave environment as it was before + gpg_delete_key(keyid) + + return armour.rstrip('\n') -- cgit v1.2.3 From 89688593e9358433cd9383d1bdfae12dbcd58f72 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 13:36:23 +0200 Subject: add_key_raw - make exceptions more specific --- cloudinit/config/cc_apt_configure.py | 4 +-- cloudinit/util.py | 1 + .../test_handler/test_handler_apt_source.py | 38 +++++++++++++++++++--- 3 files changed, 36 insertions(+), 7 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2f270662..a9ac6ea8 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -154,7 +154,7 @@ def add_key_raw(key): try: util.subp(('apt-key', 'add', '-'), key) except util.ProcessExecutionError: - raise Exception('failed add key') + raise ValueError('failed to add apt GPG Key to apt keyring') def add_key(ent): @@ -221,7 +221,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # keys can be added without specifying a source try: add_key(ent) - except Exception as detail: + except ValueError as detail: errorlist.append([ent, detail]) if 'source' not in ent: diff --git a/cloudinit/util.py b/cloudinit/util.py index d3b14f72..a1622946 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2244,6 +2244,7 @@ def gpg_export_armour(key): def gpg_recv_key(key, keyserver): """Receive gpg key from the specified keyserver""" + print("ORIGINAL gpg_recv_key") try: subp(["gpg", "--keyserver", keyserver, "--recv", key], capture=True) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a4878124..65e375a0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -4,6 +4,7 @@ Testing various config variations of the apt_source config import os import re import shutil +import socket import tempfile try: @@ -56,6 +57,7 @@ class TestAptSourceConfig(TestCase): # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") + self.orig_gpg_recv_key = util.gpg_recv_key patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") get_rel = patcher.start() @@ -445,14 +447,40 @@ class TestAptSourceConfig(TestCase): def apt_src_keyid_real(self, cfg, expectedkey): """apt_src_keyid_real Test specification of a keyid without source including - up to addition of the key (nothing but add_key_raw mocked to keep the + up to addition of the key (add_key_raw mocked to keep the environment as is) """ params = self._get_default_params() - with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: - cc_apt_configure.add_sources([cfg], params) - mockobj.assert_called_with(expectedkey) + def fake_gpg_recv_key(self, key, keyserver): + """try original gpg_recv_key, but allow fall back""" + try: + print("Try orig orig_gpg_recv_key") + self.orig_gpg_recv_key(key, keyserver) + except ValueError: + print("Fail, test net") + # if this is a networking issue mock it's effect + testsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + testsock.connect((keyserver, 80)) + testsock.close() + except socket.error: + # as fallback add the known key as a working recv would + print("Fallback import expectedkey") + util.subp(("gpg", "--import", "-"), EXPECTEDKEY) + + print("FOO") + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey: + with mock.patch.object(util, 'gpg_recv_key', + side_effect=fake_gpg_recv_key) as mockrecv: + cc_apt_configure.add_sources([cfg], params) + + # since we might mock the recv path ensure it is called right + mockrecv.assert_called_with(cfg['keyid'], + keyserver=cfg.get('keyserver', + 'keyserver.ubuntu.com')) + # no matter if really imported or faked, ensure we add the right key + mockkey.assert_called_with(expectedkey) # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) @@ -477,7 +505,7 @@ class TestAptSourceConfig(TestCase): """test_apt_src_longkeyid_ks_real - Test long keyid from other ks""" keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" cfg = {'keyid': keyid, - 'keyserver': 'keys.gnupg.net', + 'keyserver': 'knorz.gnupg.net', 'filename': self.aptlistfile} self.apt_src_keyid_real(cfg, EXPECTEDKEY) -- cgit v1.2.3 From c1ad09609805c6c4f5262eb21533799af814379a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 14:23:55 +0200 Subject: rename add_key / add_source to add_apt_key / add_apt_source The functions clearly are apt specific so the name should reflect that. --- cloudinit/config/cc_apt_configure.py | 14 ++++++------ .../test_handler/test_handler_apt_source.py | 25 +++++++++++----------- 2 files changed, 20 insertions(+), 19 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a9ac6ea8..91a117f5 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -79,8 +79,8 @@ def handle(name, cfg, cloud, log, _args): def matcher(x): return False - errors = add_sources(cfg['apt_sources'], params, - aa_repo_match=matcher) + errors = add_apt_sources(cfg['apt_sources'], params, + aa_repo_match=matcher) for e in errors: log.warn("Add source error: %s", ':'.join(e)) @@ -146,7 +146,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_key_raw(key): +def add_apt_key_raw(key): """ actual adding of a key as defined in key argument to the system @@ -157,7 +157,7 @@ def add_key_raw(key): raise ValueError('failed to add apt GPG Key to apt keyring') -def add_key(ent): +def add_apt_key(ent): """ add key to the system as defined in ent (if any) supports raw keys or keyid's @@ -170,7 +170,7 @@ def add_key(ent): ent['key'] = util.getkeybyid(ent['keyid'], keyserver) if 'key' in ent: - add_key_raw(ent['key']) + add_apt_key_raw(ent['key']) def convert_to_new_format(srclist): @@ -197,7 +197,7 @@ def convert_to_new_format(srclist): return srcdict -def add_sources(srclist, template_params=None, aa_repo_match=None): +def add_apt_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated sources.list entry in 'srclist'. When rendering template, also @@ -220,7 +220,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # keys can be added without specifying a source try: - add_key(ent) + add_apt_key(ent) except ValueError as detail: errorlist.append([ent, detail]) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index c7eeb64c..4a720213 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -89,7 +89,7 @@ class TestAptSourceConfig(TestCase): """ params = self._get_default_params() - cc_apt_configure.add_sources(cfg, params) + cc_apt_configure.add_apt_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -200,7 +200,7 @@ class TestAptSourceConfig(TestCase): Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() - cc_apt_configure.add_sources(cfg, params) + cc_apt_configure.add_apt_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -283,7 +283,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: - cc_apt_configure.add_sources(cfg, params) + cc_apt_configure.add_apt_sources(cfg, params) # check if it added the right ammount of keys calls = [] @@ -372,7 +372,7 @@ class TestAptSourceConfig(TestCase): params = self._get_default_params() with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') @@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) mockobj.assert_called_once_with(('apt-key', 'add', '-'), 'fakekey 4242') @@ -437,7 +437,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp', return_value=('fakekey 1212', '')) as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') @@ -447,7 +447,7 @@ class TestAptSourceConfig(TestCase): def apt_src_keyid_real(self, cfg, expectedkey): """apt_src_keyid_real Test specification of a keyid without source including - up to addition of the key (add_key_raw mocked to keep the + up to addition of the key (add_apt_key_raw mocked to keep the environment as is) """ params = self._get_default_params() @@ -466,10 +466,10 @@ class TestAptSourceConfig(TestCase): # as fallback add the known key as a working recv would util.subp(("gpg", "--import", "-"), EXPECTEDKEY) - with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockkey: + with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: with mock.patch.object(util, 'gpg_recv_key', side_effect=fake_gpg_recv_key) as mockrecv: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_apt_sources([cfg], params) # since we might mock the recv path ensure it is called right mockrecv.assert_called_with(cfg['keyid'], @@ -518,7 +518,8 @@ class TestAptSourceConfig(TestCase): matcher = re.compile(r'^[\w-]+:\w').search with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + cc_apt_configure.add_apt_sources([cfg], params, + aa_repo_match=matcher) mockobj.assert_called_once_with(['add-apt-repository', 'ppa:smoser/cloud-init-test']) @@ -541,8 +542,8 @@ class TestAptSourceConfig(TestCase): matcher = re.compile(r'^[\w-]+:\w').search with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_sources([cfg1, cfg2, cfg3], params, - aa_repo_match=matcher) + cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params, + aa_repo_match=matcher) calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] -- cgit v1.2.3 From 71ea8d200da164df6f5e28d28aea424b57fd8f1f Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 6 Jun 2016 15:18:59 +0200 Subject: rename fucntion to mirrorurl_to_apt_fileprefix to reflect what it actually does --- cloudinit/config/cc_apt_configure.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 91a117f5..be6324a2 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -93,7 +93,7 @@ def handle(name, cfg, cloud, log, _args): util.logexc(log, "Failed to run debconf-set-selections") -def mirror2lists_fileprefix(mirror): +def mirrorurl_to_apt_fileprefix(mirror): string = mirror # take off http:// or ftp:// if string.endswith("/"): @@ -110,8 +110,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): nmirror = new_mirrors.get(name) if not nmirror: continue - oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror)) + oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror)) + nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror)) if oprefix == nprefix: continue olen = len(oprefix) -- cgit v1.2.3 From b89ac6b7caeeade5ad21137773ac4496cdaea2c5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 9 Jun 2016 09:18:35 +0200 Subject: move gpg functions into gpg.py This helps for cleaner code structuring. ALong that makeing sure all these functions have a gpg_prefix. --- cloudinit/config/cc_apt_configure.py | 3 +- cloudinit/gpg.py | 64 ++++++++++++++++++++++ cloudinit/util.py | 38 ------------- .../test_handler/test_handler_apt_source.py | 4 +- 4 files changed, 68 insertions(+), 41 deletions(-) create mode 100644 cloudinit/gpg.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index be6324a2..ba080930 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -24,6 +24,7 @@ import re from cloudinit import templater from cloudinit import util +from cloudinit import gpg distros = ['ubuntu', 'debian'] @@ -167,7 +168,7 @@ def add_apt_key(ent): keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = util.getkeybyid(ent['keyid'], keyserver) + ent['key'] = gpg.gpg_getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_apt_key_raw(ent['key']) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py new file mode 100644 index 00000000..620dfb19 --- /dev/null +++ b/cloudinit/gpg.py @@ -0,0 +1,64 @@ +"""gpg.py - Collection of gpg key related functions""" +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# Author: Christian Ehrhardt +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit import util +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + + +def gpg_export_armour(key): + """Export gpg key, armoured key gets returned""" + (armour, _) = util.subp(["gpg", "--export", "--armour", key], capture=True) + return armour + + +def gpg_recv_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + try: + util.subp(["gpg", "--keyserver", keyserver, "--recv", key], + capture=True) + except util.ProcessExecutionError as error: + raise ValueError('Failed to import key %s from server %s - error %s' % + (key, keyserver, error)) + + +def gpg_delete_key(key): + """Delete the specified key from the local gpg ring""" + util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) + + +def gpg_getkeybyid(keyid, keyserver): + """get gpg keyid from keyserver""" + armour = gpg_export_armour(keyid) + if not armour: + try: + gpg_recv_key(keyid, keyserver=keyserver) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + + armour = gpg_export_armour(keyid) + # delete just imported key to leave environment as it was before + gpg_delete_key(keyid) + + return armour.rstrip('\n') diff --git a/cloudinit/util.py b/cloudinit/util.py index 6d16532d..d6b80dbe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2234,41 +2234,3 @@ def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(six.StringIO(string)) return email.message_from_string(string) - - -def gpg_export_armour(key): - """Export gpg key, armoured key gets returned""" - (armour, _) = subp(["gpg", "--export", "--armour", key], capture=True) - return armour - - -def gpg_recv_key(key, keyserver): - """Receive gpg key from the specified keyserver""" - try: - subp(["gpg", "--keyserver", keyserver, "--recv", key], - capture=True) - except ProcessExecutionError as error: - raise ValueError('Failed to import key %s from server %s - error %s' % - (key, keyserver, error)) - - -def gpg_delete_key(key): - """Delete the specified key from the local gpg ring""" - subp(["gpg", "--batch", "--yes", "--delete-keys", key], capture=True) - - -def getkeybyid(keyid, keyserver): - """get gpg keyid from keyserver""" - armour = gpg_export_armour(keyid) - if not armour: - try: - gpg_recv_key(keyid, keyserver=keyserver) - except ValueError: - LOG.exception('Failed to obtain gpg key %s', keyid) - raise - - armour = gpg_export_armour(keyid) - # delete just imported key to leave environment as it was before - gpg_delete_key(keyid) - - return armour.rstrip('\n') diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 3f87fad3..b09dd8ab 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -14,6 +14,7 @@ from mock import call from cloudinit.config import cc_apt_configure from cloudinit import util +from cloudinit import gpg from ..helpers import TestCase @@ -58,7 +59,6 @@ class TestAptSourceConfig(TestCase): # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - self.orig_gpg_recv_key = util.gpg_recv_key patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") get_rel = patcher.start() @@ -407,7 +407,7 @@ class TestAptSourceConfig(TestCase): params = self._get_default_params() with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: - with mock.patch.object(util, 'getkeybyid', + with mock.patch.object(gpg, 'gpg_getkeybyid', return_value=expectedkey) as mockgetkey: cc_apt_configure.add_apt_sources([cfg], params) -- cgit v1.2.3 From 7e527b1b2f3fda558fb0f3a6958c42dde4716079 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 10 Jun 2016 13:22:59 -0400 Subject: minor changes prior to merge a.) remove 'gpg_' from function names in new gpg module. b.) use --recv-keys rather than --recv --recv-keys is more obvious and works back to precise at least. c.) do not trim trailing '\n' from a armour'd key. --- cloudinit/config/cc_apt_configure.py | 4 ++-- cloudinit/gpg.py | 24 ++++++++++------------ .../test_handler/test_handler_apt_source.py | 4 ++-- 3 files changed, 15 insertions(+), 17 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ba080930..96c4a43d 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -22,9 +22,9 @@ import glob import os import re +from cloudinit import gpg from cloudinit import templater from cloudinit import util -from cloudinit import gpg distros = ['ubuntu', 'debian'] @@ -168,7 +168,7 @@ def add_apt_key(ent): keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = gpg.gpg_getkeybyid(ent['keyid'], keyserver) + ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver) if 'key' in ent: add_apt_key_raw(ent['key']) diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index baa8b534..6a76d785 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -4,8 +4,6 @@ # Copyright (C) 2016 Canonical Ltd. # # Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow # Author: Christian Ehrhardt # # This program is free software: you can redistribute it and/or modify @@ -20,13 +18,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from cloudinit import util from cloudinit import log as logging +from cloudinit import util LOG = logging.getLogger(__name__) -def gpg_export_armour(key): +def export_armour(key): """Export gpg key, armoured key gets returned""" try: (armour, _) = util.subp(["gpg", "--export", "--armour", key], @@ -38,11 +36,11 @@ def gpg_export_armour(key): return armour -def gpg_recv_key(key, keyserver): +def receive_key(key, keyserver): """Receive gpg key from the specified keyserver""" LOG.debug('Receive gpg key "%s"', key) try: - util.subp(["gpg", "--keyserver", keyserver, "--recv", key], + util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key], capture=True) except util.ProcessExecutionError as error: raise ValueError(('Failed to import key "%s" ' @@ -50,7 +48,7 @@ def gpg_recv_key(key, keyserver): (key, keyserver, error)) -def gpg_delete_key(key): +def delete_key(key): """Delete the specified key from the local gpg ring""" try: util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], @@ -59,18 +57,18 @@ def gpg_delete_key(key): LOG.warn('Failed delete key "%s": %s', key, error) -def gpg_getkeybyid(keyid, keyserver): +def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"): """get gpg keyid from keyserver""" - armour = gpg_export_armour(keyid) + armour = export_armour(keyid) if not armour: try: - gpg_recv_key(keyid, keyserver=keyserver) - armour = gpg_export_armour(keyid) + receive_key(keyid, keyserver=keyserver) + armour = export_armour(keyid) except ValueError: LOG.exception('Failed to obtain gpg key %s', keyid) raise finally: # delete just imported key to leave environment as it was before - gpg_delete_key(keyid) + delete_key(keyid) - return armour.rstrip('\n') + return armour diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 5b52f709..99a4d860 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -13,8 +13,8 @@ except ImportError: from mock import call from cloudinit.config import cc_apt_configure -from cloudinit import util from cloudinit import gpg +from cloudinit import util from ..helpers import TestCase @@ -405,7 +405,7 @@ class TestAptSourceConfig(TestCase): params = self._get_default_params() with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: - with mock.patch.object(gpg, 'gpg_getkeybyid', + with mock.patch.object(gpg, 'get_key_by_id', return_value=expectedkey) as mockgetkey: cc_apt_configure.add_apt_sources([cfg], params) -- cgit v1.2.3 From d4b587ebf500ddc2259fffc94a3c69c199c9a427 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 15 Jun 2016 22:50:12 -0400 Subject: fix some errors reported by pylint pylint --errors-only found several errors. Some of the changes here represent real errors, others just code that pylint did not like. --- cloudinit/config/cc_apt_configure.py | 3 ++- cloudinit/config/cc_growpart.py | 10 +++++----- cloudinit/distros/__init__.py | 2 +- cloudinit/distros/arch.py | 4 ++-- cloudinit/sources/DataSourceOVF.py | 10 +++++----- cloudinit/sources/DataSourceOpenNebula.py | 2 +- cloudinit/sources/helpers/openstack.py | 2 +- 7 files changed, 17 insertions(+), 16 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 96c4a43d..05ad4b03 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -208,8 +208,9 @@ def add_apt_sources(srclist, template_params=None, aa_repo_match=None): template_params = {} if aa_repo_match is None: - def aa_repo_match(x): + def _aa_repo_match(x): return False + aa_repo_match = _aa_repo_match errorlist = [] srcdict = convert_to_new_format(srclist) diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 859d69f1..40560f11 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -36,13 +36,13 @@ DEFAULT_CONFIG = { } -def enum(**enums): - return type('Enum', (), enums) +class RESIZE(object): + SKIPPED = "SKIPPED" + CHANGED = "CHANGED" + NOCHANGE = "NOCHANGE" + FAILED = "FAILED" -RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE", - FAILED="FAILED") - LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 5c29c804..14b500f8 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -454,7 +454,7 @@ class Distro(object): keys = kwargs['ssh_authorized_keys'] if isinstance(keys, six.string_types): keys = [keys] - if isinstance(keys, dict): + elif isinstance(keys, dict): keys = list(keys.values()) if keys is not None: if not isinstance(keys, (tuple, list, set)): diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index 93a2e008..66209f22 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -196,6 +196,6 @@ def convert_resolv_conf(settings): """Returns a settings string formatted for resolv.conf.""" result = '' if isinstance(settings, list): - for ns in list: + for ns in settings: result = result + 'nameserver %s\n' % ns - return result + return result diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index f2bb9366..43347cfb 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -37,16 +37,16 @@ from cloudinit.sources.helpers.vmware.imc.config_file \ import ConfigFile from cloudinit.sources.helpers.vmware.imc.config_nic \ import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.guestcust_error \ + import GuestCustErrorEnum from cloudinit.sources.helpers.vmware.imc.guestcust_event \ import GuestCustEventEnum from cloudinit.sources.helpers.vmware.imc.guestcust_state \ import GuestCustStateEnum -from cloudinit.sourceshelpers.vmware.imc.guestcust_error \ - import GuestCustErrorEnum -from cloudinit.sourceshelpers.vmware.imc.guestcust_util import ( - set_customization_status, +from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( + enable_nics, get_nics_to_enable, - enable_nics + set_customization_status ) LOG = logging.getLogger(__name__) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 8f85b115..7b3a76b9 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -104,7 +104,7 @@ class DataSourceOpenNebula(sources.DataSource): def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == sources.DSMODE_NET: + if self.dsmode == sources.DSMODE_NETWORK: resolve_ip = True else: resolve_ip = False diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 494335b3..d52cb56a 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -157,7 +157,7 @@ class BaseReader(object): pass @abc.abstractmethod - def _path_read(self, path): + def _path_read(self, path, decode=False): pass @abc.abstractmethod -- cgit v1.2.3 From a919f6e6ee3bfaf489411a4452fc708061b1239f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Jun 2016 14:37:23 -0400 Subject: write_files: if no permissions are given, just use default without warn. if no permissions were given in a write_files stanza, then a warning would be emitted. The fix here is just to special case handling of None. --- ChangeLog | 1 + cloudinit/config/cc_write_files.py | 2 ++ 2 files changed, 3 insertions(+) (limited to 'cloudinit/config') diff --git a/ChangeLog b/ChangeLog index 10bd58b8..8edf3249 100644 --- a/ChangeLog +++ b/ChangeLog @@ -124,6 +124,7 @@ - Change missing Cheetah log warning to debug [Andrew Jorgensen] - Remove trailing dot from GCE metadata URL (LP: #1581200) [Phil Roche] - support network rendering to sysconfig (for centos and RHEL) + - write_files: if no permissions are given, just use default without warn. 0.7.6: - open 0.7.6 diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py index 351cfc8c..b1096b9b 100644 --- a/cloudinit/config/cc_write_files.py +++ b/cloudinit/config/cc_write_files.py @@ -79,6 +79,8 @@ def write_files(name, files, log): def decode_perms(perm, default, log): + if perm is None: + return default try: if isinstance(perm, six.integer_types + (float,)): # Just 'downcast' it (if a float) -- cgit v1.2.3