From c6dff581a9c253170d5e3f12fb83d16a8dec8257 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 12 Apr 2018 15:32:25 -0400 Subject: Implement ntp client spec with auto support for distro selection Add a base NTP client configuration dictionary and allow Distro specific changes to be merged. Add a select client function which implements logic to preferr installed clients over clients which need to be installed. Also allow distributions to override the cloud-init defaults. LP: #1749722 --- tests/cloud_tests/testcases/modules/ntp.yaml | 1 + tests/cloud_tests/testcases/modules/ntp_chrony.py | 15 +++++++++++++++ tests/cloud_tests/testcases/modules/ntp_chrony.yaml | 17 +++++++++++++++++ tests/cloud_tests/testcases/modules/ntp_pools.yaml | 1 + tests/cloud_tests/testcases/modules/ntp_servers.yaml | 1 + tests/cloud_tests/testcases/modules/ntp_timesyncd.py | 15 +++++++++++++++ tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml | 15 +++++++++++++++ 7 files changed, 65 insertions(+) create mode 100644 tests/cloud_tests/testcases/modules/ntp_chrony.py create mode 100644 tests/cloud_tests/testcases/modules/ntp_chrony.yaml create mode 100644 tests/cloud_tests/testcases/modules/ntp_timesyncd.py create mode 100644 tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml index 2530d72e..7ea0707d 100644 --- a/tests/cloud_tests/testcases/modules/ntp.yaml +++ b/tests/cloud_tests/testcases/modules/ntp.yaml @@ -4,6 +4,7 @@ cloud_config: | #cloud-config ntp: + ntp_client: ntp pools: [] servers: [] collect_scripts: diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py new file mode 100644 index 00000000..461630a8 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -0,0 +1,15 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script.""" +from tests.cloud_tests.testcases import base + + +class TestNtpChrony(base.CloudTestCase): + """Test ntp module with chrony client""" + + def test_chrony_entires(self): + """Test chrony config entries""" + out = self.get_data_file('chrony_conf') + self.assertIn('.pool.ntp.org', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.yaml b/tests/cloud_tests/testcases/modules/ntp_chrony.yaml new file mode 100644 index 00000000..120735e2 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.yaml @@ -0,0 +1,17 @@ +# +# ntp enabled, chrony selected, check conf file +# as chrony won't start in a container +# +cloud_config: | + #cloud-config + ntp: + enabled: true + ntp_client: chrony +collect_scripts: + chrony_conf: | + #!/bin/sh + set -- /etc/chrony.conf /etc/chrony/chrony.conf + for p in "$@"; do + [ -e "$p" ] && { cat "$p"; exit; } + done +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_pools.yaml b/tests/cloud_tests/testcases/modules/ntp_pools.yaml index d490b228..60fa0fd1 100644 --- a/tests/cloud_tests/testcases/modules/ntp_pools.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_pools.yaml @@ -9,6 +9,7 @@ required_features: cloud_config: | #cloud-config ntp: + ntp_client: ntp pools: - 0.cloud-init.mypool - 1.cloud-init.mypool diff --git a/tests/cloud_tests/testcases/modules/ntp_servers.yaml b/tests/cloud_tests/testcases/modules/ntp_servers.yaml index 6b13b70e..ee636679 100644 --- a/tests/cloud_tests/testcases/modules/ntp_servers.yaml +++ b/tests/cloud_tests/testcases/modules/ntp_servers.yaml @@ -6,6 +6,7 @@ required_features: cloud_config: | #cloud-config ntp: + ntp_client: ntp servers: - 172.16.15.14 - 172.16.17.18 diff --git a/tests/cloud_tests/testcases/modules/ntp_timesyncd.py b/tests/cloud_tests/testcases/modules/ntp_timesyncd.py new file mode 100644 index 00000000..eca750bc --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_timesyncd.py @@ -0,0 +1,15 @@ +# This file is part of cloud-init. See LICENSE file for license information. + +"""cloud-init Integration Test Verify Script.""" +from tests.cloud_tests.testcases import base + + +class TestNtpTimesyncd(base.CloudTestCase): + """Test ntp module with systemd-timesyncd client""" + + def test_timesyncd_entries(self): + """Test timesyncd config entries""" + out = self.get_data_file('timesyncd_conf') + self.assertIn('.pool.ntp.org', out) + +# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml b/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml new file mode 100644 index 00000000..ee47a741 --- /dev/null +++ b/tests/cloud_tests/testcases/modules/ntp_timesyncd.yaml @@ -0,0 +1,15 @@ +# +# ntp enabled, systemd-timesyncd selected, check conf file +# as systemd-timesyncd won't start in a container +# +cloud_config: | + #cloud-config + ntp: + enabled: true + ntp_client: systemd-timesyncd +collect_scripts: + timesyncd_conf: | + #!/bin/sh + cat /etc/systemd/timesyncd.conf.d/cloud-init.conf + +# vi: ts=4 expandtab -- cgit v1.2.3 From 1081962eacf2814fea6f4fa3255c530de14e4a24 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 19 Apr 2018 21:30:08 -0600 Subject: pylint: pay attention to unused variable warnings. This enables warnings produced by pylint for unused variables (W0612), and fixes the existing errors. --- .pylintrc | 2 +- cloudinit/analyze/dump.py | 2 +- cloudinit/cmd/tests/test_main.py | 6 ++-- cloudinit/config/cc_apt_configure.py | 2 +- cloudinit/config/cc_emit_upstart.py | 2 +- cloudinit/config/cc_resizefs.py | 8 ++---- cloudinit/config/cc_rh_subscription.py | 18 ++++++------ cloudinit/config/cc_snap.py | 4 +-- cloudinit/config/cc_snappy.py | 4 +-- cloudinit/config/cc_ubuntu_advantage.py | 4 +-- cloudinit/config/schema.py | 4 +-- cloudinit/distros/freebsd.py | 2 +- cloudinit/distros/ubuntu.py | 2 +- cloudinit/net/__init__.py | 2 +- cloudinit/net/cmdline.py | 2 +- cloudinit/net/dhcp.py | 2 +- cloudinit/net/sysconfig.py | 2 +- cloudinit/reporting/events.py | 2 +- cloudinit/sources/DataSourceAliYun.py | 2 +- cloudinit/sources/DataSourceAzure.py | 33 +++++++++------------- cloudinit/sources/DataSourceMAAS.py | 2 +- cloudinit/sources/DataSourceOVF.py | 2 +- cloudinit/sources/DataSourceOpenStack.py | 4 +-- cloudinit/sources/helpers/digitalocean.py | 7 ++--- cloudinit/sources/helpers/openstack.py | 2 +- cloudinit/sources/helpers/vmware/imc/config_nic.py | 2 +- .../sources/helpers/vmware/imc/config_passwd.py | 4 +-- .../sources/helpers/vmware/imc/guestcust_util.py | 4 +-- cloudinit/sources/tests/test_init.py | 2 +- cloudinit/templater.py | 2 +- cloudinit/tests/helpers.py | 2 +- cloudinit/tests/test_util.py | 2 +- cloudinit/url_helper.py | 2 +- cloudinit/util.py | 2 +- tests/cloud_tests/bddeb.py | 2 +- tests/cloud_tests/collect.py | 3 +- tests/cloud_tests/platforms/instances.py | 2 +- tests/cloud_tests/platforms/lxd/instance.py | 10 +++---- tests/cloud_tests/setup_image.py | 11 ++++---- tests/cloud_tests/testcases/base.py | 2 +- .../testcases/examples/including_user_groups.py | 2 +- tests/cloud_tests/testcases/modules/user_groups.py | 2 +- tests/cloud_tests/util.py | 2 +- tests/unittests/test__init__.py | 2 +- tests/unittests/test_datasource/test_azure.py | 4 +-- tests/unittests/test_datasource/test_maas.py | 4 +-- tests/unittests/test_datasource/test_nocloud.py | 3 -- .../test_handler/test_handler_apt_source_v3.py | 2 +- tests/unittests/test_handler/test_handler_ntp.py | 2 +- tests/unittests/test_templating.py | 4 +-- tests/unittests/test_util.py | 6 ++-- 51 files changed, 95 insertions(+), 112 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/.pylintrc b/.pylintrc index 0bdfa59d..3bfa0c81 100644 --- a/.pylintrc +++ b/.pylintrc @@ -28,7 +28,7 @@ jobs=4 # W0703(broad-except) # W1401(anomalous-backslash-in-string) -disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0612, W0613, W0621, W0622, W0631, W0703, W1401 +disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401 [REPORTS] diff --git a/cloudinit/analyze/dump.py b/cloudinit/analyze/dump.py index b071aa19..1f3060d0 100644 --- a/cloudinit/analyze/dump.py +++ b/cloudinit/analyze/dump.py @@ -112,7 +112,7 @@ def parse_ci_logline(line): return None event_description = stage_to_description[event_name] else: - (pymodloglvl, event_type, event_name) = eventstr.split()[0:3] + (_pymodloglvl, event_type, event_name) = eventstr.split()[0:3] event_description = eventstr.split(event_name)[1].strip() event = { diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py index dbe421c0..e2c54ae8 100644 --- a/cloudinit/cmd/tests/test_main.py +++ b/cloudinit/cmd/tests/test_main.py @@ -56,7 +56,7 @@ class TestMain(FilesystemMockingTestCase): cmdargs = myargs( debug=False, files=None, force=False, local=False, reporter=None, subcommand='init') - (item1, item2) = wrap_and_call( + (_item1, item2) = wrap_and_call( 'cloudinit.cmd.main', {'util.close_stdin': True, 'netinfo.debug_info': 'my net debug info', @@ -85,7 +85,7 @@ class TestMain(FilesystemMockingTestCase): cmdargs = myargs( debug=False, files=None, force=False, local=False, reporter=None, subcommand='init') - (item1, item2) = wrap_and_call( + (_item1, item2) = wrap_and_call( 'cloudinit.cmd.main', {'util.close_stdin': True, 'netinfo.debug_info': 'my net debug info', @@ -133,7 +133,7 @@ class TestMain(FilesystemMockingTestCase): self.assertEqual(main.LOG, log) self.assertIsNone(args) - (item1, item2) = wrap_and_call( + (_item1, item2) = wrap_and_call( 'cloudinit.cmd.main', {'util.close_stdin': True, 'netinfo.debug_info': 'my net debug info', diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index afaca464..e18944ec 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -378,7 +378,7 @@ def apply_debconf_selections(cfg, target=None): # get a complete list of packages listed in input pkgs_cfgd = set() - for key, content in selsets.items(): + for _key, content in selsets.items(): for line in content.splitlines(): if line.startswith("#"): continue diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 69dc2d5e..eb9fbe66 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -43,7 +43,7 @@ def is_upstart_system(): del myenv['UPSTART_SESSION'] check_cmd = ['initctl', 'version'] try: - (out, err) = util.subp(check_cmd, env=myenv) + (out, _err) = util.subp(check_cmd, env=myenv) return 'upstart' in out except util.ProcessExecutionError as e: LOG.debug("'%s' returned '%s', not using upstart", diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 013e69b5..82f29e10 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -89,13 +89,11 @@ def _resize_zfs(mount_point, devpth): def _get_dumpfs_output(mount_point): - dumpfs_res, err = util.subp(['dumpfs', '-m', mount_point]) - return dumpfs_res + return util.subp(['dumpfs', '-m', mount_point])[0] def _get_gpart_output(part): - gpart_res, err = util.subp(['gpart', 'show', part]) - return gpart_res + return util.subp(['gpart', 'show', part])[0] def _can_skip_resize_ufs(mount_point, devpth): @@ -113,7 +111,7 @@ def _can_skip_resize_ufs(mount_point, devpth): if not line.startswith('#'): newfs_cmd = shlex.split(line) opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:' - optlist, args = getopt.getopt(newfs_cmd[1:], opt_value) + optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) for o, a in optlist: if o == "-s": cur_fs_sz = int(a) diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py index 530808ce..1c679430 100644 --- a/cloudinit/config/cc_rh_subscription.py +++ b/cloudinit/config/cc_rh_subscription.py @@ -209,8 +209,7 @@ class SubscriptionManager(object): cmd.append("--serverurl={0}".format(self.server_hostname)) try: - return_out, return_err = self._sub_man_cli(cmd, - logstring_val=True) + return_out = self._sub_man_cli(cmd, logstring_val=True)[0] except util.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " @@ -233,8 +232,7 @@ class SubscriptionManager(object): # Attempting to register the system only try: - return_out, return_err = self._sub_man_cli(cmd, - logstring_val=True) + return_out = self._sub_man_cli(cmd, logstring_val=True)[0] except util.ProcessExecutionError as e: if e.stdout == "": self.log_warn("Registration failed due " @@ -257,7 +255,7 @@ class SubscriptionManager(object): .format(self.servicelevel)] try: - return_out, return_err = self._sub_man_cli(cmd) + return_out = self._sub_man_cli(cmd)[0] except util.ProcessExecutionError as e: if e.stdout.rstrip() != '': for line in e.stdout.split("\n"): @@ -275,7 +273,7 @@ class SubscriptionManager(object): def _set_auto_attach(self): cmd = ['attach', '--auto'] try: - return_out, return_err = self._sub_man_cli(cmd) + return_out = self._sub_man_cli(cmd)[0] except util.ProcessExecutionError as e: self.log_warn("Auto-attach failed with: {0}".format(e)) return False @@ -294,12 +292,12 @@ class SubscriptionManager(object): # Get all available pools cmd = ['list', '--available', '--pool-only'] - results, errors = self._sub_man_cli(cmd) + results = self._sub_man_cli(cmd)[0] available = (results.rstrip()).split("\n") # Get all consumed pools cmd = ['list', '--consumed', '--pool-only'] - results, errors = self._sub_man_cli(cmd) + results = self._sub_man_cli(cmd)[0] consumed = (results.rstrip()).split("\n") return available, consumed @@ -311,14 +309,14 @@ class SubscriptionManager(object): ''' cmd = ['repos', '--list-enabled'] - return_out, return_err = self._sub_man_cli(cmd) + return_out = self._sub_man_cli(cmd)[0] active_repos = [] for repo in return_out.split("\n"): if "Repo ID:" in repo: active_repos.append((repo.split(':')[1]).strip()) cmd = ['repos', '--list-disabled'] - return_out, return_err = self._sub_man_cli(cmd) + return_out = self._sub_man_cli(cmd)[0] inactive_repos = [] for repo in return_out.split("\n"): diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py index a7a03214..90724b81 100644 --- a/cloudinit/config/cc_snap.py +++ b/cloudinit/config/cc_snap.py @@ -203,12 +203,12 @@ def maybe_install_squashfuse(cloud): return try: cloud.distro.update_package_sources() - except Exception as e: + except Exception: util.logexc(LOG, "Package update failed") raise try: cloud.distro.install_packages(['squashfuse']) - except Exception as e: + except Exception: util.logexc(LOG, "Failed to install squashfuse") raise diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index bab80bbe..15bee2d3 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -213,7 +213,7 @@ def render_snap_op(op, name, path=None, cfgfile=None, config=None): def read_installed_packages(): ret = [] - for (name, date, version, dev) in read_pkg_data(): + for (name, _date, _version, dev) in read_pkg_data(): if dev: ret.append(NAMESPACE_DELIM.join([name, dev])) else: @@ -222,7 +222,7 @@ def read_installed_packages(): def read_pkg_data(): - out, err = util.subp([SNAPPY_CMD, "list"]) + out, _err = util.subp([SNAPPY_CMD, "list"]) pkg_data = [] for line in out.splitlines()[1:]: toks = line.split(sep=None, maxsplit=3) diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py index 29d18c96..5e082bd6 100644 --- a/cloudinit/config/cc_ubuntu_advantage.py +++ b/cloudinit/config/cc_ubuntu_advantage.py @@ -148,12 +148,12 @@ def maybe_install_ua_tools(cloud): return try: cloud.distro.update_package_sources() - except Exception as e: + except Exception: util.logexc(LOG, "Package update failed") raise try: cloud.distro.install_packages(['ubuntu-advantage-tools']) - except Exception as e: + except Exception: util.logexc(LOG, "Failed to install ubuntu-advantage-tools") raise diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index ca7d0d5b..76826e05 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -297,8 +297,8 @@ def get_schema(): configs_dir = os.path.dirname(os.path.abspath(__file__)) potential_handlers = find_modules(configs_dir) - for (fname, mod_name) in potential_handlers.items(): - mod_locs, looked_locs = importer.find_module( + for (_fname, mod_name) in potential_handlers.items(): + mod_locs, _looked_locs = importer.find_module( mod_name, ['cloudinit.config'], ['schema']) if mod_locs: mod = importer.import_module(mod_locs[0]) diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py index 099fac5c..5b1718a4 100644 --- a/cloudinit/distros/freebsd.py +++ b/cloudinit/distros/freebsd.py @@ -113,7 +113,7 @@ class Distro(distros.Distro): n = re.search(r'\d+$', dev) index = n.group(0) - (out, err) = util.subp(['ifconfig', '-a']) + (out, _err) = util.subp(['ifconfig', '-a']) ifconfigoutput = [x for x in (out.strip()).splitlines() if len(x.split()) > 0] bsddev = 'NOT_FOUND' diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index fdc1f622..68154104 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -25,7 +25,7 @@ class Distro(debian.Distro): def preferred_ntp_clients(self): """The preferred ntp client is dependent on the version.""" if not self._preferred_ntp_clients: - (name, version, codename) = util.system_info()['dist'] + (_name, _version, codename) = util.system_info()['dist'] # Xenial cloud-init only installed ntp, UbuntuCore has timesyncd. if codename == "xenial" and not util.system_is_snappy(): self._preferred_ntp_clients = ['ntp'] diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f69c0ef2..80054546 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -295,7 +295,7 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): def _version_2(netcfg): renames = [] - for key, ent in netcfg.get('ethernets', {}).items(): + for ent in netcfg.get('ethernets', {}).values(): # only rename if configured to do so name = ent.get('set-name') if not name: diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py index 9e9fe0fe..f89a0f73 100755 --- a/cloudinit/net/cmdline.py +++ b/cloudinit/net/cmdline.py @@ -65,7 +65,7 @@ def _klibc_to_config_entry(content, mac_addrs=None): iface['mac_address'] = mac_addrs[name] # Handle both IPv4 and IPv6 values - for v, pre in (('ipv4', 'IPV4'), ('ipv6', 'IPV6')): + for pre in ('IPV4', 'IPV6'): # if no IPV4ADDR or IPV6ADDR, then go on. if pre + "ADDR" not in data: continue diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index 087c0c03..12cf5097 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -216,7 +216,7 @@ def networkd_get_option_from_leases(keyname, leases_d=None): if leases_d is None: leases_d = NETWORKD_LEASES_DIR leases = networkd_load_leases(leases_d=leases_d) - for ifindex, data in sorted(leases.items()): + for _ifindex, data in sorted(leases.items()): if data.get(keyname): return data[keyname] return None diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index 39d89c46..7a7f5093 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -364,7 +364,7 @@ class Renderer(renderer.Renderer): @classmethod def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): - for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): + for _, subnet in enumerate(subnets, start=len(iface_cfg.children)): for route in subnet.get('routes', []): is_ipv6 = subnet.get('ipv6') or is_ipv6_addr(route['gateway']) diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py index 4f62d2f9..e5dfab33 100644 --- a/cloudinit/reporting/events.py +++ b/cloudinit/reporting/events.py @@ -192,7 +192,7 @@ class ReportEventStack(object): def _childrens_finish_info(self): for cand_result in (status.FAIL, status.WARN): - for name, (value, msg) in self.children.items(): + for _name, (value, _msg) in self.children.items(): if value == cand_result: return (value, self.message) return (self.result, self.message) diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py index 22279d09..858e0827 100644 --- a/cloudinit/sources/DataSourceAliYun.py +++ b/cloudinit/sources/DataSourceAliYun.py @@ -45,7 +45,7 @@ def _is_aliyun(): def parse_public_keys(public_keys): keys = [] - for key_id, key_body in public_keys.items(): + for _key_id, key_body in public_keys.items(): if isinstance(key_body, str): keys.append(key_body.strip()) elif isinstance(key_body, list): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 0ee622e2..a71197a6 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -107,31 +107,24 @@ def find_dev_from_busdev(camcontrol_out, busdev): return None -def get_dev_storvsc_sysctl(): +def execute_or_debug(cmd, fail_ret=None): try: - sysctl_out, err = util.subp(['sysctl', 'dev.storvsc']) + return util.subp(cmd)[0] except util.ProcessExecutionError: - LOG.debug("Fail to execute sysctl dev.storvsc") - sysctl_out = "" - return sysctl_out + LOG.debug("Failed to execute: %s", ' '.join(cmd)) + return fail_ret + + +def get_dev_storvsc_sysctl(): + return execute_or_debug(["sysctl", "dev.storvsc"], fail_ret="") def get_camcontrol_dev_bus(): - try: - camcontrol_b_out, err = util.subp(['camcontrol', 'devlist', '-b']) - except util.ProcessExecutionError: - LOG.debug("Fail to execute camcontrol devlist -b") - return None - return camcontrol_b_out + return execute_or_debug(['camcontrol', 'devlist', '-b']) def get_camcontrol_dev(): - try: - camcontrol_out, err = util.subp(['camcontrol', 'devlist']) - except util.ProcessExecutionError: - LOG.debug("Fail to execute camcontrol devlist") - return None - return camcontrol_out + return execute_or_debug(['camcontrol', 'devlist']) def get_resource_disk_on_freebsd(port_id): @@ -474,7 +467,7 @@ class DataSourceAzure(sources.DataSource): before we go into our polling loop.""" try: get_metadata_from_fabric(None, lease['unknown-245']) - except Exception as exc: + except Exception: LOG.warning( "Error communicating with Azure fabric; You may experience." "connectivity issues.", exc_info=True) @@ -492,7 +485,7 @@ class DataSourceAzure(sources.DataSource): jump back into the polling loop in order to retrieve the ovf_env.""" if not ret: return False - (md, self.userdata_raw, cfg, files) = ret + (_md, self.userdata_raw, cfg, _files) = ret path = REPROVISION_MARKER_FILE if (cfg.get('PreprovisionedVm') is True or os.path.isfile(path)): @@ -528,7 +521,7 @@ class DataSourceAzure(sources.DataSource): self.ds_cfg['agent_command']) try: fabric_data = metadata_func() - except Exception as exc: + except Exception: LOG.warning( "Error communicating with Azure fabric; You may experience." "connectivity issues.", exc_info=True) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 6ac88635..aa56addb 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -204,7 +204,7 @@ def read_maas_seed_url(seed_url, read_file_or_url=None, timeout=None, seed_url = seed_url[:-1] md = {} - for path, dictname, binary, optional in DS_FIELDS: + for path, _dictname, binary, optional in DS_FIELDS: if version is None: url = "%s/%s" % (seed_url, path) else: diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index dc914a72..178ccb0f 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -556,7 +556,7 @@ def search_file(dirpath, filename): if not dirpath or not filename: return None - for root, dirs, files in os.walk(dirpath): + for root, _dirs, files in os.walk(dirpath): if filename in files: return os.path.join(root, filename) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index e55a7638..fb166ae1 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -86,7 +86,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): md_urls.append(md_url) url2base[md_url] = url - (max_wait, timeout, retries) = self._get_url_settings() + (max_wait, timeout, _retries) = self._get_url_settings() start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, timeout=timeout) @@ -106,7 +106,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): except IOError: return False - (max_wait, timeout, retries) = self._get_url_settings() + (_max_wait, timeout, retries) = self._get_url_settings() try: results = util.log_time(LOG.debug, diff --git a/cloudinit/sources/helpers/digitalocean.py b/cloudinit/sources/helpers/digitalocean.py index 693f8d5c..0e7cccac 100644 --- a/cloudinit/sources/helpers/digitalocean.py +++ b/cloudinit/sources/helpers/digitalocean.py @@ -41,10 +41,9 @@ def assign_ipv4_link_local(nic=None): "address") try: - (result, _err) = util.subp(ip_addr_cmd) + util.subp(ip_addr_cmd) LOG.debug("assigned ip4LL address '%s' to '%s'", addr, nic) - - (result, _err) = util.subp(ip_link_cmd) + util.subp(ip_link_cmd) LOG.debug("brought device '%s' up", nic) except Exception: util.logexc(LOG, "ip4LL address assignment of '%s' to '%s' failed." @@ -75,7 +74,7 @@ def del_ipv4_link_local(nic=None): ip_addr_cmd = ['ip', 'addr', 'flush', 'dev', nic] try: - (result, _err) = util.subp(ip_addr_cmd) + util.subp(ip_addr_cmd) LOG.debug("removed ip4LL addresses from %s", nic) except Exception as e: diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 26f3168d..a4cf0667 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -638,7 +638,7 @@ def convert_net_json(network_json=None, known_macs=None): known_macs = net.get_interfaces_by_mac() # go through and fill out the link_id_info with names - for link_id, info in link_id_info.items(): + for _link_id, info in link_id_info.items(): if info.get('name'): continue if info.get('mac') in known_macs: diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 2d8900e2..3ef8c624 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -73,7 +73,7 @@ class NicConfigurator(object): The mac address(es) are in the lower case """ cmd = ['ip', 'addr', 'show'] - (output, err) = util.subp(cmd) + output, _err = util.subp(cmd) sections = re.split(r'\n\d+: ', '\n' + output)[1:] macPat = r'link/ether (([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2}))' diff --git a/cloudinit/sources/helpers/vmware/imc/config_passwd.py b/cloudinit/sources/helpers/vmware/imc/config_passwd.py index 75cfbaaf..8c91fa41 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_passwd.py +++ b/cloudinit/sources/helpers/vmware/imc/config_passwd.py @@ -56,10 +56,10 @@ class PasswordConfigurator(object): LOG.info('Expiring password.') for user in uidUserList: try: - out, err = util.subp(['passwd', '--expire', user]) + util.subp(['passwd', '--expire', user]) except util.ProcessExecutionError as e: if os.path.exists('/usr/bin/chage'): - out, e = util.subp(['chage', '-d', '0', user]) + util.subp(['chage', '-d', '0', user]) else: LOG.warning('Failed to expire password for %s with error: ' '%s', user, e) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 44075255..a590f323 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -91,7 +91,7 @@ def enable_nics(nics): for attempt in range(0, enableNicsWaitRetries): logger.debug("Trying to connect interfaces, attempt %d", attempt) - (out, err) = set_customization_status( + (out, _err) = set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, nics) @@ -104,7 +104,7 @@ def enable_nics(nics): return for count in range(0, enableNicsWaitCount): - (out, err) = set_customization_status( + (out, _err) = set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, nics) diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index e7fda22a..452e9219 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -278,7 +278,7 @@ class TestDataSource(CiTestCase): base_args = get_args(DataSource.get_hostname) # pylint: disable=W1505 # Import all DataSource subclasses so we can inspect them. modules = util.find_modules(os.path.dirname(os.path.dirname(__file__))) - for loc, name in modules.items(): + for _loc, name in modules.items(): mod_locs, _ = importer.find_module(name, ['cloudinit.sources'], []) if mod_locs: importer.import_module(mod_locs[0]) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index 9a087e1c..7e7acb86 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -147,7 +147,7 @@ def render_string(content, params): Warning: py2 str with non-ascii chars will cause UnicodeDecodeError.""" if not params: params = {} - template_type, renderer, content = detect_template(content) + _template_type, renderer, content = detect_template(content) return renderer(content, params) # vi: ts=4 expandtab diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 82fd347b..5aada6e7 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -334,7 +334,7 @@ def dir2dict(startdir, prefix=None): flist = {} if prefix is None: prefix = startdir - for root, dirs, files in os.walk(startdir): + for root, _dirs, files in os.walk(startdir): for fname in files: fpath = os.path.join(root, fname) key = fpath[len(prefix):] diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py index 3f37dbb6..76eed076 100644 --- a/cloudinit/tests/test_util.py +++ b/cloudinit/tests/test_util.py @@ -135,7 +135,7 @@ class TestGetHostnameFqdn(CiTestCase): def test_get_hostname_fqdn_from_passes_metadata_only_to_cloud(self): """Calls to cloud.get_hostname pass the metadata_only parameter.""" mycloud = FakeCloud('cloudhost', 'cloudhost.mycloud.com') - hostname, fqdn = util.get_hostname_fqdn( + _hn, _fqdn = util.get_hostname_fqdn( cfg={}, cloud=mycloud, metadata_only=True) self.assertEqual( [{'fqdn': True, 'metadata_only': True}, diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 03a573af..1de07b1c 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -519,7 +519,7 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret, resource_owner_secret=token_secret, signature_method=oauth1.SIGNATURE_PLAINTEXT, timestamp=timestamp) - uri, signed_headers, body = client.sign(url) + _uri, signed_headers, _body = client.sign(url) return signed_headers # vi: ts=4 expandtab diff --git a/cloudinit/util.py b/cloudinit/util.py index 1717b529..310758dd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2214,7 +2214,7 @@ def parse_mtab(path): def find_freebsd_part(label_part): if label_part.startswith("/dev/label/"): target_label = label_part[5:] - (label_part, err) = subp(['glabel', 'status', '-s']) + (label_part, _err) = subp(['glabel', 'status', '-s']) for labels in label_part.split("\n"): items = labels.split() if len(items) > 0 and items[0].startswith(target_label): diff --git a/tests/cloud_tests/bddeb.py b/tests/cloud_tests/bddeb.py index b9cfcfa6..f04d0cd4 100644 --- a/tests/cloud_tests/bddeb.py +++ b/tests/cloud_tests/bddeb.py @@ -113,7 +113,7 @@ def bddeb(args): @return_value: fail count """ LOG.info('preparing to build cloud-init deb') - (res, failed) = run_stage('build deb', [partial(setup_build, args)]) + _res, failed = run_stage('build deb', [partial(setup_build, args)]) return failed # vi: ts=4 expandtab diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index d4f9135b..1ba72856 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -25,7 +25,8 @@ def collect_script(instance, base_dir, script, script_name): script.encode(), rcs=False, description='collect: {}'.format(script_name)) if err: - LOG.debug("collect script %s had stderr: %s", script_name, err) + LOG.debug("collect script %s exited '%s' and had stderr: %s", + script_name, err, exit) if not isinstance(out, bytes): raise util.PlatformError( "Collection of '%s' returned type %s, expected bytes: %s" % diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py index 3bad021f..cc439d29 100644 --- a/tests/cloud_tests/platforms/instances.py +++ b/tests/cloud_tests/platforms/instances.py @@ -108,7 +108,7 @@ class Instance(TargetBase): return client except (ConnectionRefusedError, AuthenticationException, BadHostKeyException, ConnectionResetError, SSHException, - OSError) as e: + OSError): retries -= 1 time.sleep(10) diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py index 0d957bca..1c17c781 100644 --- a/tests/cloud_tests/platforms/lxd/instance.py +++ b/tests/cloud_tests/platforms/lxd/instance.py @@ -152,9 +152,8 @@ class LXDInstance(Instance): return fp.read() try: - stdout, stderr = subp( - ['lxc', 'console', '--show-log', self.name], decode=False) - return stdout + return subp(['lxc', 'console', '--show-log', self.name], + decode=False)[0] except ProcessExecutionError as e: raise PlatformError( "console log", @@ -214,11 +213,10 @@ def _has_proper_console_support(): reason = "LXD Driver version not 3.x+ (%s)" % dver else: try: - stdout, stderr = subp(['lxc', 'console', '--help'], - decode=False) + stdout = subp(['lxc', 'console', '--help'], decode=False)[0] if not (b'console' in stdout and b'log' in stdout): reason = "no '--log' in lxc console --help" - except ProcessExecutionError as e: + except ProcessExecutionError: reason = "no 'console' command in lxc client" if reason: diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 6d242115..4e195709 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -25,10 +25,9 @@ def installed_package_version(image, package, ensure_installed=True): else: raise NotImplementedError - msg = 'query version for package: {}'.format(package) - (out, err, exit) = image.execute( - cmd, description=msg, rcs=(0,) if ensure_installed else range(0, 256)) - return out.strip() + return image.execute( + cmd, description='query version for package: {}'.format(package), + rcs=(0,) if ensure_installed else range(0, 256))[0].strip() def install_deb(args, image): @@ -54,7 +53,7 @@ def install_deb(args, image): remote_path], description=msg) # check installed deb version matches package fmt = ['-W', "--showformat=${Version}"] - (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path]) + out = image.execute(['dpkg-deb'] + fmt + [remote_path])[0] expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: @@ -85,7 +84,7 @@ def install_rpm(args, image): image.execute(['rpm', '-U', remote_path], description=msg) fmt = ['--queryformat', '"%{VERSION}"'] - (out, err, exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) + (out, _err, _exit) = image.execute(['rpm', '-q'] + fmt + [remote_path]) expected_version = out.strip() found_version = installed_package_version(image, 'cloud-init') if expected_version != found_version: diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 4fda8f91..0d1916b4 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -159,7 +159,7 @@ class CloudTestCase(unittest.TestCase): expected_net_keys = [ 'public-ipv4s', 'ipv4-associations', 'local-hostname', 'public-hostname'] - for mac, mac_data in macs.items(): + for mac_data in macs.values(): for key in expected_net_keys: self.assertIn(key, mac_data) self.assertIsNotNone( diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py index 93b7a82d..4067348d 100644 --- a/tests/cloud_tests/testcases/examples/including_user_groups.py +++ b/tests/cloud_tests/testcases/examples/including_user_groups.py @@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase): def test_user_root_in_secret(self): """Test root user is in 'secret' group.""" - user, _, groups = self.get_data_file('root_groups').partition(":") + _user, _, groups = self.get_data_file('root_groups').partition(":") self.assertIn("secret", groups.split(), msg="User root is not in group 'secret'") diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py index 93b7a82d..4067348d 100644 --- a/tests/cloud_tests/testcases/modules/user_groups.py +++ b/tests/cloud_tests/testcases/modules/user_groups.py @@ -42,7 +42,7 @@ class TestUserGroups(base.CloudTestCase): def test_user_root_in_secret(self): """Test root user is in 'secret' group.""" - user, _, groups = self.get_data_file('root_groups').partition(":") + _user, _, groups = self.get_data_file('root_groups').partition(":") self.assertIn("secret", groups.split(), msg="User root is not in group 'secret'") diff --git a/tests/cloud_tests/util.py b/tests/cloud_tests/util.py index 3dd4996d..06f7d865 100644 --- a/tests/cloud_tests/util.py +++ b/tests/cloud_tests/util.py @@ -358,7 +358,7 @@ class TargetBase(object): # when sh is invoked with '-c', then the first argument is "$0" # which is commonly understood as the "program name". # 'read_data' is the program name, and 'remote_path' is '$1' - stdout, stderr, rc = self._execute( + stdout, _stderr, rc = self._execute( ["sh", "-c", 'exec cat "$1"', 'read_data', remote_path]) if rc != 0: raise RuntimeError("Failed to read file '%s'" % remote_path) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 25878d7a..f1ab02e9 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -214,7 +214,7 @@ class TestCmdlineUrl(CiTestCase): def test_no_key_found(self, m_read): cmdline = "ro mykey=http://example.com/foo root=foo" fpath = self.tmp_path("ccpath") - lvl, msg = main.attempt_cmdline_url( + lvl, _msg = main.attempt_cmdline_url( fpath, network=True, cmdline=cmdline) m_read.assert_not_called() diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 3e8b7913..88fe76c7 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -214,7 +214,7 @@ scbus-1 on xpt0 bus 0 self.assertIn(tag, x) def tags_equal(x, y): - for x_tag, x_val in x.items(): + for x_val in x.values(): y_val = y.get(x_val.tag) self.assertEqual(x_val.text, y_val.text) @@ -1216,7 +1216,7 @@ class TestAzureDataSourcePreprovisioning(CiTestCase): fake_resp.return_value = mock.MagicMock(status_code=200, text=content, content=content) dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) - md, ud, cfg, d = dsa._reprovision() + md, _ud, cfg, _d = dsa._reprovision() self.assertEqual(md['local-hostname'], hostname) self.assertEqual(cfg['system_info']['default_user']['name'], username) self.assertEqual(fake_resp.call_args_list, diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index 6e4031cf..c84d067e 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -53,7 +53,7 @@ class TestMAASDataSource(CiTestCase): my_d = os.path.join(self.tmp, "valid_extra") populate_dir(my_d, data) - ud, md, vd = DataSourceMAAS.read_maas_seed_dir(my_d) + ud, md, _vd = DataSourceMAAS.read_maas_seed_dir(my_d) self.assertEqual(userdata, ud) for key in ('instance-id', 'local-hostname'): @@ -149,7 +149,7 @@ class TestMAASDataSource(CiTestCase): 'meta-data/local-hostname': 'test-hostname', 'meta-data/vendor-data': yaml.safe_dump(expected_vd).encode(), } - ud, md, vd = self.mock_read_maas_seed_url( + _ud, md, vd = self.mock_read_maas_seed_url( valid, "http://example.com/foo") self.assertEqual(valid['meta-data/instance-id'], md['instance-id']) diff --git a/tests/unittests/test_datasource/test_nocloud.py b/tests/unittests/test_datasource/test_nocloud.py index 70d50de4..cdbd1e1a 100644 --- a/tests/unittests/test_datasource/test_nocloud.py +++ b/tests/unittests/test_datasource/test_nocloud.py @@ -51,9 +51,6 @@ class TestNoCloudDataSource(CiTestCase): class PsuedoException(Exception): pass - def my_find_devs_with(*args, **kwargs): - raise PsuedoException - self.mocks.enter_context( mock.patch.object(util, 'find_devs_with', side_effect=PsuedoException)) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 7bb1b7c4..e486862d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -528,7 +528,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): expected = sorted([npre + suff for opre, npre, suff in files]) # create files - for (opre, npre, suff) in files: + for (opre, _npre, suff) in files: fpath = os.path.join(apt_lists_d, opre + suff) util.write_file(fpath, content=fpath) diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py index 02676aa6..17c53559 100644 --- a/tests/unittests/test_handler/test_handler_ntp.py +++ b/tests/unittests/test_handler/test_handler_ntp.py @@ -76,7 +76,7 @@ class TestNtp(FilesystemMockingTestCase): template = TIMESYNCD_TEMPLATE else: template = NTP_TEMPLATE - (confpath, template_fn) = self._generate_template(template=template) + (confpath, _template_fn) = self._generate_template(template=template) ntpconfig = copy.deepcopy(dcfg[client]) ntpconfig['confpath'] = confpath ntpconfig['template_name'] = os.path.basename(confpath) diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py index 1080e135..20c87efa 100644 --- a/tests/unittests/test_templating.py +++ b/tests/unittests/test_templating.py @@ -50,12 +50,12 @@ class TestTemplates(test_helpers.CiTestCase): def test_detection(self): blob = "## template:cheetah" - (template_type, renderer, contents) = templater.detect_template(blob) + (template_type, _renderer, contents) = templater.detect_template(blob) self.assertIn("cheetah", template_type) self.assertEqual("", contents.strip()) blob = "blahblah $blah" - (template_type, renderer, contents) = templater.detect_template(blob) + (template_type, _renderer, _contents) = templater.detect_template(blob) self.assertIn("cheetah", template_type) self.assertEqual(blob, contents) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index e04ea031..84941c7d 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -774,11 +774,11 @@ class TestSubp(helpers.CiTestCase): def test_subp_reads_env(self): with mock.patch.dict("os.environ", values={'FOO': 'BAR'}): - out, err = util.subp(self.printenv + ['FOO'], capture=True) + out, _err = util.subp(self.printenv + ['FOO'], capture=True) self.assertEqual('FOO=BAR', out.splitlines()[0]) def test_subp_env_and_update_env(self): - out, err = util.subp( + out, _err = util.subp( self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, env={'FOO': 'BAR'}, update_env={'HOME': '/myhome', 'K2': 'V2'}) @@ -788,7 +788,7 @@ class TestSubp(helpers.CiTestCase): def test_subp_update_env(self): extra = {'FOO': 'BAR', 'HOME': '/root', 'K1': 'V1'} with mock.patch.dict("os.environ", values=extra): - out, err = util.subp( + out, _err = util.subp( self.printenv + ['FOO', 'HOME', 'K1', 'K2'], capture=True, update_env={'HOME': '/myhome', 'K2': 'V2'}) -- cgit v1.2.3 From 323eb30940cae2069daf74517089220fccc4afb9 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 9 May 2018 09:36:56 -0600 Subject: tests: fix package and ca_cert cloud_tests on bionic package_update_upgrade_install was failing as htop is now included in Bionic images. Switch this test to install 'sl' instead. ca_certs integration test fails on cert_count test because bionic update-ca-certificates on bionic generates less symlinks for a given cert. Integration tests now collect dpkg-query --show output on every instance. Add a new assertPackageInstalled helper method which finds the package or package version installed on the instance. Adapt existing byobu, package_update_upgrade_install, ntp and salt_minion tests to use assertPackageInstalled method. LP: #1769985 --- tests/cloud_tests/testcases.yaml | 4 ++-- tests/cloud_tests/testcases/base.py | 21 +++++++++++++++++++++ tests/cloud_tests/testcases/modules/byobu.py | 3 +-- tests/cloud_tests/testcases/modules/byobu.yaml | 3 --- tests/cloud_tests/testcases/modules/ca_certs.py | 21 +++++++++++++++++---- tests/cloud_tests/testcases/modules/ca_certs.yaml | 8 ++++++-- tests/cloud_tests/testcases/modules/ntp.py | 5 ++--- .../modules/package_update_upgrade_install.py | 14 ++++++-------- .../modules/package_update_upgrade_install.yaml | 9 +++------ tests/cloud_tests/testcases/modules/salt_minion.py | 3 +-- .../cloud_tests/testcases/modules/salt_minion.yaml | 3 --- 11 files changed, 59 insertions(+), 35 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml index a3e29900..a16d1ddf 100644 --- a/tests/cloud_tests/testcases.yaml +++ b/tests/cloud_tests/testcases.yaml @@ -24,9 +24,9 @@ base_test_data: status.json: | #!/bin/sh cat /run/cloud-init/status.json - cloud-init-version: | + package-versions: | #!/bin/sh - dpkg-query -W -f='${Version}' cloud-init + dpkg-query --show system.journal.gz: | #!/bin/sh [ -d /run/systemd ] || { echo "not systemd."; exit 0; } diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index 0d1916b4..696db8dd 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -31,6 +31,27 @@ class CloudTestCase(unittest.TestCase): def is_distro(self, distro_name): return self.os_cfg['os'] == distro_name + def assertPackageInstalled(self, name, version=None): + """Check dpkg-query --show output for matching package name. + + @param name: package base name + @param version: string representing a package version or part of a + version. + """ + pkg_out = self.get_data_file('package-versions') + pkg_match = re.search( + '^%s\t(?P.*)$' % name, pkg_out, re.MULTILINE) + if pkg_match: + installed_version = pkg_match.group('version') + if not version: + return # Success + if installed_version.startswith(version): + return # Success + raise AssertionError( + 'Expected package version %s-%s not found. Found %s' % + name, version, installed_version) + raise AssertionError('Package not installed: %s' % name) + def os_version_cmp(self, cmp_version): """Compare the version of the test to comparison_version. diff --git a/tests/cloud_tests/testcases/modules/byobu.py b/tests/cloud_tests/testcases/modules/byobu.py index 005ca014..74d0529a 100644 --- a/tests/cloud_tests/testcases/modules/byobu.py +++ b/tests/cloud_tests/testcases/modules/byobu.py @@ -9,8 +9,7 @@ class TestByobu(base.CloudTestCase): def test_byobu_installed(self): """Test byobu installed.""" - out = self.get_data_file('byobu_installed') - self.assertIn('/usr/bin/byobu', out) + self.assertPackageInstalled('byobu') def test_byobu_profile_enabled(self): """Test byobu profile.d file exists.""" diff --git a/tests/cloud_tests/testcases/modules/byobu.yaml b/tests/cloud_tests/testcases/modules/byobu.yaml index a9aa1f3f..d002a611 100644 --- a/tests/cloud_tests/testcases/modules/byobu.yaml +++ b/tests/cloud_tests/testcases/modules/byobu.yaml @@ -7,9 +7,6 @@ cloud_config: | #cloud-config byobu_by_default: enable collect_scripts: - byobu_installed: | - #!/bin/bash - which byobu byobu_profile_enabled: | #!/bin/bash ls /etc/profile.d/Z97-byobu.sh diff --git a/tests/cloud_tests/testcases/modules/ca_certs.py b/tests/cloud_tests/testcases/modules/ca_certs.py index e75f0413..6b56f639 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.py +++ b/tests/cloud_tests/testcases/modules/ca_certs.py @@ -7,10 +7,23 @@ from tests.cloud_tests.testcases import base class TestCaCerts(base.CloudTestCase): """Test ca certs module.""" - def test_cert_count(self): - """Test the count is proper.""" - out = self.get_data_file('cert_count') - self.assertEqual(5, int(out)) + def test_certs_updated(self): + """Test certs have been updated in /etc/ssl/certs.""" + out = self.get_data_file('cert_links') + # Bionic update-ca-certificates creates less links debian #895075 + unlinked_files = [] + links = {} + for cert_line in out.splitlines(): + if '->' in cert_line: + fname, _sep, link = cert_line.split() + links[fname] = link + else: + unlinked_files.append(cert_line) + self.assertEqual(['ca-certificates.crt'], unlinked_files) + self.assertEqual('cloud-init-ca-certs.pem', links['a535c1f3.0']) + self.assertEqual( + '/usr/share/ca-certificates/cloud-init-ca-certs.crt', + links['cloud-init-ca-certs.pem']) def test_cert_installed(self): """Test line from our cert exists.""" diff --git a/tests/cloud_tests/testcases/modules/ca_certs.yaml b/tests/cloud_tests/testcases/modules/ca_certs.yaml index d939f435..2cd91551 100644 --- a/tests/cloud_tests/testcases/modules/ca_certs.yaml +++ b/tests/cloud_tests/testcases/modules/ca_certs.yaml @@ -43,9 +43,13 @@ cloud_config: | DiH5uEqBXExjrj0FslxcVKdVj5glVcSmkLwZKbEU1OKwleT/iXFhvooWhQ== -----END CERTIFICATE----- collect_scripts: - cert_count: | + cert_links: | #!/bin/bash - ls -l /etc/ssl/certs | wc -l + # links printed -> + # non-links printed + for file in `ls /etc/ssl/certs`; do + [ -h /etc/ssl/certs/$file ] && echo -n $file ' -> ' && readlink /etc/ssl/certs/$file || echo $file; + done cert: | #!/bin/bash md5sum /etc/ssl/certs/ca-certificates.crt diff --git a/tests/cloud_tests/testcases/modules/ntp.py b/tests/cloud_tests/testcases/modules/ntp.py index b50e52fe..c63cc15e 100644 --- a/tests/cloud_tests/testcases/modules/ntp.py +++ b/tests/cloud_tests/testcases/modules/ntp.py @@ -9,15 +9,14 @@ class TestNtp(base.CloudTestCase): def test_ntp_installed(self): """Test ntp installed""" - out = self.get_data_file('ntp_installed') - self.assertEqual(0, int(out)) + self.assertPackageInstalled('ntp') def test_ntp_dist_entries(self): """Test dist config file is empty""" out = self.get_data_file('ntp_conf_dist_empty') self.assertEqual(0, int(out)) - def test_ntp_entires(self): + def test_ntp_entries(self): """Test config entries""" out = self.get_data_file('ntp_conf_pool_list') self.assertIn('pool.ntp.org iburst', out) diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py index a92dec22..fecad768 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.py @@ -7,15 +7,13 @@ from tests.cloud_tests.testcases import base class TestPackageInstallUpdateUpgrade(base.CloudTestCase): """Test package install update upgrade module.""" - def test_installed_htop(self): - """Test htop got installed.""" - out = self.get_data_file('dpkg_htop') - self.assertEqual(1, int(out)) + def test_installed_sl(self): + """Test sl got installed.""" + self.assertPackageInstalled('sl') def test_installed_tree(self): """Test tree got installed.""" - out = self.get_data_file('dpkg_tree') - self.assertEqual(1, int(out)) + self.assertPackageInstalled('tree') def test_apt_history(self): """Test apt history for update command.""" @@ -23,13 +21,13 @@ class TestPackageInstallUpdateUpgrade(base.CloudTestCase): self.assertIn( 'Commandline: /usr/bin/apt-get --option=Dpkg::Options' '::=--force-confold --option=Dpkg::options::=--force-unsafe-io ' - '--assume-yes --quiet install htop tree', out) + '--assume-yes --quiet install sl tree', out) def test_cloud_init_output(self): """Test cloud-init-output for install & upgrade stuff.""" out = self.get_data_file('cloud-init-output.log') self.assertIn('Setting up tree (', out) - self.assertIn('Setting up htop (', out) + self.assertIn('Setting up sl (', out) self.assertIn('Reading package lists...', out) self.assertIn('Building dependency tree...', out) self.assertIn('Reading state information...', out) diff --git a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml index 71d24b83..dd79e438 100644 --- a/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml +++ b/tests/cloud_tests/testcases/modules/package_update_upgrade_install.yaml @@ -15,7 +15,7 @@ required_features: cloud_config: | #cloud-config packages: - - htop + - sl - tree package_update: true package_upgrade: true @@ -23,11 +23,8 @@ collect_scripts: apt_history_cmdline: | #!/bin/bash grep ^Commandline: /var/log/apt/history.log - dpkg_htop: | + dpkg_show: | #!/bin/bash - dpkg -l | grep htop | wc -l - dpkg_tree: | - #!/bin/bash - dpkg -l | grep tree | wc -l + dpkg-query --show # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py index 70917a4c..fc9688ed 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ b/tests/cloud_tests/testcases/modules/salt_minion.py @@ -33,7 +33,6 @@ class Test(base.CloudTestCase): def test_minion_installed(self): """Test if the salt-minion package is installed""" - out = self.get_data_file('minion_installed') - self.assertEqual(1, int(out)) + self.assertPackageInstalled('salt-minion') # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml index f20b9765..c24aa177 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml @@ -35,8 +35,5 @@ collect_scripts: grains: | #!/bin/bash cat /etc/salt/grains - minion_installed: | - #!/bin/bash - dpkg -l | grep salt-minion | grep ii | wc -l # vi: ts=4 expandtab -- cgit v1.2.3 From 5ffcb511db8783aa9d32895f7017a7278d546f2f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 12 Jun 2018 11:54:30 -0600 Subject: tests: skip chrony integration tests on lxd running artful or older A fix for chrony support per LP: #1589780 is not expected in Artful or older series. Skip the chrony suite of tests when running on a container and ubuntu series represented is <= artful as errors are expected. --- tests/cloud_tests/testcases/modules/ntp_chrony.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py index 461630a8..7d341773 100644 --- a/tests/cloud_tests/testcases/modules/ntp_chrony.py +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -1,13 +1,24 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" +import unittest + from tests.cloud_tests.testcases import base class TestNtpChrony(base.CloudTestCase): """Test ntp module with chrony client""" - def test_chrony_entires(self): + def setUp(self): + """Skip this suite of tests on lxd and artful or older.""" + if self.platform == 'lxd': + if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: + raise unittest.SkipTest( + 'No support for chrony on containers <= artful.' + ' LP: #1589780') + return super(TestNtpChrony, self).setUp() + + def test_chrony_entries(self): """Test chrony config entries""" out = self.get_data_file('chrony_conf') self.assertIn('.pool.ntp.org', out) -- cgit v1.2.3 From 27283c31f4bf85f40588cfa3b31389d70ec00243 Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Tue, 12 Jun 2018 16:42:54 -0600 Subject: tests: fix salt_minion integration test on bionic and later In ubuntu, the salt-minion package version 2017.7.4+dfsg1-1 or later automatically moves any seed keys from /etc/salt/pki/minion/ to /var/lib/salt/pki/minion/. Fix integration tests to collect either files in either /etc/salt/pki/minion/ or /var/lib/salt/pki/minion/. --- tests/cloud_tests/testcases/modules/salt_minion.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml index c24aa177..9227147c 100644 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml @@ -28,10 +28,20 @@ collect_scripts: cat /etc/salt/minion_id minion.pem: | #!/bin/bash - cat /etc/salt/pki/minion/minion.pem + PRIV_KEYFILE=/etc/salt/pki/minion/minion.pem + if [ ! -f $PRIV_KEYFILE ]; then + # Bionic and later automatically moves /etc/salt/pki/minion/* + PRIV_KEYFILE=/var/lib/salt/pki/minion/minion.pem + fi + cat $PRIV_KEYFILE minion.pub: | #!/bin/bash - cat /etc/salt/pki/minion/minion.pub + PUB_KEYFILE=/etc/salt/pki/minion/minion.pub + if [ ! -f $PUB_KEYFILE ]; then + # Bionic and later automatically moves /etc/salt/pki/minion/* + PUB_KEYFILE=/var/lib/salt/pki/minion/minion.pub + fi + cat $PUB_KEYFILE grains: | #!/bin/bash cat /etc/salt/grains -- cgit v1.2.3 From 5858136215753cffc2080c85429a3c7a3754741c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 28 Jun 2018 17:22:33 -0400 Subject: tests: drop salt-minion integration test The salt minion integration test as we had it did not do a whole lot more than the unit tests on that module did. Additionally, it caused some transient failures at least in Ubuntu 18.04. At a future date we may choose to add an integration test that installs salt-minion and salt server and configures it to be a better test. LP: #1778737 --- tests/cloud_tests/testcases/modules/salt_minion.py | 38 ----------------- .../cloud_tests/testcases/modules/salt_minion.yaml | 49 ---------------------- 2 files changed, 87 deletions(-) delete mode 100644 tests/cloud_tests/testcases/modules/salt_minion.py delete mode 100644 tests/cloud_tests/testcases/modules/salt_minion.yaml (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py deleted file mode 100644 index fc9688ed..00000000 --- a/tests/cloud_tests/testcases/modules/salt_minion.py +++ /dev/null @@ -1,38 +0,0 @@ -# This file is part of cloud-init. See LICENSE file for license information. - -"""cloud-init Integration Test Verify Script.""" -from tests.cloud_tests.testcases import base - - -class Test(base.CloudTestCase): - """Test salt minion module.""" - - def test_minon_master(self): - """Test master value in config.""" - out = self.get_data_file('minion') - self.assertIn('master: salt.mydomain.com', out) - - def test_minion_pem(self): - """Test private key.""" - out = self.get_data_file('minion.pem') - self.assertIn('------BEGIN PRIVATE KEY------', out) - self.assertIn('', out) - self.assertIn('------END PRIVATE KEY-------', out) - - def test_minion_pub(self): - """Test public key.""" - out = self.get_data_file('minion.pub') - self.assertIn('------BEGIN PUBLIC KEY-------', out) - self.assertIn('', out) - self.assertIn('------END PUBLIC KEY-------', out) - - def test_grains(self): - """Test master value in config.""" - out = self.get_data_file('grains') - self.assertIn('role: web', out) - - def test_minion_installed(self): - """Test if the salt-minion package is installed""" - self.assertPackageInstalled('salt-minion') - -# vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml deleted file mode 100644 index 9227147c..00000000 --- a/tests/cloud_tests/testcases/modules/salt_minion.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# -# Create config for a salt minion -# -# 2016-11-17: Currently takes >60 seconds results in test failure -# -enabled: True -cloud_config: | - #cloud-config - salt_minion: - conf: - master: salt.mydomain.com - public_key: | - ------BEGIN PUBLIC KEY------- - - ------END PUBLIC KEY------- - private_key: | - ------BEGIN PRIVATE KEY------ - - ------END PRIVATE KEY------- - grains: - role: web -collect_scripts: - minion: | - #!/bin/bash - cat /etc/salt/minion - minion_id: | - #!/bin/bash - cat /etc/salt/minion_id - minion.pem: | - #!/bin/bash - PRIV_KEYFILE=/etc/salt/pki/minion/minion.pem - if [ ! -f $PRIV_KEYFILE ]; then - # Bionic and later automatically moves /etc/salt/pki/minion/* - PRIV_KEYFILE=/var/lib/salt/pki/minion/minion.pem - fi - cat $PRIV_KEYFILE - minion.pub: | - #!/bin/bash - PUB_KEYFILE=/etc/salt/pki/minion/minion.pub - if [ ! -f $PUB_KEYFILE ]; then - # Bionic and later automatically moves /etc/salt/pki/minion/* - PUB_KEYFILE=/var/lib/salt/pki/minion/minion.pub - fi - cat $PUB_KEYFILE - grains: | - #!/bin/bash - cat /etc/salt/grains - -# vi: ts=4 expandtab -- cgit v1.2.3 From a03ed8bf77add3ed920137e4cc6986caa414d1da Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Mon, 20 Aug 2018 18:29:26 +0000 Subject: tests: disable snap, fix write_files binary The snap test requires access to a proxy and currently the integration tests do not handle this scenario. I am disabling the test untill I can loop back around and fix this. The write_files test, specifically, the binary test is failing on cosmic because the "binary" file we were writting was not a complete elf executable, but we expected 'file' to identify it as such. The change here is to simply use some 24 bytes of random, non-utf data and check that file was written correctly via expected checksum. --- tests/cloud_tests/testcases/modules/snappy.yaml | 3 +++ tests/cloud_tests/testcases/modules/write_files.py | 7 +++++-- tests/cloud_tests/testcases/modules/write_files.yaml | 15 +++++++++++---- 3 files changed, 19 insertions(+), 6 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/snappy.yaml b/tests/cloud_tests/testcases/modules/snappy.yaml index 43f93295..8ac322ae 100644 --- a/tests/cloud_tests/testcases/modules/snappy.yaml +++ b/tests/cloud_tests/testcases/modules/snappy.yaml @@ -1,6 +1,9 @@ # # Install snappy # +# Aug 17, 2018: Disabled due to requiring a proxy for testing +# tests do not handle the proxy well at this time. +enabled: False required_features: - snap cloud_config: | diff --git a/tests/cloud_tests/testcases/modules/write_files.py b/tests/cloud_tests/testcases/modules/write_files.py index 7bd520f6..526a2ebd 100644 --- a/tests/cloud_tests/testcases/modules/write_files.py +++ b/tests/cloud_tests/testcases/modules/write_files.py @@ -14,8 +14,11 @@ class TestWriteFiles(base.CloudTestCase): def test_binary(self): """Test binary file reads as executable.""" - out = self.get_data_file('file_binary') - self.assertIn('ELF 64-bit LSB executable, x86-64, version 1', out) + out = self.get_data_file('file_binary').strip() + md5 = "3801184b97bb8c6e63fa0e1eae2920d7" + sha256 = ("2c791c4037ea5bd7e928d6a87380f8ba7a803cd83d" + "5e4f269e28f5090f0f2c9a") + self.assertIn(out, (md5 + " -", sha256 + " -")) def test_gzip(self): """Test gzip file shows up as a shell script.""" diff --git a/tests/cloud_tests/testcases/modules/write_files.yaml b/tests/cloud_tests/testcases/modules/write_files.yaml index ce936b7b..cc7ea4bd 100644 --- a/tests/cloud_tests/testcases/modules/write_files.yaml +++ b/tests/cloud_tests/testcases/modules/write_files.yaml @@ -3,6 +3,13 @@ # # NOTE: on trusty 'file' has an output formatting error for binary files and # has 2 spaces in 'LSB executable', which causes a failure here +# +# NOTE: the binary data can be any binary data, not only executables +# and can be generated via the base 64 command as such: +# $ base64 < hello > hello.txt +# the opposite is running: +# $ base64 -d < hello.txt > hello +# required_features: - no_file_fmt_e cloud_config: | @@ -19,9 +26,7 @@ cloud_config: | SMBDOPTIONS="-D" path: /root/file_text - content: !!binary | - f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAAAAAAAAEAAOAAI - AEAAHgAdAAYAAAAFAAAAQAAAAAAAAABAAEAAAAAAAEAAQAAAAAAAwAEAAAAAAADAAQAAAAAAAAgA - AAAAAAAAAwAAAAQAAAAAAgAAAAAAAAACQAAAAAAAAAJAAAAAAAAcAAAAAAAAABwAAAAAAAAAAQAA + /Z/xrHR4WINT0UNoKPQKbuovp6+Js+JK path: /root/file_binary permissions: '0555' - encoding: gzip @@ -38,7 +43,9 @@ collect_scripts: file /root/file_text file_binary: | #!/bin/bash - file /root/file_binary + for hasher in md5sum sha256sum; do + $hasher Date: Thu, 23 Aug 2018 18:13:25 +0000 Subject: tests: disable other snap test as well The snap test requires access to a proxy and currently the integration tests do not handle this scenario. I am disabling the test until I can loop back around and fix this. --- tests/cloud_tests/testcases/modules/snap.yaml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/snap.yaml b/tests/cloud_tests/testcases/modules/snap.yaml index 44043f31..322199c3 100644 --- a/tests/cloud_tests/testcases/modules/snap.yaml +++ b/tests/cloud_tests/testcases/modules/snap.yaml @@ -1,6 +1,9 @@ # # Install snappy # +# Aug 23, 2018: Disabled due to requiring a proxy for testing +# tests do not handle the proxy well at this time. +enabled: False required_features: - snap cloud_config: | -- cgit v1.2.3 From facb92c5b55f4d27ee971eaf49c580fbf92ac2c3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 26 Sep 2018 21:29:27 +0000 Subject: lxd: adjust to snap installed lxd. Relax expectation on path to lxc and lxd. The deb path still does install them in /usr/bin/ but that is overly pedantic. Add a 'lxd waitready' (present since lxd 0.5) to wait until lxd is ready before operating on it. --- cloudinit/config/cc_lxd.py | 5 ++++- tests/cloud_tests/testcases/modules/lxd_bridge.py | 4 ++-- tests/cloud_tests/testcases/modules/lxd_dir.py | 4 ++-- tests/unittests/test_handler/test_handler_lxd.py | 12 ++++++------ 4 files changed, 14 insertions(+), 11 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index a604825a..24a8ebea 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -104,6 +104,7 @@ def handle(name, cfg, cloud, log, args): 'network_address', 'network_port', 'storage_backend', 'storage_create_device', 'storage_create_loop', 'storage_pool', 'trust_password') + util.subp(['lxd', 'waitready', '--timeout=300']) cmd = ['lxd', 'init', '--auto'] for k in init_keys: if init_cfg.get(k): @@ -260,7 +261,9 @@ def bridge_to_cmd(bridge_cfg): def _lxc(cmd): - env = {'LC_ALL': 'C'} + env = {'LC_ALL': 'C', + 'HOME': os.environ.get('HOME', '/root'), + 'USER': os.environ.get('USER', 'root')} util.subp(['lxc'] + list(cmd) + ["--force-local"], update_env=env) diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py index c0262ba3..f6011dee 100644 --- a/tests/cloud_tests/testcases/modules/lxd_bridge.py +++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py @@ -10,12 +10,12 @@ class TestLxdBridge(base.CloudTestCase): def test_lxd(self): """Test lxd installed.""" out = self.get_data_file('lxd') - self.assertIn('/usr/bin/lxd', out) + self.assertIn('/lxd', out) def test_lxc(self): """Test lxc installed.""" out = self.get_data_file('lxc') - self.assertIn('/usr/bin/lxc', out) + self.assertIn('/lxc', out) def test_bridge(self): """Test bridge config.""" diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py index 1495674e..26a3db39 100644 --- a/tests/cloud_tests/testcases/modules/lxd_dir.py +++ b/tests/cloud_tests/testcases/modules/lxd_dir.py @@ -10,11 +10,11 @@ class TestLxdDir(base.CloudTestCase): def test_lxd(self): """Test lxd installed.""" out = self.get_data_file('lxd') - self.assertIn('/usr/bin/lxd', out) + self.assertIn('/lxd', out) def test_lxc(self): """Test lxc installed.""" out = self.get_data_file('lxc') - self.assertIn('/usr/bin/lxc', out) + self.assertIn('/lxc', out) # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 4dd7e09f..2478ebc4 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -43,12 +43,12 @@ class TestLxd(t_help.CiTestCase): self.assertTrue(mock_util.which.called) # no bridge config, so maybe_cleanup should not be called. self.assertFalse(m_maybe_clean.called) - init_call = mock_util.subp.call_args_list[0][0][0] - self.assertEqual(init_call, - ['lxd', 'init', '--auto', - '--network-address=0.0.0.0', - '--storage-backend=zfs', - '--storage-pool=poolname']) + self.assertEqual( + [mock.call(['lxd', 'waitready', '--timeout=300']), + mock.call( + ['lxd', 'init', '--auto', '--network-address=0.0.0.0', + '--storage-backend=zfs', '--storage-pool=poolname'])], + mock_util.subp.call_args_list) @mock.patch("cloudinit.config.cc_lxd.maybe_cleanup_default") @mock.patch("cloudinit.config.cc_lxd.util") -- cgit v1.2.3 From fb5015fe3f5078149463838b7974523646f94eab Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 27 Sep 2018 21:02:28 +0000 Subject: cii-tests: use unittest2.SkipTest in ntp_chrony due to new deps Git commitish fc4b966ba928b30b1c586407e752e0b51b1031e8 changed integration test dependencies from unittest to unittest2. Use unittest2.SkipTest in test_chrony to avoid causing tracebacks. --- tests/cloud_tests/testcases/modules/ntp_chrony.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/ntp_chrony.py b/tests/cloud_tests/testcases/modules/ntp_chrony.py index 7d341773..0f4c3d08 100644 --- a/tests/cloud_tests/testcases/modules/ntp_chrony.py +++ b/tests/cloud_tests/testcases/modules/ntp_chrony.py @@ -1,7 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """cloud-init Integration Test Verify Script.""" -import unittest +import unittest2 from tests.cloud_tests.testcases import base @@ -13,7 +13,7 @@ class TestNtpChrony(base.CloudTestCase): """Skip this suite of tests on lxd and artful or older.""" if self.platform == 'lxd': if self.is_distro('ubuntu') and self.os_version_cmp('artful') <= 0: - raise unittest.SkipTest( + raise unittest2.SkipTest( 'No support for chrony on containers <= artful.' ' LP: #1589780') return super(TestNtpChrony, self).setUp() -- cgit v1.2.3 From c239eb82f1bfe1d0c4b79f2a733f04cfb3277fbf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 28 Sep 2018 20:47:28 +0000 Subject: tests: disable lxd tests on cosmic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip lxd tests on cosmic for two reasons: a.) bug 1795036 - 'lxd init' fails on cosmic kernel. b.) apt install lxd installs via snap which can be slow     as that will download core snap and lxd. --- tests/cloud_tests/testcases/modules/lxd_bridge.py | 9 +++++++++ tests/cloud_tests/testcases/modules/lxd_dir.py | 9 +++++++++ 2 files changed, 18 insertions(+) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py index f6011dee..8697ae99 100644 --- a/tests/cloud_tests/testcases/modules/lxd_bridge.py +++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py @@ -7,6 +7,15 @@ from tests.cloud_tests.testcases import base class TestLxdBridge(base.CloudTestCase): """Test LXD module.""" + def setUp(self): + """Skip on cosmic for two reasons: + a.) LP: #1795036 - 'lxd init' fails on cosmic kernel. + b.) apt install lxd installs via snap which can be slow + as that will download core snap and lxd.""" + if self.os_name == "cosmic": + raise self.skipTest('Skipping test on cosmic (LP: #1795036).') + return base.CloudTestCase.setUp(self) + def test_lxd(self): """Test lxd installed.""" out = self.get_data_file('lxd') diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py index 26a3db39..8bd8c6e6 100644 --- a/tests/cloud_tests/testcases/modules/lxd_dir.py +++ b/tests/cloud_tests/testcases/modules/lxd_dir.py @@ -7,6 +7,15 @@ from tests.cloud_tests.testcases import base class TestLxdDir(base.CloudTestCase): """Test LXD module.""" + def setUp(self): + """Skip on cosmic for two reasons: + a.) LP: #1795036 - 'lxd init' fails on cosmic kernel. + b.) apt install lxd installs via snap which can be slow + as that will download core snap and lxd.""" + if self.os_name == "cosmic": + raise self.skipTest('Skipping test on cosmic (LP: #1795036).') + return base.CloudTestCase.setUp(self) + def test_lxd(self): """Test lxd installed.""" out = self.get_data_file('lxd') -- cgit v1.2.3 From 40768bad4c7eb966b2a8c1308cdae22e578b2413 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 1 Oct 2018 18:23:34 +0000 Subject: tests: allow skipping an entire cloud_test without running. Individual skipTest or setUp SkipTest will still launch the instance. This allows us to stop the running of the instance so we don't waste cycles or boot systems that are known to fail. Also replace remaining unittest usage in tests/cloud_tests/ with unittest2. --- tests/cloud_tests/collect.py | 12 +++++ tests/cloud_tests/testcases/__init__.py | 58 ++++++++++++++--------- tests/cloud_tests/testcases/base.py | 5 ++ tests/cloud_tests/testcases/modules/lxd_bridge.py | 9 ++-- tests/cloud_tests/testcases/modules/lxd_dir.py | 9 ++-- tests/cloud_tests/verify.py | 4 +- 6 files changed, 64 insertions(+), 33 deletions(-) (limited to 'tests/cloud_tests/testcases/modules') diff --git a/tests/cloud_tests/collect.py b/tests/cloud_tests/collect.py index 75b50616..642745d8 100644 --- a/tests/cloud_tests/collect.py +++ b/tests/cloud_tests/collect.py @@ -9,6 +9,7 @@ from cloudinit import util as c_util from tests.cloud_tests import (config, LOG, setup_image, util) from tests.cloud_tests.stage import (PlatformComponent, run_stage, run_single) from tests.cloud_tests import platforms +from tests.cloud_tests.testcases import base, get_test_class def collect_script(instance, base_dir, script, script_name): @@ -63,6 +64,7 @@ def collect_test_data(args, snapshot, os_name, test_name): res = ({}, 1) # load test config + test_name_in = test_name test_name = config.path_to_name(test_name) test_config = config.load_test_config(test_name) user_data = test_config['cloud_config'] @@ -75,6 +77,16 @@ def collect_test_data(args, snapshot, os_name, test_name): LOG.warning('test config %s is not enabled, skipping', test_name) return ({}, 0) + test_class = get_test_class( + config.name_to_module(test_name_in), + test_data={'platform': snapshot.platform_name, 'os_name': os_name}, + test_conf=test_config['cloud_config']) + try: + test_class.maybeSkipTest() + except base.SkipTest as s: + LOG.warning('skipping test config %s: %s', test_name, s) + return ({}, 0) + # if testcase requires a feature flag that the image does not support, # skip the testcase with a warning req_features = test_config.get('required_features', []) diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py index bd548f5a..6bb39f77 100644 --- a/tests/cloud_tests/testcases/__init__.py +++ b/tests/cloud_tests/testcases/__init__.py @@ -4,8 +4,7 @@ import importlib import inspect -import unittest -from unittest.util import strclass +import unittest2 from cloudinit.util import read_conf @@ -13,7 +12,7 @@ from tests.cloud_tests import config from tests.cloud_tests.testcases.base import CloudTestCase as base_test -def discover_tests(test_name): +def discover_test(test_name): """Discover tests in test file for 'testname'. @return_value: list of test classes @@ -25,35 +24,48 @@ def discover_tests(test_name): except NameError: raise ValueError('no test verifier found at: {}'.format(testmod_name)) - return [mod for name, mod in inspect.getmembers(testmod) - if inspect.isclass(mod) and base_test in inspect.getmro(mod) and - getattr(mod, '__test__', True)] + found = [mod for name, mod in inspect.getmembers(testmod) + if (inspect.isclass(mod) + and base_test in inspect.getmro(mod) + and getattr(mod, '__test__', True))] + if len(found) != 1: + raise RuntimeError( + "Unexpected situation, multiple tests for %s: %s" % ( + test_name, found)) + return found -def get_suite(test_name, data, conf): - """Get test suite with all tests for 'testname'. - @return_value: a test suite - """ - suite = unittest.TestSuite() - for test_class in discover_tests(test_name): +def get_test_class(test_name, test_data, test_conf): + test_class = discover_test(test_name)[0] + + class DynamicTestSubclass(test_class): - class tmp(test_class): + _realclass = test_class + data = test_data + conf = test_conf + release_conf = read_conf(config.RELEASES_CONF)['releases'] - _realclass = test_class + def __str__(self): + return "%s (%s)" % (self._testMethodName, + unittest2.util.strclass(self._realclass)) - def __str__(self): - return "%s (%s)" % (self._testMethodName, - strclass(self._realclass)) + @classmethod + def setUpClass(cls): + cls.maybeSkipTest() - @classmethod - def setUpClass(cls): - cls.data = data - cls.conf = conf - cls.release_conf = read_conf(config.RELEASES_CONF)['releases'] + return DynamicTestSubclass - suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(tmp)) +def get_suite(test_name, data, conf): + """Get test suite with all tests for 'testname'. + + @return_value: a test suite + """ + suite = unittest2.TestSuite() + suite.addTest( + unittest2.defaultTestLoader.loadTestsFromTestCase( + get_test_class(test_name, data, conf))) return suite # vi: ts=4 expandtab diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py index e83cb9a6..e18d601c 100644 --- a/tests/cloud_tests/testcases/base.py +++ b/tests/cloud_tests/testcases/base.py @@ -31,6 +31,11 @@ class CloudTestCase(unittest2.TestCase): def is_distro(self, distro_name): return self.os_cfg['os'] == distro_name + @classmethod + def maybeSkipTest(cls): + """Present to allow subclasses to override and raise a skipTest.""" + pass + def assertPackageInstalled(self, name, version=None): """Check dpkg-query --show output for matching package name. diff --git a/tests/cloud_tests/testcases/modules/lxd_bridge.py b/tests/cloud_tests/testcases/modules/lxd_bridge.py index 8697ae99..ea545e0a 100644 --- a/tests/cloud_tests/testcases/modules/lxd_bridge.py +++ b/tests/cloud_tests/testcases/modules/lxd_bridge.py @@ -7,14 +7,15 @@ from tests.cloud_tests.testcases import base class TestLxdBridge(base.CloudTestCase): """Test LXD module.""" - def setUp(self): + @classmethod + def maybeSkipTest(cls): """Skip on cosmic for two reasons: a.) LP: #1795036 - 'lxd init' fails on cosmic kernel. b.) apt install lxd installs via snap which can be slow as that will download core snap and lxd.""" - if self.os_name == "cosmic": - raise self.skipTest('Skipping test on cosmic (LP: #1795036).') - return base.CloudTestCase.setUp(self) + os_name = cls.data.get('os_name', 'UNKNOWN') + if os_name == "cosmic": + raise base.SkipTest('Skipping test on cosmic (LP: #1795036).') def test_lxd(self): """Test lxd installed.""" diff --git a/tests/cloud_tests/testcases/modules/lxd_dir.py b/tests/cloud_tests/testcases/modules/lxd_dir.py index 8bd8c6e6..797bafed 100644 --- a/tests/cloud_tests/testcases/modules/lxd_dir.py +++ b/tests/cloud_tests/testcases/modules/lxd_dir.py @@ -7,14 +7,15 @@ from tests.cloud_tests.testcases import base class TestLxdDir(base.CloudTestCase): """Test LXD module.""" - def setUp(self): + @classmethod + def maybeSkipTest(cls): """Skip on cosmic for two reasons: a.) LP: #1795036 - 'lxd init' fails on cosmic kernel. b.) apt install lxd installs via snap which can be slow as that will download core snap and lxd.""" - if self.os_name == "cosmic": - raise self.skipTest('Skipping test on cosmic (LP: #1795036).') - return base.CloudTestCase.setUp(self) + os_name = cls.data.get('os_name', 'UNKNOWN') + if os_name == "cosmic": + raise base.SkipTest('Skipping test on cosmic (LP: #1795036).') def test_lxd(self): """Test lxd installed.""" diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py index bfb27444..9911ecf2 100644 --- a/tests/cloud_tests/verify.py +++ b/tests/cloud_tests/verify.py @@ -3,7 +3,7 @@ """Verify test results.""" import os -import unittest +import unittest2 from tests.cloud_tests import (config, LOG, util, testcases) @@ -18,7 +18,7 @@ def verify_data(data_dir, platform, os_name, tests): @return_value: {: {passed: True/False, failures: []}} """ base_dir = os.sep.join((data_dir, platform, os_name)) - runner = unittest.TextTestRunner(verbosity=util.current_verbosity()) + runner = unittest2.TextTestRunner(verbosity=util.current_verbosity()) res = {} for test_name in tests: LOG.debug('verifying test data for %s', test_name) -- cgit v1.2.3