From bbe91cdc6917adb503b455e6860c21ea7b3f567f Mon Sep 17 00:00:00 2001 From: Ryan McCabe Date: Mon, 20 Nov 2017 18:16:00 -0500 Subject: sysconfig: Correctly render dns and dns search info. Currently when dns and dns search info is provided, it is not rendered when outputting to sysconfig format. This patch causes the DNS and DOMAIN lines to be written out rendering sysconfig. LP: #1705804 --- cloudinit/net/network_state.py | 8 ++++++++ cloudinit/net/sysconfig.py | 15 +++++++++++++++ tests/unittests/test_net.py | 6 ++++++ 3 files changed, 29 insertions(+) diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 0e830ee8..e9e2cf4e 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -746,6 +746,14 @@ def _normalize_subnet(subnet): _normalize_net_keys(normal_subnet, address_keys=('address',))) normal_subnet['routes'] = [_normalize_route(r) for r in subnet.get('routes', [])] + + def listify(snet, name): + if name in snet and not isinstance(snet[name], list): + snet[name] = snet[name].split() + + for k in ('dns_search', 'dns_nameservers'): + listify(normal_subnet, k) + return normal_subnet diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py index f5727969..39d89c46 100644 --- a/cloudinit/net/sysconfig.py +++ b/cloudinit/net/sysconfig.py @@ -7,12 +7,15 @@ import six from cloudinit.distros.parsers import networkmanager_conf from cloudinit.distros.parsers import resolv_conf +from cloudinit import log as logging from cloudinit import util from . import renderer from .network_state import ( is_ipv6_addr, net_prefix_to_ipv4_mask, subnet_is_ipv6) +LOG = logging.getLogger(__name__) + def _make_header(sep='#'): lines = [ @@ -347,6 +350,18 @@ class Renderer(renderer.Renderer): else: iface_cfg['GATEWAY'] = subnet['gateway'] + if 'dns_search' in subnet: + iface_cfg['DOMAIN'] = ' '.join(subnet['dns_search']) + + if 'dns_nameservers' in subnet: + if len(subnet['dns_nameservers']) > 3: + # per resolv.conf(5) MAXNS sets this to 3. + LOG.debug("%s has %d entries in dns_nameservers. " + "Only 3 are used.", iface_cfg.name, + len(subnet['dns_nameservers'])) + for i, k in enumerate(subnet['dns_nameservers'][:3], 1): + iface_cfg['DNS' + str(i)] = k + @classmethod def _render_subnet_routes(cls, iface_cfg, route_cfg, subnets): for i, subnet in enumerate(subnets, start=len(iface_cfg.children)): diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index bbb63cb3..f3fa2a30 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -436,6 +436,9 @@ NETWORK_CONFIGS = { BOOTPROTO=dhcp DEFROUTE=yes DEVICE=eth99 + DNS1=8.8.8.8 + DNS2=8.8.4.4 + DOMAIN="barley.maas sach.maas" GATEWAY=65.61.151.37 HWADDR=c0:d6:9f:2c:e8:80 IPADDR=192.168.21.3 @@ -836,6 +839,9 @@ pre-down route del -net 10.0.0.0 netmask 255.0.0.0 gw 11.0.0.1 metric 3 || true BOOTPROTO=none DEFROUTE=yes DEVICE=eth0.101 + DNS1=192.168.0.10 + DNS2=10.23.23.134 + DOMAIN="barley.maas sacchromyces.maas brettanomyces.maas" GATEWAY=192.168.0.1 IPADDR=192.168.0.2 IPADDR1=192.168.2.10 -- cgit v1.2.3 From 9ac735bb8c0dec5628a33d907adb3fc02fd902e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 21 Nov 2017 14:23:36 -0500 Subject: tests: Use apt-get to install a deb so that depends get resolved. Instead of using 'dpkg -i' to install a package and then running apt-get -f install, to hope that it would install needed dependencies we can just use 'apt-get' directly to do the install. The 'dpkg/apt-get -f' path was a problem if the installed deb was older than the available deb. In that case it would get replaced. --- tests/cloud_tests/setup_image.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py index 6672ffb3..179f40db 100644 --- a/tests/cloud_tests/setup_image.py +++ b/tests/cloud_tests/setup_image.py @@ -50,9 +50,9 @@ def install_deb(args, image): LOG.debug(msg) remote_path = os.path.join('/tmp', os.path.basename(args.deb)) image.push_file(args.deb, remote_path) - cmd = 'dpkg -i {}; apt-get install --yes -f'.format(remote_path) - image.execute(cmd, description=msg) - + image.execute( + ['apt-get', 'install', '--allow-downgrades', '--assume-yes', + remote_path], description=msg) # check installed deb version matches package fmt = ['-W', "--showformat=${Version}"] (out, err, exit) = image.execute(['dpkg-deb'] + fmt + [remote_path]) -- cgit v1.2.3 From 4964fb38f11c15ed119ff4c7f4379ae3c8785a9a Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Wed, 22 Nov 2017 11:12:05 -0800 Subject: tests: Enable bionic in integration tests. --- tests/cloud_tests/releases.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml index ec7e2d5b..e5933802 100644 --- a/tests/cloud_tests/releases.yaml +++ b/tests/cloud_tests/releases.yaml @@ -122,6 +122,22 @@ features: releases: # UBUNTU ================================================================= + bionic: + # EOL: Apr 2023 + default: + enabled: true + release: bionic + version: 18.04 + family: ubuntu + feature_groups: + - base + - debian_base + - ubuntu_specific + lxd: + sstreams_server: https://cloud-images.ubuntu.com/daily + alias: bionic + setup_overrides: null + override_templates: false artful: # EOL: Jul 2018 default: -- cgit v1.2.3 From 88368f9851b29dddb5a12e4b21868cbdef906c5c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 29 Nov 2017 15:26:38 -0500 Subject: tests: NoCloudKVMImage do not modify the original local cache image. The NoCloudKVMImage.execute() would modify the image in /srv/citest that meant that after the first time you ran a test, the image was dirty. The change here is to make the image operate on a qcow backed image. Also modify Snapshot to then copy the qcow rather than creating another chained qcow. The reason being that the image might go away or change after the snapshot has been returned. Also * drop use of 'override_templates' which was only relevant to LXD. * NoCloudKVM.create_image() returned an instance before now it has create_instance which creates an instance. * NoCloudKVMInstance has a 'disk' attribute separate from 'name' --- tests/cloud_tests/images/nocloudkvm.py | 22 +++++++++++++++------- tests/cloud_tests/instances/nocloudkvm.py | 8 +++++--- tests/cloud_tests/platforms/nocloudkvm.py | 21 +++++++++++---------- tests/cloud_tests/snapshots/nocloudkvm.py | 17 +++++++++++------ 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/tests/cloud_tests/images/nocloudkvm.py b/tests/cloud_tests/images/nocloudkvm.py index 1e7962cb..8678b07f 100644 --- a/tests/cloud_tests/images/nocloudkvm.py +++ b/tests/cloud_tests/images/nocloudkvm.py @@ -4,6 +4,10 @@ from cloudinit import util as c_util +import os +import shutil +import tempfile + from tests.cloud_tests.images import base from tests.cloud_tests.snapshots import nocloudkvm as nocloud_kvm_snapshot @@ -13,7 +17,7 @@ class NoCloudKVMImage(base.Image): platform_name = "nocloud-kvm" - def __init__(self, platform, config, img_path): + def __init__(self, platform, config, orig_img_path): """Set up image. @param platform: platform object @@ -21,7 +25,13 @@ class NoCloudKVMImage(base.Image): @param img_path: path to the image """ self.modified = False - self._img_path = img_path + self._workd = tempfile.mkdtemp(prefix='NoCloudKVMImage') + self._orig_img_path = orig_img_path + self._img_path = os.path.join(self._workd, + os.path.basename(self._orig_img_path)) + + c_util.subp(['qemu-img', 'create', '-f', 'qcow2', + '-b', orig_img_path, self._img_path]) super(NoCloudKVMImage, self).__init__(platform, config) @@ -61,13 +71,9 @@ class NoCloudKVMImage(base.Image): if not self._img_path: raise RuntimeError() - instance = self.platform.create_image( - self.properties, self.config, self.features, - self._img_path, image_desc=str(self), use_desc='snapshot') - return nocloud_kvm_snapshot.NoCloudKVMSnapshot( self.platform, self.properties, self.config, - self.features, instance) + self.features, self._img_path) def destroy(self): """Unset path to signal image is no longer used. @@ -77,6 +83,8 @@ class NoCloudKVMImage(base.Image): framework decide whether to keep or destroy everything. """ self._img_path = None + shutil.rmtree(self._workd) + super(NoCloudKVMImage, self).destroy() # vi: ts=4 expandtab diff --git a/tests/cloud_tests/instances/nocloudkvm.py b/tests/cloud_tests/instances/nocloudkvm.py index cc825800..bc06a79e 100644 --- a/tests/cloud_tests/instances/nocloudkvm.py +++ b/tests/cloud_tests/instances/nocloudkvm.py @@ -25,12 +25,13 @@ class NoCloudKVMInstance(base.Instance): platform_name = "nocloud-kvm" _ssh_client = None - def __init__(self, platform, name, properties, config, features, - user_data, meta_data): + def __init__(self, platform, name, image_path, properties, config, + features, user_data, meta_data): """Set up instance. @param platform: platform object @param name: image path + @param image_path: path to disk image to boot. @param properties: dictionary of properties @param config: dictionary of configuration values @param features: dictionary of supported feature flags @@ -43,6 +44,7 @@ class NoCloudKVMInstance(base.Instance): self.pid = None self.pid_file = None self.console_file = None + self.disk = image_path super(NoCloudKVMInstance, self).__init__( platform, name, properties, config, features) @@ -145,7 +147,7 @@ class NoCloudKVMInstance(base.Instance): self.ssh_port = self.get_free_port() cmd = ['./tools/xkvm', - '--disk', '%s,cache=unsafe' % self.name, + '--disk', '%s,cache=unsafe' % self.disk, '--disk', '%s,cache=unsafe' % seed, '--netdev', ','.join(['user', 'hostfwd=tcp::%s-:22' % self.ssh_port, diff --git a/tests/cloud_tests/platforms/nocloudkvm.py b/tests/cloud_tests/platforms/nocloudkvm.py index f1f81877..76cd83ad 100644 --- a/tests/cloud_tests/platforms/nocloudkvm.py +++ b/tests/cloud_tests/platforms/nocloudkvm.py @@ -55,19 +55,20 @@ class NoCloudKVMPlatform(base.Platform): for fname in glob.iglob(search_d, recursive=True): images.append(fname) - if len(images) != 1: - raise Exception('No unique images found') + if len(images) < 1: + raise RuntimeError("No images found under '%s'" % search_d) + if len(images) > 1: + raise RuntimeError( + "Multiple images found in '%s': %s" % (search_d, + ' '.join(images))) image = nocloud_kvm_image.NoCloudKVMImage(self, img_conf, images[0]) - if img_conf.get('override_templates', False): - image.update_templates(self.config.get('template_overrides', {}), - self.config.get('template_files', {})) return image - def create_image(self, properties, config, features, - src_img_path, image_desc=None, use_desc=None, - user_data=None, meta_data=None): - """Create an image + def create_instance(self, properties, config, features, + src_img_path, image_desc=None, use_desc=None, + user_data=None, meta_data=None): + """Create an instance @param src_img_path: image path to launch from @param properties: image properties @@ -82,7 +83,7 @@ class NoCloudKVMPlatform(base.Platform): c_util.subp(['qemu-img', 'create', '-f', 'qcow2', '-b', src_img_path, img_path]) - return nocloud_kvm_instance.NoCloudKVMInstance(self, img_path, + return nocloud_kvm_instance.NoCloudKVMInstance(self, name, img_path, properties, config, features, user_data, meta_data) diff --git a/tests/cloud_tests/snapshots/nocloudkvm.py b/tests/cloud_tests/snapshots/nocloudkvm.py index 09998349..21e908da 100644 --- a/tests/cloud_tests/snapshots/nocloudkvm.py +++ b/tests/cloud_tests/snapshots/nocloudkvm.py @@ -2,6 +2,8 @@ """Base NoCloud KVM snapshot.""" import os +import shutil +import tempfile from tests.cloud_tests.snapshots import base @@ -11,16 +13,19 @@ class NoCloudKVMSnapshot(base.Snapshot): platform_name = "nocloud-kvm" - def __init__(self, platform, properties, config, features, - instance): + def __init__(self, platform, properties, config, features, image_path): """Set up snapshot. @param platform: platform object @param properties: image properties @param config: image config @param features: supported feature flags + @param image_path: image file to snapshot. """ - self.instance = instance + self._workd = tempfile.mkdtemp(prefix='NoCloudKVMSnapshot') + snapshot = os.path.join(self._workd, 'snapshot') + shutil.copyfile(image_path, snapshot) + self._image_path = snapshot super(NoCloudKVMSnapshot, self).__init__( platform, properties, config, features) @@ -40,9 +45,9 @@ class NoCloudKVMSnapshot(base.Snapshot): self.platform.config['public_key']) user_data = self.inject_ssh_key(user_data, key_file) - instance = self.platform.create_image( + instance = self.platform.create_instance( self.properties, self.config, self.features, - self.instance.name, image_desc=str(self), use_desc=use_desc, + self._image_path, image_desc=str(self), use_desc=use_desc, user_data=user_data, meta_data=meta_data) if start: @@ -68,7 +73,7 @@ class NoCloudKVMSnapshot(base.Snapshot): def destroy(self): """Clean up snapshot data.""" - self.instance.destroy() + shutil.rmtree(self._workd) super(NoCloudKVMSnapshot, self).destroy() # vi: ts=4 expandtab -- cgit v1.2.3 From 7acc9e68fafbbd7c56587aebe752ba6ba8c8a3db Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Thu, 30 Nov 2017 13:09:30 -0700 Subject: ec2: Fix sandboxed dhclient background process cleanup. There is a race condition where our sandboxed dhclient properly writes a lease file but has not yet written a pid file. If the sandbox temporary directory is torn down before the dhclient subprocess writes a pidfile DataSourceEc2Local gets a traceback and the instance will fallback to DataSourceEc2 in the init-network stage. This wastes boot cycles we'd rather not spend. Fix handling of sandboxed dhclient to wait for both pidfile and leasefile before proceding. If either file doesn't show in 5 seconds, log a warning and return empty lease results {}. LP: #1735331 --- cloudinit/net/dhcp.py | 44 ++++++++++++------ cloudinit/net/tests/test_dhcp.py | 66 +++++++++++++++++++++++++-- cloudinit/sources/DataSourceAzure.py | 29 ++---------- cloudinit/util.py | 22 +++++++++ tests/unittests/test_datasource/test_azure.py | 5 +- 5 files changed, 118 insertions(+), 48 deletions(-) diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py index d8624d82..875a4609 100644 --- a/cloudinit/net/dhcp.py +++ b/cloudinit/net/dhcp.py @@ -36,22 +36,23 @@ def maybe_perform_dhcp_discovery(nic=None): skip dhcp_discovery and return an empty dict. @param nic: Name of the network interface we want to run dhclient on. - @return: A dict of dhcp options from the dhclient discovery if run, - otherwise an empty dict is returned. + @return: A list of dicts representing dhcp options for each lease obtained + from the dhclient discovery if run, otherwise an empty list is + returned. """ if nic is None: nic = find_fallback_nic() if nic is None: LOG.debug('Skip dhcp_discovery: Unable to find fallback nic.') - return {} + return [] elif nic not in get_devicelist(): LOG.debug( 'Skip dhcp_discovery: nic %s not found in get_devicelist.', nic) - return {} + return [] dhclient_path = util.which('dhclient') if not dhclient_path: LOG.debug('Skip dhclient configuration: No dhclient command found.') - return {} + return [] with temp_utils.tempdir(prefix='cloud-init-dhcp-', needs_exe=True) as tdir: # Use /var/tmp because /run/cloud-init/tmp is mounted noexec return dhcp_discovery(dhclient_path, nic, tdir) @@ -60,8 +61,8 @@ def maybe_perform_dhcp_discovery(nic=None): def parse_dhcp_lease_file(lease_file): """Parse the given dhcp lease file for the most recent lease. - Return a dict of dhcp options as key value pairs for the most recent lease - block. + Return a list of dicts of dhcp options. Each dict contains key value pairs + a specific lease in order from oldest to newest. @raises: InvalidDHCPLeaseFileError on empty of unparseable leasefile content. @@ -96,8 +97,8 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): @param cleandir: The directory from which to run dhclient as well as store dhcp leases. - @return: A dict of dhcp options parsed from the dhcp.leases file or empty - dict. + @return: A list of dicts of representing the dhcp leases parsed from the + dhcp.leases file or empty list. """ LOG.debug('Performing a dhcp discovery on %s', interface) @@ -119,13 +120,26 @@ def dhcp_discovery(dhclient_cmd_path, interface, cleandir): cmd = [sandbox_dhclient_cmd, '-1', '-v', '-lf', lease_file, '-pf', pid_file, interface, '-sf', '/bin/true'] util.subp(cmd, capture=True) - pid = None + + # dhclient doesn't write a pid file until after it forks when it gets a + # proper lease response. Since cleandir is a temp directory that gets + # removed, we need to wait for that pidfile creation before the + # cleandir is removed, otherwise we get FileNotFound errors. + missing = util.wait_for_files( + [pid_file, lease_file], maxwait=5, naplen=0.01) + if missing: + LOG.warning("dhclient did not produce expected files: %s", + ', '.join(os.path.basename(f) for f in missing)) + return [] + pid_content = util.load_file(pid_file).strip() try: - pid = int(util.load_file(pid_file).strip()) - return parse_dhcp_lease_file(lease_file) - finally: - if pid: - os.kill(pid, signal.SIGKILL) + pid = int(pid_content) + except ValueError: + LOG.debug( + "pid file contains non-integer content '%s'", pid_content) + else: + os.kill(pid, signal.SIGKILL) + return parse_dhcp_lease_file(lease_file) def networkd_parse_lease(content): diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py index 3d8e15c0..db25b6f2 100644 --- a/cloudinit/net/tests/test_dhcp.py +++ b/cloudinit/net/tests/test_dhcp.py @@ -1,6 +1,5 @@ # This file is part of cloud-init. See LICENSE file for license information. -import mock import os import signal from textwrap import dedent @@ -9,7 +8,8 @@ from cloudinit.net.dhcp import ( InvalidDHCPLeaseFileError, maybe_perform_dhcp_discovery, parse_dhcp_lease_file, dhcp_discovery, networkd_load_leases) from cloudinit.util import ensure_file, write_file -from cloudinit.tests.helpers import CiTestCase, wrap_and_call, populate_dir +from cloudinit.tests.helpers import ( + CiTestCase, mock, populate_dir, wrap_and_call) class TestParseDHCPLeasesFile(CiTestCase): @@ -69,14 +69,14 @@ class TestDHCPDiscoveryClean(CiTestCase): def test_no_fallback_nic_found(self, m_fallback_nic): """Log and do nothing when nic is absent and no fallback is found.""" m_fallback_nic.return_value = None # No fallback nic found - self.assertEqual({}, maybe_perform_dhcp_discovery()) + self.assertEqual([], maybe_perform_dhcp_discovery()) self.assertIn( 'Skip dhcp_discovery: Unable to find fallback nic.', self.logs.getvalue()) def test_provided_nic_does_not_exist(self): """When the provided nic doesn't exist, log a message and no-op.""" - self.assertEqual({}, maybe_perform_dhcp_discovery('idontexist')) + self.assertEqual([], maybe_perform_dhcp_discovery('idontexist')) self.assertIn( 'Skip dhcp_discovery: nic idontexist not found in get_devicelist.', self.logs.getvalue()) @@ -87,7 +87,7 @@ class TestDHCPDiscoveryClean(CiTestCase): """When dhclient doesn't exist in the OS, log the issue and no-op.""" m_fallback.return_value = 'eth9' m_which.return_value = None # dhclient isn't found - self.assertEqual({}, maybe_perform_dhcp_discovery()) + self.assertEqual([], maybe_perform_dhcp_discovery()) self.assertIn( 'Skip dhclient configuration: No dhclient command found.', self.logs.getvalue()) @@ -115,6 +115,62 @@ class TestDHCPDiscoveryClean(CiTestCase): self.assertEqual('eth9', call[0][1]) self.assertIn('/var/tmp/cloud-init/cloud-init-dhcp-', call[0][2]) + @mock.patch('cloudinit.net.dhcp.os.kill') + @mock.patch('cloudinit.net.dhcp.util.subp') + def test_dhcp_discovery_run_in_sandbox_warns_invalid_pid(self, m_subp, + m_kill): + """dhcp_discovery logs a warning when pidfile contains invalid content. + + Lease processing still occurs and no proc kill is attempted. + """ + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + write_file(self.tmp_path('dhclient.pid', tmpdir), '') # Empty pid '' + lease_content = dedent(""" + lease { + interface "eth9"; + fixed-address 192.168.2.74; + option subnet-mask 255.255.255.0; + option routers 192.168.2.1; + } + """) + write_file(self.tmp_path('dhcp.leases', tmpdir), lease_content) + + self.assertItemsEqual( + [{'interface': 'eth9', 'fixed-address': '192.168.2.74', + 'subnet-mask': '255.255.255.0', 'routers': '192.168.2.1'}], + dhcp_discovery(dhclient_script, 'eth9', tmpdir)) + self.assertIn( + "pid file contains non-integer content ''", self.logs.getvalue()) + m_kill.assert_not_called() + + @mock.patch('cloudinit.net.dhcp.os.kill') + @mock.patch('cloudinit.net.dhcp.util.wait_for_files') + @mock.patch('cloudinit.net.dhcp.util.subp') + def test_dhcp_discovery_run_in_sandbox_waits_on_lease_and_pid(self, + m_subp, + m_wait, + m_kill): + """dhcp_discovery waits for the presence of pidfile and dhcp.leases.""" + tmpdir = self.tmp_dir() + dhclient_script = os.path.join(tmpdir, 'dhclient.orig') + script_content = '#!/bin/bash\necho fake-dhclient' + write_file(dhclient_script, script_content, mode=0o755) + # Don't create pid or leases file + pidfile = self.tmp_path('dhclient.pid', tmpdir) + leasefile = self.tmp_path('dhcp.leases', tmpdir) + m_wait.return_value = [pidfile] # Return the missing pidfile wait for + self.assertEqual([], dhcp_discovery(dhclient_script, 'eth9', tmpdir)) + self.assertEqual( + mock.call([pidfile, leasefile], maxwait=5, naplen=0.01), + m_wait.call_args_list[0]) + self.assertIn( + 'WARNING: dhclient did not produce expected files: dhclient.pid', + self.logs.getvalue()) + m_kill.assert_not_called() + @mock.patch('cloudinit.net.dhcp.os.kill') @mock.patch('cloudinit.net.dhcp.util.subp') def test_dhcp_discovery_run_in_sandbox(self, m_subp, m_kill): diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 8c3492d9..14367e9c 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -11,7 +11,6 @@ from functools import partial import os import os.path import re -import time from xml.dom import minidom import xml.etree.ElementTree as ET @@ -321,7 +320,7 @@ class DataSourceAzure(sources.DataSource): # https://bugs.launchpad.net/cloud-init/+bug/1717611 missing = util.log_time(logfunc=LOG.debug, msg="waiting for SSH public key files", - func=wait_for_files, + func=util.wait_for_files, args=(fp_files, 900)) if len(missing): @@ -556,8 +555,8 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, is_new_instance=False): # wait for ephemeral disk to come up naplen = .2 - missing = wait_for_files([devpath], maxwait=maxwait, naplen=naplen, - log_pre="Azure ephemeral disk: ") + missing = util.wait_for_files([devpath], maxwait=maxwait, naplen=naplen, + log_pre="Azure ephemeral disk: ") if missing: LOG.warning("ephemeral device '%s' did not appear after %d seconds.", @@ -639,28 +638,6 @@ def pubkeys_from_crt_files(flist): return pubkeys -def wait_for_files(flist, maxwait, naplen=.5, log_pre=""): - need = set(flist) - waited = 0 - while True: - need -= set([f for f in need if os.path.exists(f)]) - if len(need) == 0: - LOG.debug("%sAll files appeared after %s seconds: %s", - log_pre, waited, flist) - return [] - if waited == 0: - LOG.info("%sWaiting up to %s seconds for the following files: %s", - log_pre, maxwait, flist) - if waited + naplen > maxwait: - break - time.sleep(naplen) - waited += naplen - - LOG.warning("%sStill missing files after %s seconds: %s", - log_pre, maxwait, need) - return need - - def write_files(datadir, files, dirmode=None): def _redact_password(cnt, fname): diff --git a/cloudinit/util.py b/cloudinit/util.py index e1290aa8..6c014ba5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2541,4 +2541,26 @@ def load_shell_content(content, add_empty=False, empty_val=None): return data +def wait_for_files(flist, maxwait, naplen=.5, log_pre=""): + need = set(flist) + waited = 0 + while True: + need -= set([f for f in need if os.path.exists(f)]) + if len(need) == 0: + LOG.debug("%sAll files appeared after %s seconds: %s", + log_pre, waited, flist) + return [] + if waited == 0: + LOG.debug("%sWaiting up to %s seconds for the following files: %s", + log_pre, maxwait, flist) + if waited + naplen > maxwait: + break + time.sleep(naplen) + waited += naplen + + LOG.debug("%sStill missing files after %s seconds: %s", + log_pre, maxwait, need) + return need + + # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 0a117771..7cb1812a 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -171,7 +171,6 @@ scbus-1 on xpt0 bus 0 self.apply_patches([ (dsaz, 'list_possible_azure_ds_devs', dsdevs), (dsaz, 'invoke_agent', _invoke_agent), - (dsaz, 'wait_for_files', _wait_for_files), (dsaz, 'pubkeys_from_crt_files', _pubkeys_from_crt_files), (dsaz, 'perform_hostname_bounce', mock.MagicMock()), (dsaz, 'get_hostname', mock.MagicMock()), @@ -179,6 +178,8 @@ scbus-1 on xpt0 bus 0 (dsaz, 'get_metadata_from_fabric', self.get_metadata_from_fabric), (dsaz.util, 'read_dmi_data', mock.MagicMock( side_effect=_dmi_mocks)), + (dsaz.util, 'wait_for_files', mock.MagicMock( + side_effect=_wait_for_files)), ]) dsrc = dsaz.DataSourceAzure( @@ -647,7 +648,7 @@ class TestAzureBounce(TestCase): self.patches.enter_context( mock.patch.object(dsaz, 'invoke_agent')) self.patches.enter_context( - mock.patch.object(dsaz, 'wait_for_files')) + mock.patch.object(dsaz.util, 'wait_for_files')) self.patches.enter_context( mock.patch.object(dsaz, 'list_possible_azure_ds_devs', mock.MagicMock(return_value=[]))) -- cgit v1.2.3