From a6faf3acef02bd8cd4d46ac9efeebf24b3f21d81 Mon Sep 17 00:00:00 2001 From: Janos Lenart Date: Mon, 15 Jul 2019 21:26:50 +0000 Subject: Update debian eni network configuration location, retain Ubuntu setting On Debian, ifupdown uses `source-directory /etc/network/interfaces.d` (for new installs) to include files. https://salsa.debian.org/debian/ifupdown/blob/master/debian/postinst#L23 The current filename, 50-cloud-init.cfg, does not match against the RE that is used to scan the directory for configurations (ASCII upper- and lower-case letters, ASCII digits, ASCII underscores, and ASCII minus-hyphens): https://salsa.debian.org/debian/ifupdown/blob/master/interfaces.5.pre#L122 Of course many installations use `source /etc/network/interfaces.d/*`, but not all. --- cloudinit/distros/debian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index d517fb88..0ad93ffe 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -36,14 +36,14 @@ ENI_HEADER = """# This file is generated from information provided by # network: {config: disabled} """ -NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init.cfg" +NETWORK_CONF_FN = "/etc/network/interfaces.d/50-cloud-init" LOCALE_CONF_FN = "/etc/default/locale" class Distro(distros.Distro): hostname_conf_fn = "/etc/hostname" network_conf_fn = { - "eni": "/etc/network/interfaces.d/50-cloud-init.cfg", + "eni": "/etc/network/interfaces.d/50-cloud-init", "netplan": "/etc/netplan/50-cloud-init.yaml" } renderer_configs = { -- cgit v1.2.3 From 052d655cfba37243396c491a1e7892ba3736ab6f Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 27 Sep 2019 20:31:57 +0000 Subject: debian/ubuntu: add missing word to netplan/ENI header Specifically, add in "reboot" to make it clear what people should expect when modifying the file. This also renames the variable to indicate it is used for netplan and ENI, not just ENI. LP: #1845669 --- cloudinit/distros/debian.py | 11 ++++++----- cloudinit/distros/ubuntu.py | 4 ++-- tests/unittests/test_distros/test_netconfig.py | 18 +++++++++--------- 3 files changed, 17 insertions(+), 16 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 0ad93ffe..cf082c73 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -29,9 +29,10 @@ APT_GET_WRAPPER = { 'enabled': 'auto', } -ENI_HEADER = """# This file is generated from information provided by -# the datasource. Changes to it will not persist across an instance. -# To disable cloud-init's network configuration capabilities, write a file +NETWORK_FILE_HEADER = """\ +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} """ @@ -48,9 +49,9 @@ class Distro(distros.Distro): } renderer_configs = { "eni": {"eni_path": network_conf_fn["eni"], - "eni_header": ENI_HEADER}, + "eni_header": NETWORK_FILE_HEADER}, "netplan": {"netplan_path": network_conf_fn["netplan"], - "netplan_header": ENI_HEADER, + "netplan_header": NETWORK_FILE_HEADER, "postcmds": True} } diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index e5fcbc58..23be3bdd 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -30,9 +30,9 @@ class Distro(debian.Distro): } self.renderer_configs = { "eni": {"eni_path": self.network_conf_fn["eni"], - "eni_header": debian.ENI_HEADER}, + "eni_header": debian.NETWORK_FILE_HEADER}, "netplan": {"netplan_path": self.network_conf_fn["netplan"], - "netplan_header": debian.ENI_HEADER, + "netplan_header": debian.NETWORK_FILE_HEADER, "postcmds": True} } diff --git a/tests/unittests/test_distros/test_netconfig.py b/tests/unittests/test_distros/test_netconfig.py index 07b5c0a7..67209955 100644 --- a/tests/unittests/test_distros/test_netconfig.py +++ b/tests/unittests/test_distros/test_netconfig.py @@ -91,9 +91,9 @@ V1_NET_CFG = {'config': [{'name': 'eth0', 'version': 1} V1_NET_CFG_OUTPUT = """\ -# This file is generated from information provided by -# the datasource. Changes to it will not persist across an instance. -# To disable cloud-init's network configuration capabilities, write a file +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} auto lo @@ -125,9 +125,9 @@ V1_NET_CFG_IPV6 = {'config': [{'name': 'eth0', V1_TO_V2_NET_CFG_OUTPUT = """\ -# This file is generated from information provided by -# the datasource. Changes to it will not persist across an instance. -# To disable cloud-init's network configuration capabilities, write a file +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} network: @@ -154,9 +154,9 @@ V2_NET_CFG = { V2_TO_V2_NET_CFG_OUTPUT = """\ -# This file is generated from information provided by -# the datasource. Changes to it will not persist across an instance. -# To disable cloud-init's network configuration capabilities, write a file +# This file is generated from information provided by the datasource. Changes +# to it will not persist across an instance reboot. To disable cloud-init's +# network configuration capabilities, write a file # /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following: # network: {config: disabled} network: -- cgit v1.2.3 From 259d9c501472315e539701f00095743390eb89e5 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 16 Jan 2020 11:29:59 -0600 Subject: Ensure util.get_architecture() runs only once (#172) * Ensure util.get_architecture() runs only once util.get_architecture() recently was wrapped using python3's lru_cache() which will cache the result so we only invoke 'dpkg --print-architecture' once. In practice, cloud-init.log will show multiple invocations of the command. The source of this was that the debian Distro object implements the get_primary_arch() with this command, but it was not calling it from util, but issuing a util.subp() directly. This branch also updates cc_apt_configure methods to fetch the arch value from the distro class, and then ensure that the methods apt_configure calls pass the arch value around. * utils: remove lsb_release and get_architecture wrappers The original lsb_release wrapper was used to prevent polluting the single value we cached, however lru_cache() already handles this case by using args, kwargs values to cache different calls to the method. * rename_apt_list: use all positional parameters --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/distros/debian.py | 3 +-- cloudinit/util.py | 10 +--------- tests/unittests/test_handler/test_handler_apt_source_v3.py | 7 ++++--- 4 files changed, 9 insertions(+), 17 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index f01e2aaf..4a3aed36 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -309,7 +309,7 @@ def apply_apt(cfg, cloud, target): if util.is_false(cfg.get('preserve_sources_list', False)): generate_sources_list(cfg, release, mirrors, cloud) - rename_apt_lists(mirrors, target) + rename_apt_lists(mirrors, target, arch) try: apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) @@ -427,9 +427,9 @@ def mirrorurl_to_apt_fileprefix(mirror): return string -def rename_apt_lists(new_mirrors, target=None): +def rename_apt_lists(new_mirrors, target, arch): """rename_apt_lists - rename apt lists to preserve old cache data""" - default_mirrors = get_default_mirrors(util.get_architecture(target)) + default_mirrors = get_default_mirrors(arch) pre = util.target_path(target, APT_LISTS) for (name, omirror) in default_mirrors.items(): diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index cf082c73..79268371 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -205,8 +205,7 @@ class Distro(distros.Distro): ["update"], freq=PER_INSTANCE) def get_primary_arch(self): - (arch, _err) = util.subp(['dpkg', '--print-architecture']) - return str(arch).strip() + return util.get_architecture() def _get_wrapper_prefix(cmd, mode): diff --git a/cloudinit/util.py b/cloudinit/util.py index 76d7db78..87480767 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -86,7 +86,7 @@ def get_architecture(target=None): @lru_cache() -def _lsb_release(target=None): +def lsb_release(target=None): fmap = {'Codename': 'codename', 'Description': 'description', 'Distributor ID': 'id', 'Release': 'release'} @@ -109,14 +109,6 @@ def _lsb_release(target=None): return data -def lsb_release(target=None): - if target_path(target) != "/": - # do not use or update cache if target is provided - return _lsb_release(target) - - return _lsb_release() - - def target_path(target, path=None): # return 'path' inside target, accepting target as None if target in (None, ""): diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index 2f21b6dc..dcffbe13 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -487,7 +487,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): with mock.patch.object(os, 'rename') as mockren: with mock.patch.object(glob, 'glob', return_value=[fromfn]): - cc_apt_configure.rename_apt_lists(mirrors, TARGET) + cc_apt_configure.rename_apt_lists(mirrors, TARGET, arch) mockren.assert_any_call(fromfn, tofn) @@ -496,7 +496,8 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): target = os.path.join(self.tmp, "rename_non_slash") apt_lists_d = os.path.join(target, "./" + cc_apt_configure.APT_LISTS) - m_get_architecture.return_value = 'amd64' + arch = 'amd64' + m_get_architecture.return_value = arch mirror_path = "some/random/path/" primary = "http://test.ubuntu.com/" + mirror_path @@ -532,7 +533,7 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): fpath = os.path.join(apt_lists_d, opre + suff) util.write_file(fpath, content=fpath) - cc_apt_configure.rename_apt_lists(mirrors, target) + cc_apt_configure.rename_apt_lists(mirrors, target, arch) found = sorted(os.listdir(apt_lists_d)) self.assertEqual(expected, found) -- cgit v1.2.3 From 651e24066ba4b9464c7843553f60d17a459cf06e Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 16 Jan 2020 13:30:12 -0500 Subject: util: rename get_architecture to get_dpkg_architecture (#173) This makes it clearer that we should only use this in code paths that will definitely have dpkg available to them. - Rename get_architecture -> get_dpkg_architecture - Add docstring to get_dpkg_architecture --- cloudinit/config/cc_apt_configure.py | 6 +++--- cloudinit/distros/debian.py | 2 +- cloudinit/util.py | 7 ++++++- tests/cloud_tests/config.py | 2 +- tests/cloud_tests/platforms/nocloudkvm/platform.py | 10 +++++++--- .../test_handler_apt_configure_sources_list_v1.py | 2 +- .../test_handler_apt_configure_sources_list_v3.py | 2 +- .../test_handler/test_handler_apt_source_v1.py | 2 +- .../test_handler/test_handler_apt_source_v3.py | 20 +++++++++++--------- 9 files changed, 32 insertions(+), 21 deletions(-) (limited to 'cloudinit/distros/debian.py') diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 4a3aed36..c44dec45 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -253,7 +253,7 @@ def get_default_mirrors(arch=None, target=None): architecture, for more see: https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" if arch is None: - arch = util.get_architecture(target) + arch = util.get_dpkg_architecture(target) if arch in PRIMARY_ARCHES: return PRIMARY_ARCH_MIRRORS.copy() if arch in PORTS_ARCHES: @@ -303,7 +303,7 @@ def apply_apt(cfg, cloud, target): LOG.debug("handling apt config: %s", cfg) release = util.lsb_release(target=target)['codename'] - arch = util.get_architecture(target) + arch = util.get_dpkg_architecture(target) mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) LOG.debug("Apt Mirror info: %s", mirrors) @@ -896,7 +896,7 @@ def find_apt_mirror_info(cfg, cloud, arch=None): """ if arch is None: - arch = util.get_architecture() + arch = util.get_dpkg_architecture() LOG.debug("got arch for mirror selection: %s", arch) pmirror = get_mirror(cfg, "primary", arch, cloud) LOG.debug("got primary mirror: %s", pmirror) diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 79268371..128bb523 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -205,7 +205,7 @@ class Distro(distros.Distro): ["update"], freq=PER_INSTANCE) def get_primary_arch(self): - return util.get_architecture() + return util.get_dpkg_architecture() def _get_wrapper_prefix(cmd, mode): diff --git a/cloudinit/util.py b/cloudinit/util.py index 87480767..d99e82fa 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -79,7 +79,12 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], @lru_cache() -def get_architecture(target=None): +def get_dpkg_architecture(target=None): + """Return the sanitized string output by `dpkg --print-architecture`. + + N.B. This function is wrapped in functools.lru_cache, so repeated calls + won't shell out every time. + """ out, _ = subp(['dpkg', '--print-architecture'], capture=True, target=target) return out.strip() diff --git a/tests/cloud_tests/config.py b/tests/cloud_tests/config.py index 8bd569fd..06536edc 100644 --- a/tests/cloud_tests/config.py +++ b/tests/cloud_tests/config.py @@ -114,7 +114,7 @@ def load_os_config(platform_name, os_name, require_enabled=False, feature_conf = main_conf['features'] feature_groups = conf.get('feature_groups', []) overrides = merge_config(get(conf, 'features'), feature_overrides) - conf['arch'] = c_util.get_architecture() + conf['arch'] = c_util.get_dpkg_architecture() conf['features'] = merge_feature_groups( feature_conf, feature_groups, overrides) diff --git a/tests/cloud_tests/platforms/nocloudkvm/platform.py b/tests/cloud_tests/platforms/nocloudkvm/platform.py index 85933463..2d1480f5 100644 --- a/tests/cloud_tests/platforms/nocloudkvm/platform.py +++ b/tests/cloud_tests/platforms/nocloudkvm/platform.py @@ -29,9 +29,13 @@ class NoCloudKVMPlatform(Platform): """ (url, path) = s_util.path_from_mirror_url(img_conf['mirror_url'], None) - filter = filters.get_filters(['arch=%s' % c_util.get_architecture(), - 'release=%s' % img_conf['release'], - 'ftype=disk1.img']) + filter = filters.get_filters( + [ + 'arch=%s' % c_util.get_dpkg_architecture(), + 'release=%s' % img_conf['release'], + 'ftype=disk1.img', + ] + ) mirror_config = {'filters': filter, 'keep_items': False, 'max_items': 1, diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py index 23bd6e10..7c17a264 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -78,7 +78,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): get_rel = rpatcher.start() get_rel.return_value = {'codename': "fakerelease"} self.addCleanup(rpatcher.stop) - apatcher = mock.patch("cloudinit.util.get_architecture") + apatcher = mock.patch("cloudinit.util.get_dpkg_architecture") get_arch = apatcher.start() get_arch.return_value = 'amd64' self.addCleanup(apatcher.stop) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py index f7608c28..0a68cb8f 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -106,7 +106,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): get_rel = rpatcher.start() get_rel.return_value = {'codename': "fakerel"} self.addCleanup(rpatcher.stop) - apatcher = mock.patch("cloudinit.util.get_architecture") + apatcher = mock.patch("cloudinit.util.get_dpkg_architecture") get_arch = apatcher.start() get_arch.return_value = 'amd64' self.addCleanup(apatcher.stop) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py index a3132fbd..652d97ab 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -77,7 +77,7 @@ class TestAptSourceConfig(TestCase): get_rel = rpatcher.start() get_rel.return_value = {'codename': self.release} self.addCleanup(rpatcher.stop) - apatcher = mock.patch("cloudinit.util.get_architecture") + apatcher = mock.patch("cloudinit.util.get_dpkg_architecture") get_arch = apatcher.start() get_arch.return_value = 'amd64' self.addCleanup(apatcher.stop) diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py index dcffbe13..c5cf6785 100644 --- a/tests/unittests/test_handler/test_handler_apt_source_v3.py +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -453,14 +453,14 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertFalse(os.path.isfile(self.aptlistfile2)) self.assertFalse(os.path.isfile(self.aptlistfile3)) - @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") - def test_apt_v3_list_rename(self, m_get_architecture): + @mock.patch("cloudinit.config.cc_apt_configure.util.get_dpkg_architecture") + def test_apt_v3_list_rename(self, m_get_dpkg_architecture): """test_apt_v3_list_rename - Test find mirror and apt list renaming""" pre = "/var/lib/apt/lists" # filenames are archive dependent arch = 's390x' - m_get_architecture.return_value = arch + m_get_dpkg_architecture.return_value = arch component = "ubuntu-ports" archive = "ports.ubuntu.com" @@ -491,13 +491,13 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): mockren.assert_any_call(fromfn, tofn) - @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") - def test_apt_v3_list_rename_non_slash(self, m_get_architecture): + @mock.patch("cloudinit.config.cc_apt_configure.util.get_dpkg_architecture") + def test_apt_v3_list_rename_non_slash(self, m_get_dpkg_architecture): target = os.path.join(self.tmp, "rename_non_slash") apt_lists_d = os.path.join(target, "./" + cc_apt_configure.APT_LISTS) arch = 'amd64' - m_get_architecture.return_value = arch + m_get_dpkg_architecture.return_value = arch mirror_path = "some/random/path/" primary = "http://test.ubuntu.com/" + mirror_path @@ -626,10 +626,12 @@ class TestAptSourceConfig(t_help.FilesystemMockingTestCase): self.assertEqual(mirrors['SECURITY'], smir) - @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") - def test_apt_v3_get_def_mir_non_intel_no_arch(self, m_get_architecture): + @mock.patch("cloudinit.config.cc_apt_configure.util.get_dpkg_architecture") + def test_apt_v3_get_def_mir_non_intel_no_arch( + self, m_get_dpkg_architecture + ): arch = 'ppc64el' - m_get_architecture.return_value = arch + m_get_dpkg_architecture.return_value = arch expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports', 'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'} self.assertEqual(expected, cc_apt_configure.get_default_mirrors()) -- cgit v1.2.3