From 648dbbf6b090c81e989f1ab70bf99f4de16a6a70 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Wed, 10 Aug 2016 16:36:49 -0600 Subject: Get Azure endpoint server from DHCP client It is more efficient and cross-distribution safe to use the hooks function from dhclient to obtain the Azure endpoint server (DHCP option 245). This is done by providing shell scritps that are called by the hooks infrastructure of both dhclient and NetworkManager. The hooks then invoke 'cloud-init dhclient-hook' that maintains json data with the dhclient options in /run/cloud-init/dhclient.hooks/.json . The azure helper then pulls the value from /run/cloud-init/dhclient.hooks/.json file(s). If that file does not exist or the value is not present, it will then fall back to the original method of scraping the dhcp client lease file. --- cloudinit/atomic_helper.py | 25 ++++++ cloudinit/cmd/main.py | 45 ++++++---- cloudinit/dhclient_hook.py | 50 +++++++++++ cloudinit/sources/DataSourceAzure.py | 15 ++-- cloudinit/sources/helpers/azure.py | 99 +++++++++++++++++++--- config/cloud.cfg | 6 ++ doc/sources/azure/README.rst | 32 ++++++- setup.py | 2 + .../unittests/test_datasource/test_azure_helper.py | 15 +++- tools/hook-dhclient | 9 ++ tools/hook-network-manager | 9 ++ tools/hook-rhel.sh | 12 +++ 12 files changed, 277 insertions(+), 42 deletions(-) create mode 100644 cloudinit/atomic_helper.py create mode 100644 cloudinit/dhclient_hook.py create mode 100755 tools/hook-dhclient create mode 100755 tools/hook-network-manager create mode 100755 tools/hook-rhel.sh diff --git a/cloudinit/atomic_helper.py b/cloudinit/atomic_helper.py new file mode 100644 index 00000000..15319f71 --- /dev/null +++ b/cloudinit/atomic_helper.py @@ -0,0 +1,25 @@ +#!/usr/bin/python +# vi: ts=4 expandtab + +import json +import os +import tempfile + + +def atomic_write_file(path, content, mode='w'): + tf = None + try: + tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), + delete=False, mode=mode) + tf.write(content) + tf.close() + os.rename(tf.name, path) + except Exception as e: + if tf is not None: + os.unlink(tf.name) + raise e + + +def atomic_write_json(path, data): + return atomic_write_file(path, json.dumps(data, indent=1, + sort_keys=True) + "\n") diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py index 63621c1d..ba22b168 100644 --- a/cloudinit/cmd/main.py +++ b/cloudinit/cmd/main.py @@ -25,7 +25,6 @@ import argparse import json import os import sys -import tempfile import time import traceback @@ -47,6 +46,10 @@ from cloudinit.reporting import events from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, CLOUD_CONFIG) +from cloudinit.atomic_helper import atomic_write_json + +from cloudinit.dhclient_hook import LogDhclient + # Pretty little cheetah formatted welcome message template WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at " @@ -452,22 +455,10 @@ def main_single(name, args): return 0 -def atomic_write_file(path, content, mode='w'): - tf = None - try: - tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path), - delete=False, mode=mode) - tf.write(content) - tf.close() - os.rename(tf.name, path) - except Exception as e: - if tf is not None: - os.unlink(tf.name) - raise e - - -def atomic_write_json(path, data): - return atomic_write_file(path, json.dumps(data, indent=1) + "\n") +def dhclient_hook(name, args): + record = LogDhclient(args) + record.check_hooks_dir() + record.record() def status_wrapper(name, args, data_d=None, link_d=None): @@ -627,7 +618,6 @@ def main(sysv_args=None): # This subcommand allows you to run a single module parser_single = subparsers.add_parser('single', help=('run a single module ')) - parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", help="module name to run", required=True) @@ -644,6 +634,16 @@ def main(sysv_args=None): ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) + parser_dhclient = subparsers.add_parser('dhclient-hook', + help=('run the dhclient hook' + 'to record network info')) + parser_dhclient.add_argument("net_action", + help=('action taken on the interface')) + parser_dhclient.add_argument("net_interface", + help=('the network interface being acted' + ' upon')) + parser_dhclient.set_defaults(action=('dhclient_hook', dhclient_hook)) + args = parser.parse_args(args=sysv_args) try: @@ -677,9 +677,18 @@ def main(sysv_args=None): "running single module %s" % args.name) report_on = args.report + elif name == 'dhclient_hook': + rname, rdesc = ("dhclient-hook", + "running dhclient-hook module") + args.reporter = events.ReportEventStack( rname, rdesc, reporting_enabled=report_on) + with args.reporter: return util.log_time( logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, get_uptime=True, func=functor, args=(name, args)) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/cloudinit/dhclient_hook.py b/cloudinit/dhclient_hook.py new file mode 100644 index 00000000..9dcbe39c --- /dev/null +++ b/cloudinit/dhclient_hook.py @@ -0,0 +1,50 @@ +#!/usr/bin/python +# vi: ts=4 expandtab + +import os + +from cloudinit.atomic_helper import atomic_write_json +from cloudinit import log as logging +from cloudinit import stages + +LOG = logging.getLogger(__name__) + + +class LogDhclient(object): + + def __init__(self, cli_args): + self.hooks_dir = self._get_hooks_dir() + self.net_interface = cli_args.net_interface + self.net_action = cli_args.net_action + self.hook_file = os.path.join(self.hooks_dir, + self.net_interface + ".json") + + @staticmethod + def _get_hooks_dir(): + i = stages.Init() + return os.path.join(i.paths.get_runpath(), 'dhclient.hooks') + + def check_hooks_dir(self): + if not os.path.exists(self.hooks_dir): + os.makedirs(self.hooks_dir) + else: + # If the action is down and the json file exists, we need to + # delete the file + if self.net_action is 'down' and os.path.exists(self.hook_file): + os.remove(self.hook_file) + + @staticmethod + def get_vals(info): + new_info = {} + for k, v in info.items(): + if k.startswith("DHCP4_") or k.startswith("new_"): + key = (k.replace('DHCP4_', '').replace('new_', '')).lower() + new_info[key] = v + return new_info + + def record(self): + envs = os.environ + if self.hook_file is None: + return + atomic_write_json(self.hook_file, self.get_vals(envs)) + LOG.debug("Wrote dhclient options in %s", self.hook_file) diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 8c7e8673..a251fe01 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -20,18 +20,17 @@ import base64 import contextlib import crypt import fnmatch +from functools import partial import os import os.path import time -import xml.etree.ElementTree as ET - from xml.dom import minidom - -from cloudinit.sources.helpers.azure import get_metadata_from_fabric +import xml.etree.ElementTree as ET from cloudinit import log as logging from cloudinit.settings import PER_ALWAYS from cloudinit import sources +from cloudinit.sources.helpers.azure import get_metadata_from_fabric from cloudinit import util LOG = logging.getLogger(__name__) @@ -107,6 +106,8 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): class DataSourceAzureNet(sources.DataSource): + FALLBACK_LEASE = '/var/lib/dhcp/dhclient.eth0.leases' + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'azure') @@ -115,6 +116,8 @@ class DataSourceAzureNet(sources.DataSource): self.ds_cfg = util.mergemanydict([ util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG]) + self.dhclient_lease_file = self.paths.cfgs.get('dhclient_lease', + self.FALLBACK_LEASE) def __str__(self): root = sources.DataSource.__str__(self) @@ -226,7 +229,9 @@ class DataSourceAzureNet(sources.DataSource): write_files(ddir, files, dirmode=0o700) if self.ds_cfg['agent_command'] == '__builtin__': - metadata_func = get_metadata_from_fabric + metadata_func = partial(get_metadata_from_fabric, + fallback_lease_file=self. + dhclient_lease_file) else: metadata_func = self.get_metadata_from_agent try: diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 63ccf10e..6e43440f 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -1,3 +1,4 @@ +import json import logging import os import re @@ -6,6 +7,7 @@ import struct import tempfile import time +from cloudinit import stages from contextlib import contextmanager from xml.etree import ElementTree @@ -187,19 +189,32 @@ class WALinuxAgentShim(object): ' ', '']) - def __init__(self): + def __init__(self, fallback_lease_file=None): LOG.debug('WALinuxAgentShim instantiated...') - self.endpoint = self.find_endpoint() + self.dhcpoptions = None + self._endpoint = None self.openssl_manager = None self.values = {} + self.lease_file = fallback_lease_file def clean_up(self): if self.openssl_manager is not None: self.openssl_manager.clean_up() @staticmethod - def get_ip_from_lease_value(lease_value): - unescaped_value = lease_value.replace('\\', '') + def _get_hooks_dir(): + _paths = stages.Init() + return os.path.join(_paths.paths.get_runpath(), "dhclient.hooks") + + @property + def endpoint(self): + if self._endpoint is None: + self._endpoint = self.find_endpoint(self.lease_file) + return self._endpoint + + @staticmethod + def get_ip_from_lease_value(fallback_lease_value): + unescaped_value = fallback_lease_value.replace('\\', '') if len(unescaped_value) > 4: hex_string = '' for hex_pair in unescaped_value.split(':'): @@ -213,15 +228,75 @@ class WALinuxAgentShim(object): return socket.inet_ntoa(packed_bytes) @staticmethod - def find_endpoint(): - LOG.debug('Finding Azure endpoint...') - content = util.load_file('/var/lib/dhcp/dhclient.eth0.leases') - value = None + def _get_value_from_leases_file(fallback_lease_file): + leases = [] + content = util.load_file(fallback_lease_file) + LOG.debug("content is {}".format(content)) for line in content.splitlines(): if 'unknown-245' in line: - value = line.strip(' ').split(' ', 2)[-1].strip(';\n"') + # Example line from Ubuntu + # option unknown-245 a8:3f:81:10; + leases.append(line.strip(' ').split(' ', 2)[-1].strip(';\n"')) + # Return the "most recent" one in the list + if len(leases) < 1: + return None + else: + return leases[-1] + + @staticmethod + def _load_dhclient_json(): + dhcp_options = {} + hooks_dir = WALinuxAgentShim._get_hooks_dir() + if not os.path.exists(hooks_dir): + LOG.debug("%s not found.", hooks_dir) + return None + hook_files = [os.path.join(hooks_dir, x) + for x in os.listdir(hooks_dir)] + for hook_file in hook_files: + try: + name = os.path.basename(hook_file).replace('.json', '') + dhcp_options[name] = json.loads(util.load_file((hook_file))) + except ValueError: + raise ValueError("%s is not valid JSON data", hook_file) + return dhcp_options + + @staticmethod + def _get_value_from_dhcpoptions(dhcp_options): + if dhcp_options is None: + return None + # the MS endpoint server is given to us as DHPC option 245 + _value = None + for interface in dhcp_options: + _value = dhcp_options[interface].get('unknown_245', None) + if _value is not None: + LOG.debug("Endpoint server found in dhclient options") + break + return _value + + @staticmethod + def find_endpoint(fallback_lease_file=None): + LOG.debug('Finding Azure endpoint...') + value = None + # Option-245 stored in /run/cloud-init/dhclient.hooks/.json + # a dhclient exit hook that calls cloud-init-dhclient-hook + dhcp_options = WALinuxAgentShim._load_dhclient_json() + value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options) if value is None: - raise ValueError('No endpoint found in DHCP config.') + # Fallback and check the leases file if unsuccessful + LOG.debug("Unable to find endpoint in dhclient logs. " + " Falling back to check lease files") + if fallback_lease_file is None: + LOG.warn("No fallback lease file was specified.") + value = None + else: + LOG.debug("Looking for endpoint in lease file %s", + fallback_lease_file) + value = WALinuxAgentShim._get_value_from_leases_file( + fallback_lease_file) + + if value is None: + raise ValueError('No endpoint found.') + endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value) LOG.debug('Azure endpoint found at %s', endpoint_ip_address) return endpoint_ip_address @@ -271,8 +346,8 @@ class WALinuxAgentShim(object): LOG.info('Reported ready to Azure fabric.') -def get_metadata_from_fabric(): - shim = WALinuxAgentShim() +def get_metadata_from_fabric(fallback_lease_file=None): + shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file) try: return shim.register_with_azure_and_fetch_data() finally: diff --git a/config/cloud.cfg b/config/cloud.cfg index 2d7fb473..93ef3423 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -98,6 +98,7 @@ system_info: cloud_dir: /var/lib/cloud/ templates_dir: /etc/cloud/templates/ upstart_dir: /etc/init/ + dhclient_lease: package_mirrors: - arches: [i386, amd64] failsafe: @@ -114,3 +115,8 @@ system_info: primary: http://ports.ubuntu.com/ubuntu-ports security: http://ports.ubuntu.com/ubuntu-ports ssh_svcname: ssh +datasource: + Azure: + set_hostname: False + agent_command: __builtin__ + diff --git a/doc/sources/azure/README.rst b/doc/sources/azure/README.rst index 8239d1fa..48f3cc7a 100644 --- a/doc/sources/azure/README.rst +++ b/doc/sources/azure/README.rst @@ -9,10 +9,34 @@ Azure Platform The azure cloud-platform provides initial data to an instance via an attached CD formated in UDF. That CD contains a 'ovf-env.xml' file that provides some information. Additional information is obtained via interaction with the -"endpoint". The ip address of the endpoint is advertised to the instance -inside of dhcp option 245. On ubuntu, that can be seen in -/var/lib/dhcp/dhclient.eth0.leases as a colon delimited hex value (example: -``option unknown-245 64:41:60:82;`` is 100.65.96.130) +"endpoint". + +To find the endpoint, we now leverage the dhcp client's ability to log its +known values on exit. The endpoint server is special DHCP option 245. +Depending on your networking stack, this can be done +by calling a script in /etc/dhcp/dhclient-exit-hooks or a file in +/etc/NetworkManager/dispatcher.d. Both of these call a sub-command +'dhclient_hook' of cloud-init itself. This sub-command will write the client +information in json format to /run/cloud-init/dhclient.hook/.json. + +In order for cloud-init to leverage this method to find the endpoint, the +cloud.cfg file must contain: + +datasource: + Azure: + set_hostname: False + agent_command: __builtin__ + +If those files are not available, the fallback is to check the leases file +for the endpoint server (again option 245). + +You can define the path to the lease file with the 'dhclient_lease' configuration +value under system_info: and paths:. For example: + + dhclient_lease: /var/lib/dhcp/dhclient.eth0.leases + +If no configuration value is provided, the dhclient_lease value will fallback to +/var/lib/dhcp/dhclient.eth0.leases. walinuxagent ------------ diff --git a/setup.py b/setup.py index 4abbb67e..bbadd7bf 100755 --- a/setup.py +++ b/setup.py @@ -176,6 +176,8 @@ else: (ETC + '/cloud', glob('config/*.cfg')), (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')), (ETC + '/cloud/templates', glob('templates/*')), + (ETC + '/NetworkManager/dispatcher.d/', ['tools/hook-network-manager']), + (ETC + '/dhcp/dhclient-exit-hooks.d/', ['tools/hook-dhclient']), (USR_LIB_EXEC + '/cloud-init', ['tools/uncloud-init', 'tools/write-ssh-key-fingerprints']), (USR + '/share/doc/cloud-init', [f for f in glob('doc/*') if is_f(f)]), diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index 65202ff0..64523e16 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -54,13 +54,17 @@ class TestFindEndpoint(TestCase): self.load_file = patches.enter_context( mock.patch.object(azure_helper.util, 'load_file')) + self.dhcp_options = patches.enter_context( + mock.patch.object(azure_helper.WALinuxAgentShim, + '_load_dhclient_json')) + def test_missing_file(self): - self.load_file.side_effect = IOError - self.assertRaises(IOError, + self.assertRaises(ValueError, azure_helper.WALinuxAgentShim.find_endpoint) def test_missing_special_azure_line(self): self.load_file.return_value = '' + self.dhcp_options.return_value = {'eth0': {'key': 'value'}} self.assertRaises(ValueError, azure_helper.WALinuxAgentShim.find_endpoint) @@ -72,13 +76,18 @@ class TestFindEndpoint(TestCase): ' option unknown-245 {0};'.format(encoded_address), '}']) + def test_from_dhcp_client(self): + self.dhcp_options.return_value = {"eth0": {"unknown_245": "5:4:3:2"}} + self.assertEqual('5.4.3.2', + azure_helper.WALinuxAgentShim.find_endpoint(None)) + def test_latest_lease_used(self): encoded_addresses = ['5:4:3:2', '4:3:2:1'] file_content = '\n'.join([self._build_lease_content(encoded_address) for encoded_address in encoded_addresses]) self.load_file.return_value = file_content self.assertEqual(encoded_addresses[-1].replace(':', '.'), - azure_helper.WALinuxAgentShim.find_endpoint()) + azure_helper.WALinuxAgentShim.find_endpoint("foobar")) class TestExtractIpAddressFromLeaseValue(TestCase): diff --git a/tools/hook-dhclient b/tools/hook-dhclient new file mode 100755 index 00000000..d099979a --- /dev/null +++ b/tools/hook-dhclient @@ -0,0 +1,9 @@ +#!/bin/sh +# This script writes DHCP lease information into the cloud-init run directory +# It is sourced, not executed. For more information see dhclient-script(8). + +case "$reason" in + BOUND) cloud-init dhclient-hook up "$interface";; + DOWN|RELEASE|REBOOT|STOP|EXPIRE) + cloud-init dhclient-hook down "$interface";; +esac diff --git a/tools/hook-network-manager b/tools/hook-network-manager new file mode 100755 index 00000000..447b134e --- /dev/null +++ b/tools/hook-network-manager @@ -0,0 +1,9 @@ +#!/bin/sh +# This script hooks into NetworkManager(8) via its scripts +# arguments are 'interface-name' and 'action' +# + +case "$1:$2" in + *:up) exec cloud-init dhclient-hook up "$1";; + *:down) exec cloud-init dhclient-hook down "$1";; +esac diff --git a/tools/hook-rhel.sh b/tools/hook-rhel.sh new file mode 100755 index 00000000..5e963a89 --- /dev/null +++ b/tools/hook-rhel.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Current versions of RHEL and CentOS do not honor the directory +# /etc/dhcp/dhclient-exit-hooks.d so this file can be placed in +# /etc/dhcp/dhclient.d instead + +hook-rhel_config(){ + cloud-init dhclient-hook up "$interface" +} + +hook-rhel_restore(){ + cloud-init dhclient-hook down "$interface" +} -- cgit v1.2.3 From d861415ff9ab816b1183b8c58ec35348be4fd458 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 10 Aug 2016 16:43:14 +0200 Subject: Apt: add new apt configuration format This adds an improved apt configuration format that is fully backwards compatible with previous behavior. This is mostly copied from curtin's implementation. It does: * clean up and centralizes many of the top level 'apt_*' values that previously existed into a single top level 'apt'key. * support a 'source' in apt/sources/entry that has only a key * documents new features and adds tests. See the added doc/examples/cloud-config-apt.txt for more information. --- cloudinit/config/cc_apt_configure.py | 703 ++++++++++--- cloudinit/gpg.py | 8 +- cloudinit/util.py | 147 ++- doc/examples/cloud-config-add-apt-repos.txt | 42 +- doc/examples/cloud-config-apt.txt | 328 ++++++ doc/examples/cloud-config-chef-oneiric.txt | 67 +- doc/examples/cloud-config-chef.txt | 67 +- doc/examples/cloud-config.txt | 251 +---- tests/configs/sample1.yaml | 3 - tests/unittests/test_distros/test_generic.py | 3 - .../test_handler/test_handler_apt_conf_v1.py | 109 ++ .../test_handler/test_handler_apt_configure.py | 109 -- .../test_handler_apt_configure_sources_list.py | 180 ---- .../test_handler_apt_configure_sources_list_v1.py | 200 ++++ .../test_handler_apt_configure_sources_list_v3.py | 187 ++++ .../test_handler/test_handler_apt_source.py | 516 --------- .../test_handler/test_handler_apt_source_v1.py | 551 ++++++++++ .../test_handler/test_handler_apt_source_v3.py | 1103 ++++++++++++++++++++ tests/unittests/test_util.py | 69 ++ 19 files changed, 3325 insertions(+), 1318 deletions(-) create mode 100644 doc/examples/cloud-config-apt.txt create mode 100644 tests/unittests/test_handler/test_handler_apt_conf_v1.py delete mode 100644 tests/unittests/test_handler/test_handler_apt_configure.py delete mode 100644 tests/unittests/test_handler/test_handler_apt_configure_sources_list.py create mode 100644 tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py create mode 100644 tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py delete mode 100644 tests/unittests/test_handler/test_handler_apt_source.py create mode 100644 tests/unittests/test_handler/test_handler_apt_source_v1.py create mode 100644 tests/unittests/test_handler/test_handler_apt_source_v3.py diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 05ad4b03..609dbb51 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -23,80 +23,182 @@ import os import re from cloudinit import gpg +from cloudinit import log as logging from cloudinit import templater from cloudinit import util -distros = ['ubuntu', 'debian'] - -PROXY_TPL = "Acquire::HTTP::Proxy \"%s\";\n" -APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" -APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy" +LOG = logging.getLogger(__name__) # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') ADD_APT_REPO_MATCH = r"^[\w-]+:\w" +# place where apt stores cached repository data +APT_LISTS = "/var/lib/apt/lists" -def handle(name, cfg, cloud, log, _args): - if util.is_false(cfg.get('apt_configure_enabled', True)): - log.debug("Skipping module named %s, disabled by config.", name) - return - - release = get_release() - mirrors = find_apt_mirror_info(cloud, cfg) - if not mirrors or "primary" not in mirrors: - log.debug(("Skipping module named %s," - " no package 'mirror' located"), name) - return - - # backwards compatibility - mirror = mirrors["primary"] - mirrors["mirror"] = mirror - - log.debug("Mirror info: %s" % mirrors) - - if not util.get_cfg_option_bool(cfg, - 'apt_preserve_sources_list', False): - generate_sources_list(cfg, release, mirrors, cloud, log) - old_mirrors = cfg.get('apt_old_mirrors', - {"primary": "archive.ubuntu.com/ubuntu", - "security": "security.ubuntu.com/ubuntu"}) - rename_apt_lists(old_mirrors, mirrors) +# Files to store proxy information +APT_CONFIG_FN = "/etc/apt/apt.conf.d/94cloud-init-config" +APT_PROXY_FN = "/etc/apt/apt.conf.d/90cloud-init-aptproxy" + +# Default keyserver to use +DEFAULT_KEYSERVER = "keyserver.ubuntu.com" + +# Default archive mirrors +PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/", + "SECURITY": "http://security.ubuntu.com/ubuntu/"} +PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", + "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} +PRIMARY_ARCHES = ['amd64', 'i386'] +PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el'] + + +def get_default_mirrors(arch=None, target=None): + """returns the default mirrors for the target. These depend on the + architecture, for more see: + https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" + if arch is None: + arch = util.get_architecture(target) + if arch in PRIMARY_ARCHES: + return PRIMARY_ARCH_MIRRORS.copy() + if arch in PORTS_ARCHES: + return PORTS_MIRRORS.copy() + raise ValueError("No default mirror known for arch %s" % arch) + + +def handle(name, ocfg, cloud, log, _): + """process the config for apt_config. This can be called from + curthooks if a global apt config was provided or via the "apt" + standalone command.""" + # keeping code close to curtin codebase via entry handler + target = None + if log is not None: + global LOG + LOG = log + # feed back converted config, but only work on the subset under 'apt' + ocfg = convert_to_v3_apt_format(ocfg) + cfg = ocfg.get('apt', {}) + + if not isinstance(cfg, dict): + raise ValueError("Expected dictionary for 'apt' config, found %s", + type(cfg)) + + LOG.debug("handling apt (module %s) with apt config '%s'", name, cfg) + + release = util.lsb_release(target=target)['codename'] + arch = util.get_architecture(target) + mirrors = find_apt_mirror_info(cfg, cloud, arch=arch) + LOG.debug("Apt Mirror info: %s", mirrors) + + apply_debconf_selections(cfg, target) + + if util.is_false(cfg.get('preserve_sources_list', False)): + generate_sources_list(cfg, release, mirrors, cloud) + rename_apt_lists(mirrors, target) try: apply_apt_config(cfg, APT_PROXY_FN, APT_CONFIG_FN) - except Exception as e: - log.warn("failed to proxy or apt config info: %s", e) + except (IOError, OSError): + LOG.exception("Failed to apply proxy or apt config info:") - # Process 'apt_sources' - if 'apt_sources' in cfg: + # Process 'apt_source -> sources {dict}' + if 'sources' in cfg: params = mirrors params['RELEASE'] = release - params['MIRROR'] = mirror + params['MIRROR'] = mirrors["MIRROR"] + matcher = None matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) if matchcfg: matcher = re.compile(matchcfg).search + + add_apt_sources(cfg['sources'], cloud, target=target, + template_params=params, aa_repo_match=matcher) + + +def debconf_set_selections(selections, target=None): + util.subp(['debconf-set-selections'], data=selections, target=target, + capture=True) + + +def dpkg_reconfigure(packages, target=None): + # For any packages that are already installed, but have preseed data + # we populate the debconf database, but the filesystem configuration + # would be preferred on a subsequent dpkg-reconfigure. + # so, what we have to do is "know" information about certain packages + # to unconfigure them. + unhandled = [] + to_config = [] + for pkg in packages: + if pkg in CONFIG_CLEANERS: + LOG.debug("unconfiguring %s", pkg) + CONFIG_CLEANERS[pkg](target) + to_config.append(pkg) else: - def matcher(x): - return False + unhandled.append(pkg) + + if len(unhandled): + LOG.warn("The following packages were installed and preseeded, " + "but cannot be unconfigured: %s", unhandled) + + if len(to_config): + util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + list(to_config), data=None, target=target, capture=True) + + +def apply_debconf_selections(cfg, target=None): + """apply_debconf_selections - push content to debconf""" + # debconf_selections: + # set1: | + # cloud-init cloud-init/datasources multiselect MAAS + # set2: pkg pkg/value string bar + selsets = cfg.get('debconf_selections') + if not selsets: + LOG.debug("debconf_selections was not set in config") + return - errors = add_apt_sources(cfg['apt_sources'], params, - aa_repo_match=matcher) - for e in errors: - log.warn("Add source error: %s", ':'.join(e)) + selections = '\n'.join( + [selsets[key] for key in sorted(selsets.keys())]) + debconf_set_selections(selections.encode() + b"\n", target=target) - dconf_sel = util.get_cfg_option_str(cfg, 'debconf_selections', False) - if dconf_sel: - log.debug("Setting debconf selections per cloud config") - try: - util.subp(('debconf-set-selections', '-'), dconf_sel) - except Exception: - util.logexc(log, "Failed to run debconf-set-selections") + # get a complete list of packages listed in input + pkgs_cfgd = set() + for key, content in selsets.items(): + for line in content.splitlines(): + if line.startswith("#"): + continue + pkg = re.sub(r"[:\s].*", "", line) + pkgs_cfgd.add(pkg) + + pkgs_installed = util.get_installed_packages(target) + + LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) + need_reconfig = pkgs_cfgd.intersection(pkgs_installed) + + if len(need_reconfig) == 0: + LOG.debug("no need for reconfig") + return + + dpkg_reconfigure(need_reconfig, target=target) + + +def clean_cloud_init(target): + """clean out any local cloud-init config""" + flist = glob.glob( + util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + + LOG.debug("cleaning cloud-init config from: %s", flist) + for dpkg_cfg in flist: + os.unlink(dpkg_cfg) def mirrorurl_to_apt_fileprefix(mirror): + """mirrorurl_to_apt_fileprefix + Convert a mirror url to the file prefix used by apt on disk to + store cache information for that mirror. + To do so do: + - take off ???:// + - drop tailing / + - convert in string / to _""" string = mirror - # take off http:// or ftp:// if string.endswith("/"): string = string[0:-1] pos = string.find("://") @@ -106,174 +208,365 @@ def mirrorurl_to_apt_fileprefix(mirror): return string -def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): - for (name, omirror) in old_mirrors.items(): +def rename_apt_lists(new_mirrors, target=None): + """rename_apt_lists - rename apt lists to preserve old cache data""" + default_mirrors = get_default_mirrors(util.get_architecture(target)) + + pre = util.target_path(target, APT_LISTS) + for (name, omirror) in default_mirrors.items(): nmirror = new_mirrors.get(name) if not nmirror: continue - oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror)) - nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror)) + + oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror) + nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror) if oprefix == nprefix: continue olen = len(oprefix) for filename in glob.glob("%s_*" % oprefix): - util.rename(filename, "%s%s" % (nprefix, filename[olen:])) - - -def get_release(): - (stdout, _stderr) = util.subp(['lsb_release', '-cs']) - return stdout.strip() - - -def generate_sources_list(cfg, codename, mirrors, cloud, log): - params = {'codename': codename} + newname = "%s%s" % (nprefix, filename[olen:]) + LOG.debug("Renaming apt list %s to %s", filename, newname) + try: + os.rename(filename, newname) + except OSError: + # since this is a best effort task, warn with but don't fail + LOG.warn("Failed to rename apt list:", exc_info=True) + + +def mirror_to_placeholder(tmpl, mirror, placeholder): + """mirror_to_placeholder + replace the specified mirror in a template with a placeholder string + Checks for existance of the expected mirror and warns if not found""" + if mirror not in tmpl: + LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl) + return tmpl.replace(mirror, placeholder) + + +def map_known_suites(suite): + """there are a few default names which will be auto-extended. + This comes at the inability to use those names literally as suites, + but on the other hand increases readability of the cfg quite a lot""" + mapping = {'updates': '$RELEASE-updates', + 'backports': '$RELEASE-backports', + 'security': '$RELEASE-security', + 'proposed': '$RELEASE-proposed', + 'release': '$RELEASE'} + try: + retsuite = mapping[suite] + except KeyError: + retsuite = suite + return retsuite + + +def disable_suites(disabled, src, release): + """reads the config for suites to be disabled and removes those + from the template""" + if not disabled: + return src + + retsrc = src + for suite in disabled: + suite = map_known_suites(suite) + releasesuite = templater.render_string(suite, {'RELEASE': release}) + LOG.debug("Disabling suite %s as %s", suite, releasesuite) + + newsrc = "" + for line in retsrc.splitlines(True): + if line.startswith("#"): + newsrc += line + continue + + # sources.list allow options in cols[1] which can have spaces + # so the actual suite can be [2] or later. example: + # deb [ arch=amd64,armel k=v ] http://example.com/debian + cols = line.split() + if len(cols) > 1: + pcol = 2 + if cols[1].startswith("["): + for col in cols[1:]: + pcol += 1 + if col.endswith("]"): + break + + if cols[pcol] == releasesuite: + line = '# suite disabled by cloud-init: %s' % line + newsrc += line + retsrc = newsrc + + return retsrc + + +def generate_sources_list(cfg, release, mirrors, cloud): + """generate_sources_list + create a source.list file based on a custom or default template + by replacing mirrors and release in the template""" + aptsrc = "/etc/apt/sources.list" + params = {'RELEASE': release, 'codename': release} for k in mirrors: params[k] = mirrors[k] + params[k.lower()] = mirrors[k] - custtmpl = cfg.get('apt_custom_sources_list', None) - if custtmpl is not None: - templater.render_string_to_file(custtmpl, - '/etc/apt/sources.list', params) - return - - template_fn = cloud.get_template_filename('sources.list.%s' % - (cloud.distro.name)) - if not template_fn: - template_fn = cloud.get_template_filename('sources.list') + tmpl = cfg.get('sources_list', None) + if tmpl is None: + LOG.info("No custom template provided, fall back to builtin") + template_fn = cloud.get_template_filename('sources.list.%s' % + (cloud.distro.name)) if not template_fn: - log.warn("No template found, not rendering /etc/apt/sources.list") + template_fn = cloud.get_template_filename('sources.list') + if not template_fn: + LOG.warn("No template found, not rendering /etc/apt/sources.list") return + tmpl = util.load_file(template_fn) - templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + rendered = templater.render_string(tmpl, params) + disabled = disable_suites(cfg.get('disable_suites'), rendered, release) + util.write_file(aptsrc, disabled, mode=0o644) -def add_apt_key_raw(key): +def add_apt_key_raw(key, target=None): """ actual adding of a key as defined in key argument to the system """ + LOG.debug("Adding key:\n'%s'", key) try: - util.subp(('apt-key', 'add', '-'), key) + util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) except util.ProcessExecutionError: - raise ValueError('failed to add apt GPG Key to apt keyring') + LOG.exception("failed to add apt GPG Key to apt keyring") + raise -def add_apt_key(ent): +def add_apt_key(ent, target=None): """ - add key to the system as defined in ent (if any) - supports raw keys or keyid's - The latter will as a first step fetch the raw key from a keyserver + Add key to the system as defined in ent (if any). + Supports raw keys or keyid's + The latter will as a first step fetched to get the raw key """ if 'keyid' in ent and 'key' not in ent: - keyserver = "keyserver.ubuntu.com" + keyserver = DEFAULT_KEYSERVER if 'keyserver' in ent: keyserver = ent['keyserver'] - ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver) + + ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver) if 'key' in ent: - add_apt_key_raw(ent['key']) + add_apt_key_raw(ent['key'], target) -def convert_to_new_format(srclist): - """convert_to_new_format - convert the old list based format to the new dict based one - """ - srcdict = {} - if isinstance(srclist, list): - for srcent in srclist: - if 'filename' not in srcent: - # file collides for multiple !filename cases for compatibility - # yet we need them all processed, so not same dictionary key - srcent['filename'] = "cloud_config_sources.list" - key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - else: - # all with filename use that as key (matching new format) - key = srcent['filename'] - srcdict[key] = srcent - elif isinstance(srclist, dict): - srcdict = srclist - else: - raise ValueError("unknown apt_sources format") - - return srcdict +def update_packages(cloud): + cloud.distro.update_package_sources() -def add_apt_sources(srclist, template_params=None, aa_repo_match=None): +def add_apt_sources(srcdict, cloud, target=None, template_params=None, + aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also + sources.list entry in 'srcdict'. When rendering template, also include the values in dictionary searchList """ if template_params is None: template_params = {} if aa_repo_match is None: - def _aa_repo_match(x): - return False - aa_repo_match = _aa_repo_match + raise ValueError('did not get a valid repo matcher') - errorlist = [] - srcdict = convert_to_new_format(srclist) + if not isinstance(srcdict, dict): + raise TypeError('unknown apt format: %s' % (srcdict)) for filename in srcdict: ent = srcdict[filename] + LOG.debug("adding source/key '%s'", ent) if 'filename' not in ent: ent['filename'] = filename - # keys can be added without specifying a source - try: - add_apt_key(ent) - except ValueError as detail: - errorlist.append([ent, detail]) + add_apt_key(ent, target) if 'source' not in ent: - errorlist.append(["", "missing source"]) continue source = ent['source'] source = templater.render_string(source, template_params) - if not ent['filename'].startswith(os.path.sep): + if not ent['filename'].startswith("/"): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) + if not ent['filename'].endswith(".list"): + ent['filename'] += ".list" if aa_repo_match(source): try: - util.subp(["add-apt-repository", source]) - except util.ProcessExecutionError as e: - errorlist.append([source, - ("add-apt-repository failed. " + str(e))]) + util.subp(["add-apt-repository", source], target=target) + except util.ProcessExecutionError: + LOG.exception("add-apt-repository failed.") + raise continue + sourcefn = util.target_path(target, ent['filename']) try: contents = "%s\n" % (source) - util.write_file(ent['filename'], contents, omode="ab") - except Exception: - errorlist.append([source, - "failed write to file %s" % ent['filename']]) + util.write_file(sourcefn, contents, omode="a") + except IOError as detail: + LOG.exception("failed write to file %s: %s", sourcefn, detail) + raise - return errorlist + update_packages(cloud) + return -def find_apt_mirror_info(cloud, cfg): - """find an apt_mirror given the cloud and cfg provided.""" - mirror = None +def convert_v1_to_v2_apt_format(srclist): + """convert v1 apt format to v2 (dict in apt_sources)""" + srcdict = {} + if isinstance(srclist, list): + LOG.debug("apt config: convert V1 to V2 format (source list to dict)") + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + elif isinstance(srclist, dict): + srcdict = srclist + else: + raise ValueError("unknown apt_sources format") + + return srcdict + + +def convert_key(oldcfg, aptcfg, oldkey, newkey): + """convert an old key to the new one if the old one exists + returns true if a key was found and converted""" + if oldcfg.get(oldkey, None) is not None: + aptcfg[newkey] = oldcfg.get(oldkey) + del oldcfg[oldkey] + return True + return False + + +def convert_mirror(oldcfg, aptcfg): + """convert old apt_mirror keys into the new more advanced mirror spec""" + keymap = [('apt_mirror', 'uri'), + ('apt_mirror_search', 'search'), + ('apt_mirror_search_dns', 'search_dns')] + converted = False + newmcfg = {'arches': ['default']} + for oldkey, newkey in keymap: + if convert_key(oldcfg, newmcfg, oldkey, newkey): + converted = True + + # only insert new style config if anything was converted + if converted: + aptcfg['primary'] = [newmcfg] + + +def convert_v2_to_v3_apt_format(oldcfg): + """convert old to new keys and adapt restructured mirror spec""" + oldkeys = ['apt_sources', 'apt_mirror', 'apt_mirror_search', + 'apt_mirror_search_dns', 'apt_proxy', 'apt_http_proxy', + 'apt_ftp_proxy', 'apt_https_proxy', + 'apt_preserve_sources_list', 'apt_custom_sources_list', + 'add_apt_repo_match'] + needtoconvert = [] + for oldkey in oldkeys: + if oldcfg.get(oldkey, None) is not None: + needtoconvert.append(oldkey) + + # no old config, so no new one to be created + if not needtoconvert: + return oldcfg + LOG.debug("apt config: convert V2 to V3 format for keys '%s'", + ", ".join(needtoconvert)) + + if oldcfg.get('apt', None) is not None: + msg = ("Error in apt configuration: " + "old and new format of apt features are mutually exclusive " + "('apt':'%s' vs '%s' key)" % (oldcfg.get('apt', None), + ", ".join(needtoconvert))) + LOG.error(msg) + raise ValueError(msg) + + # create new format from old keys + aptcfg = {} + + # renames / moves under the apt key + convert_key(oldcfg, aptcfg, 'add_apt_repo_match', 'add_apt_repo_match') + convert_key(oldcfg, aptcfg, 'apt_proxy', 'proxy') + convert_key(oldcfg, aptcfg, 'apt_http_proxy', 'http_proxy') + convert_key(oldcfg, aptcfg, 'apt_https_proxy', 'https_proxy') + convert_key(oldcfg, aptcfg, 'apt_ftp_proxy', 'ftp_proxy') + convert_key(oldcfg, aptcfg, 'apt_custom_sources_list', 'sources_list') + convert_key(oldcfg, aptcfg, 'apt_preserve_sources_list', + 'preserve_sources_list') + # dict format not changed since v2, just renamed and moved + convert_key(oldcfg, aptcfg, 'apt_sources', 'sources') + + convert_mirror(oldcfg, aptcfg) + + for oldkey in oldkeys: + if oldcfg.get(oldkey, None) is not None: + raise ValueError("old apt key '%s' left after conversion" % oldkey) + + # insert new format into config and return full cfg with only v3 content + oldcfg['apt'] = aptcfg + return oldcfg + + +def convert_to_v3_apt_format(cfg): + """convert the old list based format to the new dict based one. After that + convert the old dict keys/format to v3 a.k.a 'new apt config'""" + # V1 -> V2, the apt_sources entry from list to dict + apt_sources = cfg.get('apt_sources', None) + if apt_sources is not None: + cfg['apt_sources'] = convert_v1_to_v2_apt_format(apt_sources) + + # V2 -> V3, move all former globals under the "apt" key + # Restructure into new key names and mirror hierarchy + cfg = convert_v2_to_v3_apt_format(cfg) + + return cfg + + +def search_for_mirror(candidates): + """ + Search through a list of mirror urls for one that works + This needs to return quickly. + """ + if candidates is None: + return None + + LOG.debug("search for mirror in candidates: '%s'", candidates) + for cand in candidates: + try: + if util.is_resolvable_url(cand): + LOG.debug("found working mirror: '%s'", cand) + return cand + except Exception: + pass + return None - # this is less preferred way of specifying mirror preferred would be to - # use the distro's search or package_mirror. - mirror = cfg.get("apt_mirror", None) - search = cfg.get("apt_mirror_search", None) - if not mirror and search: - mirror = util.search_for_mirror(search) +def search_for_mirror_dns(configured, mirrortype, cfg, cloud): + """ + Try to resolve a list of predefines DNS names to pick mirrors + """ + mirror = None - if (not mirror and - util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): + if configured: mydom = "" doms = [] + if mirrortype == "primary": + mirrordns = "mirror" + elif mirrortype == "security": + mirrordns = "security-mirror" + else: + raise ValueError("unknown mirror type") + # if we have a fqdn, then search its domain portion first - (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) + (_, fqdn) = util.get_hostname_fqdn(cfg, cloud) mydom = ".".join(fqdn.split(".")[1:]) if mydom: doms.append(".%s" % mydom) @@ -282,38 +575,136 @@ def find_apt_mirror_info(cloud, cfg): mirror_list = [] distro = cloud.distro.name - mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) + mirrorfmt = "http://%s-%s%s/%s" % (distro, mirrordns, "%s", distro) for post in doms: mirror_list.append(mirrorfmt % (post)) - mirror = util.search_for_mirror(mirror_list) + mirror = search_for_mirror(mirror_list) + + return mirror + +def update_mirror_info(pmirror, smirror, arch, cloud): + """sets security mirror to primary if not defined. + returns defaults if no mirrors are defined""" + if pmirror is not None: + if smirror is None: + smirror = pmirror + return {'PRIMARY': pmirror, + 'SECURITY': smirror} + + # None specified at all, get default mirrors from cloud mirror_info = cloud.datasource.get_package_mirror_info() + if mirror_info: + # get_package_mirror_info() returns a dictionary with + # arbitrary key/value pairs including 'primary' and 'security' keys. + # caller expects dict with PRIMARY and SECURITY. + m = mirror_info.copy() + m['PRIMARY'] = m['primary'] + m['SECURITY'] = m['security'] + + return m + + # if neither apt nor cloud configured mirrors fall back to + return get_default_mirrors(arch) + + +def get_arch_mirrorconfig(cfg, mirrortype, arch): + """out of a list of potential mirror configurations select + and return the one matching the architecture (or default)""" + # select the mirror specification (if-any) + mirror_cfg_list = cfg.get(mirrortype, None) + if mirror_cfg_list is None: + return None + + # select the specification matching the target arch + default = None + for mirror_cfg_elem in mirror_cfg_list: + arches = mirror_cfg_elem.get("arches") + if arch in arches: + return mirror_cfg_elem + if "default" in arches: + default = mirror_cfg_elem + return default + + +def get_mirror(cfg, mirrortype, arch, cloud): + """pass the three potential stages of mirror specification + returns None is neither of them found anything otherwise the first + hit is returned""" + mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch) + if mcfg is None: + return None + + # directly specified + mirror = mcfg.get("uri", None) + + # fallback to search if specified + if mirror is None: + # list of mirrors to try to resolve + mirror = search_for_mirror(mcfg.get("search", None)) + + # fallback to search_dns if specified + if mirror is None: + # list of mirrors to try to resolve + mirror = search_for_mirror_dns(mcfg.get("search_dns", None), + mirrortype, cfg, cloud) + + return mirror + + +def find_apt_mirror_info(cfg, cloud, arch=None): + """find_apt_mirror_info + find an apt_mirror given the cfg provided. + It can check for separate config of primary and security mirrors + If only primary is given security is assumed to be equal to primary + If the generic apt_mirror is given that is defining for both + """ - # this is a bit strange. - # if mirror is set, then one of the legacy options above set it - # but they do not cover security. so we need to get that from - # get_package_mirror_info - if mirror: - mirror_info.update({'primary': mirror}) + if arch is None: + arch = util.get_architecture() + LOG.debug("got arch for mirror selection: %s", arch) + pmirror = get_mirror(cfg, "primary", arch, cloud) + LOG.debug("got primary mirror: %s", pmirror) + smirror = get_mirror(cfg, "security", arch, cloud) + LOG.debug("got security mirror: %s", smirror) + + mirror_info = update_mirror_info(pmirror, smirror, arch, cloud) + + # less complex replacements use only MIRROR, derive from primary + mirror_info["MIRROR"] = mirror_info["PRIMARY"] return mirror_info def apply_apt_config(cfg, proxy_fname, config_fname): + """apply_apt_config + Applies any apt*proxy config from if specified + """ # Set up any apt proxy - cfgs = (('apt_proxy', 'Acquire::HTTP::Proxy "%s";'), - ('apt_http_proxy', 'Acquire::HTTP::Proxy "%s";'), - ('apt_ftp_proxy', 'Acquire::FTP::Proxy "%s";'), - ('apt_https_proxy', 'Acquire::HTTPS::Proxy "%s";')) + cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), + ('http_proxy', 'Acquire::http::Proxy "%s";'), + ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'), + ('https_proxy', 'Acquire::https::Proxy "%s";')) proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] if len(proxies): + LOG.debug("write apt proxy info to %s", proxy_fname) util.write_file(proxy_fname, '\n'.join(proxies) + '\n') elif os.path.isfile(proxy_fname): util.del_file(proxy_fname) + LOG.debug("no apt proxy configured, removed %s", proxy_fname) - if cfg.get('apt_config', None): - util.write_file(config_fname, cfg.get('apt_config')) + if cfg.get('conf', None): + LOG.debug("write apt config info to %s", config_fname) + util.write_file(config_fname, cfg.get('conf')) elif os.path.isfile(config_fname): util.del_file(config_fname) + LOG.debug("no apt config configured, removed %s", config_fname) + + +CONFIG_CLEANERS = { + 'cloud-init': clean_cloud_init, +} + +# vi: ts=4 expandtab syntax=python diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py index 6a76d785..5bbff513 100644 --- a/cloudinit/gpg.py +++ b/cloudinit/gpg.py @@ -36,11 +36,11 @@ def export_armour(key): return armour -def receive_key(key, keyserver): +def recv_key(key, keyserver): """Receive gpg key from the specified keyserver""" LOG.debug('Receive gpg key "%s"', key) try: - util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key], + util.subp(["gpg", "--keyserver", keyserver, "--recv", key], capture=True) except util.ProcessExecutionError as error: raise ValueError(('Failed to import key "%s" ' @@ -57,12 +57,12 @@ def delete_key(key): LOG.warn('Failed delete key "%s": %s', key, error) -def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"): +def getkeybyid(keyid, keyserver='keyserver.ubuntu.com'): """get gpg keyid from keyserver""" armour = export_armour(keyid) if not armour: try: - receive_key(keyid, keyserver=keyserver) + recv_key(keyid, keyserver=keyserver) armour = export_armour(keyid) except ValueError: LOG.exception('Failed to obtain gpg key %s', keyid) diff --git a/cloudinit/util.py b/cloudinit/util.py index 226628cc..db80ca96 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -61,6 +61,10 @@ from cloudinit import version from cloudinit.settings import (CFG_BUILTIN) +try: + string_types = (basestring,) +except NameError: + string_types = (str,) _DNS_REDIRECT_IP = None LOG = logging.getLogger(__name__) @@ -82,6 +86,71 @@ CONTAINER_TESTS = (['systemd-detect-virt', '--quiet', '--container'], PROC_CMDLINE = None +_LSB_RELEASE = {} + + +def get_architecture(target=None): + out, _ = subp(['dpkg', '--print-architecture'], capture=True, + target=target) + return out.strip() + + +def _lsb_release(target=None): + fmap = {'Codename': 'codename', 'Description': 'description', + 'Distributor ID': 'id', 'Release': 'release'} + + data = {} + try: + out, _ = subp(['lsb_release', '--all'], capture=True, target=target) + for line in out.splitlines(): + fname, _, val = line.partition(":") + if fname in fmap: + data[fmap[fname]] = val.strip() + missing = [k for k in fmap.values() if k not in data] + if len(missing): + LOG.warn("Missing fields in lsb_release --all output: %s", + ','.join(missing)) + + except ProcessExecutionError as err: + LOG.warn("Unable to get lsb_release --all: %s", err) + data = {v: "UNAVAILABLE" for v in fmap.values()} + + return data + + +def lsb_release(target=None): + if target_path(target) != "/": + # do not use or update cache if target is provided + return _lsb_release(target) + + global _LSB_RELEASE + if not _LSB_RELEASE: + data = _lsb_release() + _LSB_RELEASE.update(data) + return _LSB_RELEASE + + +def target_path(target, path=None): + # return 'path' inside target, accepting target as None + if target in (None, ""): + target = "/" + elif not isinstance(target, string_types): + raise ValueError("Unexpected input for target: %s" % target) + else: + target = os.path.abspath(target) + # abspath("//") returns "//" specifically for 2 slashes. + if target.startswith("//"): + target = target[1:] + + if not path: + return target + + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. + while len(path) and path[0] == "/": + path = path[1:] + + return os.path.join(target, path) + def decode_binary(blob, encoding='utf-8'): # Converts a binary type into a text type using given encoding. @@ -1688,10 +1757,20 @@ def delete_dir_contents(dirname): def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, - logstring=False): + logstring=False, decode="replace", target=None): + + # not supported in cloud-init (yet), for now kept in the call signature + # to ease maintaining code shared between cloud-init and curtin + if target is not None: + raise ValueError("target arg not supported by cloud-init") + if rcs is None: rcs = [0] + + devnull_fp = None try: + if target_path(target) != "/": + args = ['chroot', target] + list(args) if not logstring: LOG.debug(("Running command %s with allowed return codes %s" @@ -1700,33 +1779,52 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, LOG.debug(("Running hidden command to protect sensitive " "input/output logstring: %s"), logstring) - if not capture: - stdout = None - stderr = None - else: + stdin = None + stdout = None + stderr = None + if capture: stdout = subprocess.PIPE stderr = subprocess.PIPE - stdin = subprocess.PIPE - kws = dict(stdout=stdout, stderr=stderr, stdin=stdin, - env=env, shell=shell) - if six.PY3: - # Use this so subprocess output will be (Python 3) str, not bytes. - kws['universal_newlines'] = True - sp = subprocess.Popen(args, **kws) + if data is None: + # using devnull assures any reads get null, rather + # than possibly waiting on input. + devnull_fp = open(os.devnull) + stdin = devnull_fp + else: + stdin = subprocess.PIPE + if not isinstance(data, bytes): + data = data.encode() + + sp = subprocess.Popen(args, stdout=stdout, + stderr=stderr, stdin=stdin, + env=env, shell=shell) (out, err) = sp.communicate(data) + + # Just ensure blank instead of none. + if not out and capture: + out = b'' + if not err and capture: + err = b'' + if decode: + def ldecode(data, m='utf-8'): + if not isinstance(data, bytes): + return data + return data.decode(m, errors=decode) + + out = ldecode(out) + err = ldecode(err) except OSError as e: raise ProcessExecutionError(cmd=args, reason=e, errno=e.errno) + finally: + if devnull_fp: + devnull_fp.close() + rc = sp.returncode if rc not in rcs: raise ProcessExecutionError(stdout=out, stderr=err, exit_code=rc, cmd=args) - # Just ensure blank instead of none?? (iff capturing) - if not out and capture: - out = '' - if not err and capture: - err = '' return (out, err) @@ -2251,3 +2349,18 @@ def message_from_string(string): if sys.version_info[:2] < (2, 7): return email.message_from_file(six.StringIO(string)) return email.message_from_string(string) + + +def get_installed_packages(target=None): + (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) + + pkgs_inst = set() + for line in out.splitlines(): + try: + (state, pkg, _) = line.split(None, 2) + except ValueError: + continue + if state.startswith("hi") or state.startswith("ii"): + pkgs_inst.add(re.sub(":.*", "", pkg)) + + return pkgs_inst diff --git a/doc/examples/cloud-config-add-apt-repos.txt b/doc/examples/cloud-config-add-apt-repos.txt index be9d5472..22ef7612 100644 --- a/doc/examples/cloud-config-add-apt-repos.txt +++ b/doc/examples/cloud-config-add-apt-repos.txt @@ -4,18 +4,21 @@ # # Default: auto select based on cloud metadata # in ec2, the default is .archive.ubuntu.com -# apt_mirror: -# use the provided mirror -# apt_mirror_search: -# search the list for the first mirror. -# this is currently very limited, only verifying that -# the mirror is dns resolvable or an IP address +# apt: +# primary: +# - arches [default] +# uri: +# use the provided mirror +# search: +# search the list for the first mirror. +# this is currently very limited, only verifying that +# the mirror is dns resolvable or an IP address # -# if neither apt_mirror nor apt_mirror search is set (the default) +# if neither mirror is set (the default) # then use the mirror provided by the DataSource found. # In EC2, that means using .ec2.archive.ubuntu.com -# -# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is +# +# if no mirror is provided by the DataSource, but 'search_dns' is # true, then search for dns names '-mirror' in each of # - fqdn of this host per cloud metadata # - localdomain @@ -27,8 +30,19 @@ # up and expose them only by creating dns entries. # # if none of that is found, then the default distro mirror is used -apt_mirror: http://us.archive.ubuntu.com/ubuntu/ -apt_mirror_search: - - http://local-mirror.mydomain - - http://archive.ubuntu.com -apt_mirror_search_dns: False +apt: + primary: + - arches: [default] + uri: http://us.archive.ubuntu.com/ubuntu/ +# or +apt: + primary: + - arches: [default] + search: + - http://local-mirror.mydomain + - http://archive.ubuntu.com +# or +apt: + primary: + - arches: [default] + search_dns: True diff --git a/doc/examples/cloud-config-apt.txt b/doc/examples/cloud-config-apt.txt new file mode 100644 index 00000000..1a0fc6f2 --- /dev/null +++ b/doc/examples/cloud-config-apt.txt @@ -0,0 +1,328 @@ +# apt_pipelining (configure Acquire::http::Pipeline-Depth) +# Default: disables HTTP pipelining. Certain web servers, such +# as S3 do not pipeline properly (LP: #948461). +# Valid options: +# False/default: Disables pipelining for APT +# None/Unchanged: Use OS default +# Number: Set pipelining to some number (not recommended) +apt_pipelining: False + +## apt config via system_info: +# under the 'system_info', you can customize cloud-init's interaction +# with apt. +# system_info: +# apt_get_command: [command, argument, argument] +# apt_get_upgrade_subcommand: dist-upgrade +# +# apt_get_command: +# To specify a different 'apt-get' command, set 'apt_get_command'. +# This must be a list, and the subcommand (update, upgrade) is appended to it. +# default is: +# ['apt-get', '--option=Dpkg::Options::=--force-confold', +# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] +# +# apt_get_upgrade_subcommand: "dist-upgrade" +# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. +# This is the subcommand that is invoked for package_upgrade. +# +# apt_get_wrapper: +# command: eatmydata +# enabled: [True, False, "auto"] +# + +# Install additional packages on first boot +# +# Default: none +# +# if packages are specified, this apt_update will be set to true + +packages: ['pastebinit'] + +apt: + # The apt config consists of two major "areas". + # + # On one hand there is the global configuration for the apt feature. + # + # On one hand (down in this file) there is the source dictionary which allows + # to define various entries to be considered by apt. + + ############################################################################## + # Section 1: global apt configuration + # + # The following examples number the top keys to ease identification in + # discussions. + + # 1.1 preserve_sources_list + # + # Preserves the existing /etc/apt/sources.list + # Default: false - do overwrite sources_list. If set to true then any + # "mirrors" configuration will have no effect. + # Set to true to avoid affecting sources.list. In that case only + # "extra" source specifications will be written into + # /etc/apt/sources.list.d/* + preserve_sources_list: true + + # 1.2 disable_suites + # + # This is an empty list by default, so nothing is disabled. + # + # If given, those suites are removed from sources.list after all other + # modifications have been made. + # Suites are even disabled if no other modification was made, + # but not if is preserve_sources_list is active. + # There is a special alias “$RELEASE” as in the sources that will be replace + # by the matching release. + # + # To ease configuration and improve readability the following common ubuntu + # suites will be automatically mapped to their full definition. + # updates => $RELEASE-updates + # backports => $RELEASE-backports + # security => $RELEASE-security + # proposed => $RELEASE-proposed + # release => $RELEASE + # + # There is no harm in specifying a suite to be disabled that is not found in + # the source.list file (just a no-op then) + # + # Note: Lines don’t get deleted, but disabled by being converted to a comment. + # The following example disables all usual defaults except $RELEASE-security. + # On top it disables a custom suite called "mysuite" + disable_suites: [$RELEASE-updates, backports, $RELEASE, mysuite] + + # 1.3 primary/security archives + # + # Default: none - instead it is auto select based on cloud metadata + # so if neither "uri" nor "search", nor "search_dns" is set (the default) + # then use the mirror provided by the DataSource found. + # In EC2, that means using .ec2.archive.ubuntu.com + # + # define a custom (e.g. localized) mirror that will be used in sources.list + # and any custom sources entries for deb / deb-src lines. + # + # One can set primary and security mirror to different uri's + # the child elements to the keys primary and secondary are equivalent + primary: + # arches is list of architectures the following config applies to + # the special keyword "default" applies to any architecture not explicitly + # listed. + - arches: [amd64, i386, default] + # uri is just defining the target as-is + uri: http://us.archive.ubuntu.com/ubuntu + # + # via search one can define lists that are tried one by one. + # The first with a working DNS resolution (or if it is an IP) will be + # picked. That way one can keep one configuration for multiple + # subenvironments that select the working one. + search: + - http://cool.but-sometimes-unreachable.com/ubuntu + - http://us.archive.ubuntu.com/ubuntu + # if no mirror is provided by uri or search but 'search_dns' is + # true, then search for dns names '-mirror' in each of + # - fqdn of this host per cloud metadata + # - localdomain + # - no domain (which would search domains listed in /etc/resolv.conf) + # If there is a dns entry for -mirror, then it is assumed that + # there is a distro mirror at http://-mirror./ + # + # That gives the cloud provider the opportunity to set mirrors of a distro + # up and expose them only by creating dns entries. + # + # if none of that is found, then the default distro mirror is used + search_dns: true + # + # If multiple of a category are given + # 1. uri + # 2. search + # 3. search_dns + # the first defining a valid mirror wins (in the order as defined here, + # not the order as listed in the config). + # + - arches: [s390x, arm64] + # as above, allowing to have one config for different per arch mirrors + # security is optional, if not defined it is set to the same value as primary + security: + uri: http://security.ubuntu.com/ubuntu + # If search_dns is set for security the searched pattern is: + # -security-mirror + + # if no mirrors are specified at all, or all lookups fail it will try + # to get them from the cloud datasource and if those neither provide one fall + # back to: + # primary: http://archive.ubuntu.com/ubuntu + # security: http://security.ubuntu.com/ubuntu + + # 1.4 sources_list + # + # Provide a custom template for rendering sources.list + # without one provided cloud-init uses builtin templates for + # ubuntu and debian. + # Within these sources.list templates you can use the following replacement + # variables (all have sane Ubuntu defaults, but mirrors can be overwritten + # as needed (see above)): + # => $RELEASE, $MIRROR, $PRIMARY, $SECURITY + sources_list: | # written by cloud-init custom template + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + + # 1.5 conf + # + # Any apt config string that will be made available to apt + # see the APT.CONF(5) man page for details what can be specified + conf: | # APT config + APT { + Get { + Assume-Yes "true"; + Fix-Broken "true"; + }; + }; + + # 1.6 (http_|ftp_|https_)proxy + # + # Proxies are the most common apt.conf option, so that for simplified use + # there is a shortcut for those. Those get automatically translated into the + # correct Acquire::*::Proxy statements. + # + # note: proxy actually being a short synonym to http_proxy + proxy: http://[[user][:pass]@]host[:port]/ + http_proxy: http://[[user][:pass]@]host[:port]/ + ftp_proxy: ftp://[[user][:pass]@]host[:port]/ + https_proxy: https://[[user][:pass]@]host[:port]/ + + # 1.7 add_apt_repo_match + # + # 'source' entries in apt-sources that match this python regex + # expression will be passed to add-apt-repository + # The following example is also the builtin default if nothing is specified + add_apt_repo_match: '^[\w-]+:\w' + + + ############################################################################## + # Section 2: source list entries + # + # This is a dictionary (unlike most block/net which are lists) + # + # The key of each source entry is the filename and will be prepended by + # /etc/apt/sources.list.d/ if it doesn't start with a '/'. + # If it doesn't end with .list it will be appended so that apt picks up it's + # configuration. + # + # Whenever there is no content to be written into such a file, the key is + # not used as filename - yet it can still be used as index for merging + # configuration. + # + # The values inside the entries consost of the following optional entries: + # 'source': a sources.list entry (some variable replacements apply) + # 'keyid': providing a key to import via shortid or fingerprint + # 'key': providing a raw PGP key + # 'keyserver': specify an alternate keyserver to pull keys from that + # were specified by keyid + + # This allows merging between multiple input files than a list like: + # cloud-config1 + # sources: + # s1: {'key': 'key1', 'source': 'source1'} + # cloud-config2 + # sources: + # s2: {'key': 'key2'} + # s1: {'keyserver': 'foo'} + # This would be merged to + # sources: + # s1: + # keyserver: foo + # key: key1 + # source: source1 + # s2: + # key: key2 + # + # The following examples number the subfeatures per sources entry to ease + # identification in discussions. + + + sources: + curtin-dev-ppa.list: + # 2.1 source + # + # Creates a file in /etc/apt/sources.list.d/ for the sources list entry + # based on the key: "/etc/apt/sources.list.d/curtin-dev-ppa.list" + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + + # 2.2 keyid + # + # Importing a gpg key for a given key id. Used keyserver defaults to + # keyserver.ubuntu.com + keyid: F430BBA5 # GPG key ID published on a key server + + ignored1: + # 2.3 PPA shortcut + # + # Setup correct apt sources.list line and Auto-Import the signing key + # from LP + # + # See https://help.launchpad.net/Packaging/PPA for more information + # this requires 'add-apt-repository'. This will create a file in + # /etc/apt/sources.list.d automatically, therefore the key here is + # ignored as filename in those cases. + source: "ppa:curtin-dev/test-archive" # Quote the string + + my-repo2.list: + # 2.4 replacement variables + # + # sources can use $MIRROR, $PRIMARY, $SECURITY and $RELEASE replacement + # variables. + # They will be replaced with the default or specified mirrors and the + # running release. + # The entry below would be possibly turned into: + # source: deb http://archive.ubuntu.com/ubuntu xenial multiverse + source: deb $MIRROR $RELEASE multiverse + + my-repo3.list: + # this would have the same end effect as 'ppa:curtin-dev/test-archive' + source: "deb http://ppa.launchpad.net/curtin-dev/test-archive/ubuntu xenial main" + keyid: F430BBA5 # GPG key ID published on the key server + filename: curtin-dev-ppa.list + + ignored2: + # 2.5 key only + # + # this would only import the key without adding a ppa or other source spec + # since this doesn't generate a source.list file the filename key is ignored + keyid: F430BBA5 # GPG key ID published on a key server + + ignored3: + # 2.6 key id alternatives + # + # Keyid's can also be specified via their long fingerprints + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + + ignored4: + # 2.7 alternative keyservers + # + # One can also specify alternative keyservers to fetch keys from. + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + keyserver: pgp.mit.edu + + + my-repo4.list: + # 2.8 raw key + # + # The apt signing key can also be specified by providing a pgp public key + # block. Providing the PGP key this way is the most robust method for + # specifying a key, as it removes dependency on a remote key server. + # + # As with keyid's this can be specified with or without some actual source + # content. + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- diff --git a/doc/examples/cloud-config-chef-oneiric.txt b/doc/examples/cloud-config-chef-oneiric.txt index 2e5f4b16..75c9aeed 100644 --- a/doc/examples/cloud-config-chef-oneiric.txt +++ b/doc/examples/cloud-config-chef-oneiric.txt @@ -11,39 +11,40 @@ # The default is to install from packages. # Key from http://apt.opscode.com/packages@opscode.com.gpg.key -apt_sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) - - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- +apt: + sources: + - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) + + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt index b886cba2..75d78a15 100644 --- a/doc/examples/cloud-config-chef.txt +++ b/doc/examples/cloud-config-chef.txt @@ -11,39 +11,40 @@ # The default is to install from packages. # Key from http://apt.opscode.com/packages@opscode.com.gpg.key -apt_sources: - - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" - key: | - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: GnuPG v1.4.9 (GNU/Linux) - - mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu - twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 - dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC - JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W - ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I - XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe - DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm - sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO - Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ - YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG - CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K - +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR - lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh - DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu - wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx - EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g - w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 - AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN - QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X - Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ - 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V - Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL - zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb - DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG - 0GLl8EkfA8uhluM= - =zKAm - -----END PGP PUBLIC KEY BLOCK----- +apt: + sources: + - source: "deb http://apt.opscode.com/ $RELEASE-0.10 main" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1.4.9 (GNU/Linux) + + mQGiBEppC7QRBADfsOkZU6KZK+YmKw4wev5mjKJEkVGlus+NxW8wItX5sGa6kdUu + twAyj7Yr92rF+ICFEP3gGU6+lGo0Nve7KxkN/1W7/m3G4zuk+ccIKmjp8KS3qn99 + dxy64vcji9jIllVa+XXOGIp0G8GEaj7mbkixL/bMeGfdMlv8Gf2XPpp9vwCgn/GC + JKacfnw7MpLKUHOYSlb//JsEAJqao3ViNfav83jJKEkD8cf59Y8xKia5OpZqTK5W + ShVnNWS3U5IVQk10ZDH97Qn/YrK387H4CyhLE9mxPXs/ul18ioiaars/q2MEKU2I + XKfV21eMLO9LYd6Ny/Kqj8o5WQK2J6+NAhSwvthZcIEphcFignIuobP+B5wNFQpe + DbKfA/0WvN2OwFeWRcmmd3Hz7nHTpcnSF+4QX6yHRF/5BgxkG6IqBIACQbzPn6Hm + sMtm/SVf11izmDqSsQptCrOZILfLX/mE+YOl+CwWSHhl+YsFts1WOuh1EhQD26aO + Z84HuHV5HFRWjDLw9LriltBVQcXbpfSrRP5bdr7Wh8vhqJTPjrQnT3BzY29kZSBQ + YWNrYWdlcyA8cGFja2FnZXNAb3BzY29kZS5jb20+iGAEExECACAFAkppC7QCGwMG + CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRApQKupg++Caj8sAKCOXmdG36gWji/K + +o+XtBfvdMnFYQCfTCEWxRy2BnzLoBBFCjDSK6sJqCu5Ag0ESmkLtBAIAIO2SwlR + lU5i6gTOp42RHWW7/pmW78CwUqJnYqnXROrt3h9F9xrsGkH0Fh1FRtsnncgzIhvh + DLQnRHnkXm0ws0jV0PF74ttoUT6BLAUsFi2SPP1zYNJ9H9fhhK/pjijtAcQwdgxu + wwNJ5xCEscBZCjhSRXm0d30bK1o49Cow8ZIbHtnXVP41c9QWOzX/LaGZsKQZnaMx + EzDk8dyyctR2f03vRSVyTFGgdpUcpbr9eTFVgikCa6ODEBv+0BnCH6yGTXwBid9g + w0o1e/2DviKUWCC+AlAUOubLmOIGFBuI4UR+rux9affbHcLIOTiKQXv79lW3P7W8 + AAfniSQKfPWXrrcAAwUH/2XBqD4Uxhbs25HDUUiM/m6Gnlj6EsStg8n0nMggLhuN + QmPfoNByMPUqvA7sULyfr6xCYzbzRNxABHSpf85FzGQ29RF4xsA4vOOU8RDIYQ9X + Q8NqqR6pydprRFqWe47hsAN7BoYuhWqTtOLSBmnAnzTR5pURoqcquWYiiEavZixJ + 3ZRAq/HMGioJEtMFrvsZjGXuzef7f0ytfR1zYeLVWnL9Bd32CueBlI7dhYwkFe+V + Ep5jWOCj02C1wHcwt+uIRDJV6TdtbIiBYAdOMPk15+VBdweBXwMuYXr76+A7VeDL + zIhi7tKFo6WiwjKZq0dzctsJJjtIfr4K4vbiD9Ojg1iISQQYEQIACQUCSmkLtAIb + DAAKCRApQKupg++CauISAJ9CxYPOKhOxalBnVTLeNUkAHGg2gACeIsbobtaD4ZHG + 0GLl8EkfA8uhluM= + =zKAm + -----END PGP PUBLIC KEY BLOCK----- chef: diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 3cc9c055..190029e4 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -18,256 +18,7 @@ package_upgrade: true # Aliases: apt_reboot_if_required package_reboot_if_required: true -# Add apt repositories -# -# Default: auto select based on cloud metadata -# in ec2, the default is .archive.ubuntu.com -# apt_mirror: -# use the provided mirror -# apt_mirror_search: -# search the list for the first mirror. -# this is currently very limited, only verifying that -# the mirror is dns resolvable or an IP address -# -# if neither apt_mirror nor apt_mirror search is set (the default) -# then use the mirror provided by the DataSource found. -# In EC2, that means using .ec2.archive.ubuntu.com -# -# if no mirror is provided by the DataSource, and 'apt_mirror_search_dns' is -# true, then search for dns names '-mirror' in each of -# - fqdn of this host per cloud metadata -# - localdomain -# - no domain (which would search domains listed in /etc/resolv.conf) -# If there is a dns entry for -mirror, then it is assumed that there -# is a distro mirror at http://-mirror./ -# -# That gives the cloud provider the opportunity to set mirrors of a distro -# up and expose them only by creating dns entries. -# -# if none of that is found, then the default distro mirror is used -apt_mirror: http://us.archive.ubuntu.com/ubuntu/ -apt_mirror_search: - - http://local-mirror.mydomain - - http://archive.ubuntu.com - -apt_mirror_search_dns: False - -# apt_proxy (configure Acquire::HTTP::Proxy) -# 'apt_http_proxy' is an alias for 'apt_proxy'. -# Also, available are 'apt_ftp_proxy' and 'apt_https_proxy'. -# These affect Acquire::FTP::Proxy and Acquire::HTTPS::Proxy respectively -apt_proxy: http://my.apt.proxy:3128 - -# apt_pipelining (configure Acquire::http::Pipeline-Depth) -# Default: disables HTTP pipelining. Certain web servers, such -# as S3 do not pipeline properly (LP: #948461). -# Valid options: -# False/default: Disables pipelining for APT -# None/Unchanged: Use OS default -# Number: Set pipelining to some number (not recommended) -apt_pipelining: False - -# Preserve existing /etc/apt/sources.list -# Default: overwrite sources_list with mirror. If this is true -# then apt_mirror above will have no effect -apt_preserve_sources_list: true - -# Provide a custom template for rendering sources.list -# Default: a default template for Ubuntu/Debain will be used as packaged in -# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl -# Debian: /etc/cloud/templates/sources.list.debian.tmpl -# Others: n/a -# This will follow the normal mirror/codename replacement rules before -# being written to disk. -apt_custom_sources_list: | - ## template:jinja - ## Note, this file is written by cloud-init on first boot of an instance - ## modifications made here will not survive a re-bundle. - ## if you wish to make changes you can: - ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg - ## or do the same in user-data - ## b.) add sources in /etc/apt/sources.list.d - ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - deb {{mirror}} {{codename}} main restricted - deb-src {{mirror}} {{codename}} main restricted - - # could drop some of the usually used entries - - # could refer to other mirrors - deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse - deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse - deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse - - # or even more uncommon examples like local or NFS mounted repos, - # eventually whatever is compatible with sources.list syntax - deb file:/home/apt/debian unstable main contrib non-free - -# 'source' entries in apt-sources that match this python regex -# expression will be passed to add-apt-repository -add_apt_repo_match: '^[\w-]+:\w' - -# 'apt_sources' is a dictionary -# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if -# it doesn't start with a '/'. -# There are certain cases - where no content is written into a source.list file -# where the filename will be ignored - yet it can still be used as index for -# merging. -# The value it maps to is a dictionary with the following optional entries: -# source: a sources.list entry (some variable replacements apply) -# keyid: providing a key to import via shortid or fingerprint -# key: providing a raw PGP key -# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com -# filename: for compatibility with the older format (now the key to this -# dictionary is the filename). If specified this overwrites the -# filename given as key. - -# the new "filename: {specification-dictionary}, filename2: ..." format allows -# better merging between multiple input files than a list like: -# cloud-config1 -# sources: -# s1: {'key': 'key1', 'source': 'source1'} -# cloud-config2 -# sources: -# s2: {'key': 'key2'} -# s1: {filename: 'foo'} -# this would be merged to -#sources: -# s1: -# filename: foo -# key: key1 -# source: source1 -# s2: -# key: key2 -# Be aware that this style of merging is not the default (for backward -# compatibility reasons). You should specify the following merge_how to get -# this more complete and modern merging behaviour: -# merge_how: "list()+dict()+str()" -# This would then also be equivalent to the config merging used in curtin -# (https://launchpad.net/curtin). - -# for more details see below in the various examples - -apt_sources: - byobu-ppa.list: - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" - keyid: F430BBA5 # GPG key ID published on a key server - # adding a source.list line, importing a gpg key for a given key id and - # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list - - # PPA shortcut: - # * Setup correct apt sources.list line - # * Import the signing key from LP - # - # See https://help.launchpad.net/Packaging/PPA for more information - # this requires 'add-apt-repository' - # due to that the filename key is ignored in this case - ignored1: - source: "ppa:smoser/ppa" # Quote the string - - # Custom apt repository: - # * all that is required is 'source' - # * Creates a file in /etc/apt/sources.list.d/ for the sources list entry - # * [optional] Import the apt signing key from the keyserver - # * Defaults: - # + keyserver: keyserver.ubuntu.com - # - # See sources.list man page for more information about the format - my-repo.list: - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted - - # sources can use $MIRROR and $RELEASE and they will be replaced - # with the local mirror for this cloud, and the running release - # the entry below would be possibly turned into: - # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse - my-repo.list: - source: deb $MIRROR $RELEASE multiverse - - # this would have the same end effect as 'ppa:byobu/ppa' - my-repo.list: - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" - keyid: F430BBA5 # GPG key ID published on a key server - filename: byobu-ppa.list - - # this would only import the key without adding a ppa or other source spec - # since this doesn't generate a source.list file the filename key is ignored - ignored2: - keyid: F430BBA5 # GPG key ID published on a key server - - # In general keyid's can also be specified via their long fingerprints - # since this doesn't generate a source.list file the filename key is ignored - ignored3: - keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 - - # Custom apt repository: - # * The apt signing key can also be specified - # by providing a pgp public key block - # * Providing the PGP key here is the most robust method for - # specifying a key, as it removes dependency on a remote key server - my-repo.list: - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: SKS 1.0.10 - - mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 - qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj - 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y - IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H - 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP - t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o - uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey - =Y2oI - -----END PGP PUBLIC KEY BLOCK----- - - # Custom gpg key: - # * As with keyid, a key may also be specified without a related source. - # * all other facts mentioned above still apply - # since this doesn't generate a source.list file the filename key is ignored - ignored4: - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- - -----BEGIN PGP PUBLIC KEY BLOCK----- - Version: SKS 1.0.10 - - mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 - qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj - 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y - IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H - 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP - t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o - uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey - =Y2oI - -----END PGP PUBLIC KEY BLOCK----- - - -## apt config via system_info: -# under the 'system_info', you can further customize cloud-init's interaction -# with apt. -# system_info: -# apt_get_command: [command, argument, argument] -# apt_get_upgrade_subcommand: dist-upgrade -# -# apt_get_command: -# To specify a different 'apt-get' command, set 'apt_get_command'. -# This must be a list, and the subcommand (update, upgrade) is appended to it. -# default is: -# ['apt-get', '--option=Dpkg::Options::=--force-confold', -# '--option=Dpkg::options::=--force-unsafe-io', '--assume-yes', '--quiet'] -# -# apt_get_upgrade_subcommand: -# Specify a different subcommand for 'upgrade. The default is 'dist-upgrade'. -# This is the subcommand that is invoked if package_upgrade is set to true above. -# -# apt_get_wrapper: -# command: eatmydata -# enabled: [True, False, "auto"] -# - -# Install additional packages on first boot -# -# Default: none -# -# if packages are specified, this apt_update will be set to true -# +# For 'apt' specific config, see cloud-config-apt.txt packages: - pwgen - pastebinit diff --git a/tests/configs/sample1.yaml b/tests/configs/sample1.yaml index 6231f293..ae935cc0 100644 --- a/tests/configs/sample1.yaml +++ b/tests/configs/sample1.yaml @@ -3,9 +3,6 @@ #apt_upgrade: true packages: [ bzr, pastebinit, ubuntu-dev-tools, ccache, bzr-builddeb, vim-nox, git-core, lftp ] -#apt_sources: -# - source: ppa:smoser/ppa - #disable_root: False # mounts: diff --git a/tests/unittests/test_distros/test_generic.py b/tests/unittests/test_distros/test_generic.py index 96fa0811..24ad115f 100644 --- a/tests/unittests/test_distros/test_generic.py +++ b/tests/unittests/test_distros/test_generic.py @@ -226,8 +226,5 @@ class TestGenericDistro(helpers.FilesystemMockingTestCase): os.symlink('/', '/run/systemd/system') self.assertFalse(d.uses_systemd()) -# def _get_package_mirror_info(mirror_info, availability_zone=None, -# mirror_filter=util.search_for_mirror): - # vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py new file mode 100644 index 00000000..95fd1da2 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -0,0 +1,109 @@ +from cloudinit.config import cc_apt_configure +from cloudinit import util + +from ..helpers import TestCase + +import os +import re +import shutil +import tempfile + + +def load_tfile_or_url(*args, **kwargs): + return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)) + + +class TestAptProxyConfig(TestCase): + def setUp(self): + super(TestAptProxyConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.pfile = os.path.join(self.tmp, "proxy.cfg") + self.cfile = os.path.join(self.tmp, "config.cfg") + + def _search_apt_config(self, contents, ptype, value): + return re.search( + r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value), + contents, flags=re.IGNORECASE) + + def test_apt_proxy_written(self): + cfg = {'proxy': 'myproxy'} + cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) + + self.assertTrue(os.path.isfile(self.pfile)) + self.assertFalse(os.path.isfile(self.cfile)) + + contents = load_tfile_or_url(self.pfile) + self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) + + def test_apt_http_proxy_written(self): + cfg = {'http_proxy': 'myproxy'} + cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) + + self.assertTrue(os.path.isfile(self.pfile)) + self.assertFalse(os.path.isfile(self.cfile)) + + contents = load_tfile_or_url(self.pfile) + self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) + + def test_apt_all_proxy_written(self): + cfg = {'http_proxy': 'myproxy_http_proxy', + 'https_proxy': 'myproxy_https_proxy', + 'ftp_proxy': 'myproxy_ftp_proxy'} + + values = {'http': cfg['http_proxy'], + 'https': cfg['https_proxy'], + 'ftp': cfg['ftp_proxy'], + } + + cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) + + self.assertTrue(os.path.isfile(self.pfile)) + self.assertFalse(os.path.isfile(self.cfile)) + + contents = load_tfile_or_url(self.pfile) + + for ptype, pval in values.items(): + self.assertTrue(self._search_apt_config(contents, ptype, pval)) + + def test_proxy_deleted(self): + util.write_file(self.cfile, "content doesnt matter") + cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile) + self.assertFalse(os.path.isfile(self.pfile)) + self.assertFalse(os.path.isfile(self.cfile)) + + def test_proxy_replaced(self): + util.write_file(self.cfile, "content doesnt matter") + cc_apt_configure.apply_apt_config({'proxy': "foo"}, + self.pfile, self.cfile) + self.assertTrue(os.path.isfile(self.pfile)) + contents = load_tfile_or_url(self.pfile) + self.assertTrue(self._search_apt_config(contents, "http", "foo")) + + def test_config_written(self): + payload = 'this is my apt config' + cfg = {'conf': payload} + + cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) + + self.assertTrue(os.path.isfile(self.cfile)) + self.assertFalse(os.path.isfile(self.pfile)) + + self.assertEqual(load_tfile_or_url(self.cfile), payload) + + def test_config_replaced(self): + util.write_file(self.pfile, "content doesnt matter") + cc_apt_configure.apply_apt_config({'conf': "foo"}, + self.pfile, self.cfile) + self.assertTrue(os.path.isfile(self.cfile)) + self.assertEqual(load_tfile_or_url(self.cfile), "foo") + + def test_config_deleted(self): + # if no 'conf' is provided, delete any previously written file + util.write_file(self.pfile, "content doesnt matter") + cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile) + self.assertFalse(os.path.isfile(self.pfile)) + self.assertFalse(os.path.isfile(self.cfile)) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py deleted file mode 100644 index d1dca2c4..00000000 --- a/tests/unittests/test_handler/test_handler_apt_configure.py +++ /dev/null @@ -1,109 +0,0 @@ -from cloudinit.config import cc_apt_configure -from cloudinit import util - -from ..helpers import TestCase - -import os -import re -import shutil -import tempfile - - -def load_tfile_or_url(*args, **kwargs): - return(util.decode_binary(util.read_file_or_url(*args, **kwargs).contents)) - - -class TestAptProxyConfig(TestCase): - def setUp(self): - super(TestAptProxyConfig, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - self.pfile = os.path.join(self.tmp, "proxy.cfg") - self.cfile = os.path.join(self.tmp, "config.cfg") - - def _search_apt_config(self, contents, ptype, value): - return re.search( - r"acquire::%s::proxy\s+[\"']%s[\"'];\n" % (ptype, value), - contents, flags=re.IGNORECASE) - - def test_apt_proxy_written(self): - cfg = {'apt_proxy': 'myproxy'} - cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) - - self.assertTrue(os.path.isfile(self.pfile)) - self.assertFalse(os.path.isfile(self.cfile)) - - contents = load_tfile_or_url(self.pfile) - self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) - - def test_apt_http_proxy_written(self): - cfg = {'apt_http_proxy': 'myproxy'} - cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) - - self.assertTrue(os.path.isfile(self.pfile)) - self.assertFalse(os.path.isfile(self.cfile)) - - contents = load_tfile_or_url(self.pfile) - self.assertTrue(self._search_apt_config(contents, "http", "myproxy")) - - def test_apt_all_proxy_written(self): - cfg = {'apt_http_proxy': 'myproxy_http_proxy', - 'apt_https_proxy': 'myproxy_https_proxy', - 'apt_ftp_proxy': 'myproxy_ftp_proxy'} - - values = {'http': cfg['apt_http_proxy'], - 'https': cfg['apt_https_proxy'], - 'ftp': cfg['apt_ftp_proxy'], - } - - cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) - - self.assertTrue(os.path.isfile(self.pfile)) - self.assertFalse(os.path.isfile(self.cfile)) - - contents = load_tfile_or_url(self.pfile) - - for ptype, pval in values.items(): - self.assertTrue(self._search_apt_config(contents, ptype, pval)) - - def test_proxy_deleted(self): - util.write_file(self.cfile, "content doesnt matter") - cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile) - self.assertFalse(os.path.isfile(self.pfile)) - self.assertFalse(os.path.isfile(self.cfile)) - - def test_proxy_replaced(self): - util.write_file(self.cfile, "content doesnt matter") - cc_apt_configure.apply_apt_config({'apt_proxy': "foo"}, - self.pfile, self.cfile) - self.assertTrue(os.path.isfile(self.pfile)) - contents = load_tfile_or_url(self.pfile) - self.assertTrue(self._search_apt_config(contents, "http", "foo")) - - def test_config_written(self): - payload = 'this is my apt config' - cfg = {'apt_config': payload} - - cc_apt_configure.apply_apt_config(cfg, self.pfile, self.cfile) - - self.assertTrue(os.path.isfile(self.cfile)) - self.assertFalse(os.path.isfile(self.pfile)) - - self.assertEqual(load_tfile_or_url(self.cfile), payload) - - def test_config_replaced(self): - util.write_file(self.pfile, "content doesnt matter") - cc_apt_configure.apply_apt_config({'apt_config': "foo"}, - self.pfile, self.cfile) - self.assertTrue(os.path.isfile(self.cfile)) - self.assertEqual(load_tfile_or_url(self.cfile), "foo") - - def test_config_deleted(self): - # if no 'apt_config' is provided, delete any previously written file - util.write_file(self.pfile, "content doesnt matter") - cc_apt_configure.apply_apt_config({}, self.pfile, self.cfile) - self.assertFalse(os.path.isfile(self.pfile)) - self.assertFalse(os.path.isfile(self.cfile)) - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py deleted file mode 100644 index acde0863..00000000 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ /dev/null @@ -1,180 +0,0 @@ -""" test_handler_apt_configure_sources_list -Test templating of sources list -""" -import logging -import os -import shutil -import tempfile - -try: - from unittest import mock -except ImportError: - import mock - -from cloudinit import cloud -from cloudinit import distros -from cloudinit import helpers -from cloudinit import templater -from cloudinit import util - -from cloudinit.config import cc_apt_configure -from cloudinit.sources import DataSourceNone - -from cloudinit.distros.debian import Distro - -from .. import helpers as t_help - -LOG = logging.getLogger(__name__) - -YAML_TEXT_CUSTOM_SL = """ -apt_mirror: http://archive.ubuntu.com/ubuntu/ -apt_custom_sources_list: | - ## template:jinja - ## Note, this file is written by cloud-init on first boot of an instance - ## modifications made here will not survive a re-bundle. - ## if you wish to make changes you can: - ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg - ## or do the same in user-data - ## b.) add sources in /etc/apt/sources.list.d - ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - - # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to - # newer versions of the distribution. - deb {{mirror}} {{codename}} main restricted - deb-src {{mirror}} {{codename}} main restricted - # FIND_SOMETHING_SPECIAL -""" - -EXPECTED_CONVERTED_CONTENT = ( - """## Note, this file is written by cloud-init on first boot of an instance -## modifications made here will not survive a re-bundle. -## if you wish to make changes you can: -## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg -## or do the same in user-data -## b.) add sources in /etc/apt/sources.list.d -## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl - -# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to -# newer versions of the distribution. -deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted -deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted -# FIND_SOMETHING_SPECIAL -""") - - -def load_tfile_or_url(*args, **kwargs): - """load_tfile_or_url - load file and return content after decoding - """ - return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) - - -class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): - """TestAptSourceConfigSourceList - Main Class to test sources list rendering - """ - def setUp(self): - super(TestAptSourceConfigSourceList, self).setUp() - self.subp = util.subp - self.new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.new_root) - - def _get_cloud(self, distro, metadata=None): - self.patchUtils(self.new_root) - paths = helpers.Paths({}) - cls = distros.fetch(distro) - mydist = cls(distro, {}, paths) - myds = DataSourceNone.DataSourceNone({}, mydist, paths) - if metadata: - myds.metadata.update(metadata) - return cloud.Cloud(myds, paths, {}, mydist, None) - - def apt_source_list(self, distro, mirror, mirrorcheck=None): - """apt_source_list - Test rendering of a source.list from template for a given distro - """ - if mirrorcheck is None: - mirrorcheck = mirror - - if isinstance(mirror, list): - cfg = {'apt_mirror_search': mirror} - else: - cfg = {'apt_mirror': mirror} - mycloud = self._get_cloud(distro) - - with mock.patch.object(templater, 'render_to_file') as mocktmpl: - with mock.patch.object(os.path, 'isfile', - return_value=True) as mockisfile: - with mock.patch.object(util, 'rename'): - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) - - mockisfile.assert_any_call( - ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) - mocktmpl.assert_called_once_with( - ('/etc/cloud/templates/sources.list.%s.tmpl' % distro), - '/etc/apt/sources.list', - {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) - - def test_apt_source_list_debian(self): - """Test rendering of a source.list from template for debian""" - self.apt_source_list('debian', 'http://httpredir.debian.org/debian') - - def test_apt_source_list_ubuntu(self): - """Test rendering of a source.list from template for ubuntu""" - self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') - - @staticmethod - def myresolve(name): - """Fake util.is_resolvable for mirrorfail tests""" - if name == "does.not.exist": - print("Faking FAIL for '%s'" % name) - return False - else: - print("Faking SUCCESS for '%s'" % name) - return True - - def test_apt_srcl_debian_mirrorfail(self): - """Test rendering of a source.list from template for debian""" - with mock.patch.object(util, 'is_resolvable', - side_effect=self.myresolve) as mockresolve: - self.apt_source_list('debian', - ['http://does.not.exist', - 'http://httpredir.debian.org/debian'], - 'http://httpredir.debian.org/debian') - mockresolve.assert_any_call("does.not.exist") - mockresolve.assert_any_call("httpredir.debian.org") - - def test_apt_srcl_ubuntu_mirrorfail(self): - """Test rendering of a source.list from template for ubuntu""" - with mock.patch.object(util, 'is_resolvable', - side_effect=self.myresolve) as mockresolve: - self.apt_source_list('ubuntu', - ['http://does.not.exist', - 'http://archive.ubuntu.com/ubuntu/'], - 'http://archive.ubuntu.com/ubuntu/') - mockresolve.assert_any_call("does.not.exist") - mockresolve.assert_any_call("archive.ubuntu.com") - - def test_apt_srcl_custom(self): - """Test rendering from a custom source.list template""" - cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) - mycloud = self._get_cloud('ubuntu') - - # the second mock restores the original subp - with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp): - with mock.patch.object(cc_apt_configure, 'get_release', - return_value='fakerelease'): - with mock.patch.object(Distro, 'get_primary_arch', - return_value='amd64'): - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) - - mockwrite.assert_called_once_with( - '/etc/apt/sources.list', - EXPECTED_CONVERTED_CONTENT, - mode=420) - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py new file mode 100644 index 00000000..f4411869 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v1.py @@ -0,0 +1,200 @@ +""" test_handler_apt_configure_sources_list +Test templating of sources list +""" +import logging +import os +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import templater +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from cloudinit.distros.debian import Distro + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + +YAML_TEXT_CUSTOM_SL = """ +apt_mirror: http://archive.ubuntu.com/ubuntu/ +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + # FIND_SOMETHING_SPECIAL +""" + +EXPECTED_CONVERTED_CONTENT = ( + """## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +# FIND_SOMETHING_SPECIAL +""") + + +def load_tfile_or_url(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """TestAptSourceConfigSourceList + Main Class to test sources list rendering + """ + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + rpatcher = mock.patch("cloudinit.util.lsb_release") + get_rel = rpatcher.start() + get_rel.return_value = {'codename': "fakerelease"} + self.addCleanup(rpatcher.stop) + apatcher = mock.patch("cloudinit.util.get_architecture") + get_arch = apatcher.start() + get_arch.return_value = 'amd64' + self.addCleanup(apatcher.stop) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def apt_source_list(self, distro, mirror, mirrorcheck=None): + """apt_source_list + Test rendering of a source.list from template for a given distro + """ + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} + mycloud = self._get_cloud(distro) + + with mock.patch.object(util, 'write_file') as mockwf: + with mock.patch.object(util, 'load_file', + return_value="faketmpl") as mocklf: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + with mock.patch.object(templater, 'render_string', + return_value="fake") as mockrnd: + with mock.patch.object(util, 'rename'): + cc_apt_configure.handle("test", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocklf.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mockrnd.assert_called_once_with('faketmpl', + {'RELEASE': 'fakerelease', + 'PRIMARY': mirrorcheck, + 'MIRROR': mirrorcheck, + 'SECURITY': mirrorcheck, + 'codename': 'fakerelease', + 'primary': mirrorcheck, + 'mirror': mirrorcheck, + 'security': mirrorcheck}) + mockwf.assert_called_once_with('/etc/apt/sources.list', 'fake', + mode=0o644) + + def test_apt_v1_source_list_debian(self): + """Test rendering of a source.list from template for debian""" + self.apt_source_list('debian', 'http://httpredir.debian.org/debian') + + def test_apt_v1_source_list_ubuntu(self): + """Test rendering of a source.list from template for ubuntu""" + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + @staticmethod + def myresolve(name): + """Fake util.is_resolvable for mirrorfail tests""" + if name == "does.not.exist": + print("Faking FAIL for '%s'" % name) + return False + else: + print("Faking SUCCESS for '%s'" % name) + return True + + def test_apt_v1_srcl_debian_mirrorfail(self): + """Test rendering of a source.list from template for debian""" + with mock.patch.object(util, 'is_resolvable', + side_effect=self.myresolve) as mockresolve: + self.apt_source_list('debian', + ['http://does.not.exist', + 'http://httpredir.debian.org/debian'], + 'http://httpredir.debian.org/debian') + mockresolve.assert_any_call("does.not.exist") + mockresolve.assert_any_call("httpredir.debian.org") + + def test_apt_v1_srcl_ubuntu_mirrorfail(self): + """Test rendering of a source.list from template for ubuntu""" + with mock.patch.object(util, 'is_resolvable', + side_effect=self.myresolve) as mockresolve: + self.apt_source_list('ubuntu', + ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + mockresolve.assert_any_call("does.not.exist") + mockresolve.assert_any_call("archive.ubuntu.com") + + def test_apt_v1_srcl_custom(self): + """Test rendering from a custom source.list template""" + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(Distro, 'get_primary_arch', + return_value='amd64'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockwrite.assert_called_once_with( + '/etc/apt/sources.list', + EXPECTED_CONVERTED_CONTENT, + mode=420) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py new file mode 100644 index 00000000..e53b0450 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list_v3.py @@ -0,0 +1,187 @@ +""" test_apt_custom_sources_list +Test templating of custom sources list +""" +import logging +import os +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from cloudinit.distros.debian import Distro + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + +TARGET = "/" + +# Input and expected output for the custom template +YAML_TEXT_CUSTOM_SL = """ +apt: + primary: + - arches: [default] + uri: http://test.ubuntu.com/ubuntu/ + security: + - arches: [default] + uri: http://testsec.ubuntu.com/ubuntu/ + sources_list: | + + # Note, this file is written by cloud-init at install time. It should not + # end up on the installed system itself. + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb $MIRROR $RELEASE main restricted + deb-src $MIRROR $RELEASE main restricted + deb $PRIMARY $RELEASE universe restricted + deb $SECURITY $RELEASE-security multiverse + # FIND_SOMETHING_SPECIAL +""" + +EXPECTED_CONVERTED_CONTENT = """ +# Note, this file is written by cloud-init at install time. It should not +# end up on the installed system itself. +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://test.ubuntu.com/ubuntu/ fakerel main restricted +deb-src http://test.ubuntu.com/ubuntu/ fakerel main restricted +deb http://test.ubuntu.com/ubuntu/ fakerel universe restricted +deb http://testsec.ubuntu.com/ubuntu/ fakerel-security multiverse +# FIND_SOMETHING_SPECIAL +""" + +# mocked to be independent to the unittest system +MOCKED_APT_SRC_LIST = """ +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""" + +EXPECTED_BASE_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""") + +EXPECTED_MIRROR_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-security main restricted +""") + +EXPECTED_PRIMSEC_CONTENT = (""" +deb http://test.ubuntu.com/ubuntu/ notouched main restricted +deb-src http://test.ubuntu.com/ubuntu/ notouched main restricted +deb http://test.ubuntu.com/ubuntu/ notouched-updates main restricted +deb http://testsec.ubuntu.com/ubuntu/ notouched-security main restricted +""") + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """TestAptSourceConfigSourceList - Class to test sources list rendering""" + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + rpatcher = mock.patch("cloudinit.util.lsb_release") + get_rel = rpatcher.start() + get_rel.return_value = {'codename': "fakerel"} + self.addCleanup(rpatcher.stop) + apatcher = mock.patch("cloudinit.util.get_architecture") + get_arch = apatcher.start() + get_arch.return_value = 'amd64' + self.addCleanup(apatcher.stop) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def _apt_source_list(self, cfg, expected, distro): + "_apt_source_list - Test rendering from template (generic)" + + # entry at top level now, wrap in 'apt' key + cfg = {'apt': cfg} + mycloud = self._get_cloud(distro) + with mock.patch.object(util, 'write_file') as mockwf: + with mock.patch.object(util, 'load_file', + return_value=MOCKED_APT_SRC_LIST) as mocklf: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + with mock.patch.object(util, 'rename'): + cc_apt_configure.handle("test", cfg, mycloud, + LOG, None) + + # check if it would have loaded the distro template + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocklf.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + # check expected content in result + mockwf.assert_called_once_with('/etc/apt/sources.list', expected, + mode=0o644) + + def test_apt_v3_source_list_debian(self): + """test_apt_v3_source_list_debian - without custom sources or parms""" + cfg = {} + self._apt_source_list(cfg, EXPECTED_BASE_CONTENT, 'debian') + + def test_apt_v3_source_list_ubuntu(self): + """test_apt_v3_source_list_ubuntu - without custom sources or parms""" + cfg = {} + self._apt_source_list(cfg, EXPECTED_BASE_CONTENT, 'ubuntu') + + def test_apt_v3_source_list_psm(self): + """test_apt_v3_source_list_psm - Test specifying prim+sec mirrors""" + pm = 'http://test.ubuntu.com/ubuntu/' + sm = 'http://testsec.ubuntu.com/ubuntu/' + cfg = {'preserve_sources_list': False, + 'primary': [{'arches': ["default"], + 'uri': pm}], + 'security': [{'arches': ["default"], + 'uri': sm}]} + + self._apt_source_list(cfg, EXPECTED_PRIMSEC_CONTENT, 'ubuntu') + + def test_apt_v3_srcl_custom(self): + """test_apt_v3_srcl_custom - Test rendering a custom source template""" + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp): + with mock.patch.object(Distro, 'get_primary_arch', + return_value='amd64'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + calls = [call('/etc/apt/sources.list', + EXPECTED_CONVERTED_CONTENT, + mode=0o644)] + mockwrite.assert_has_calls(calls) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py deleted file mode 100644 index 99a4d860..00000000 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ /dev/null @@ -1,516 +0,0 @@ -""" test_handler_apt_source -Testing various config variations of the apt_source config -""" -import os -import re -import shutil -import tempfile - -try: - from unittest import mock -except ImportError: - import mock -from mock import call - -from cloudinit.config import cc_apt_configure -from cloudinit import gpg -from cloudinit import util - -from ..helpers import TestCase - -EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ -NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy -8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 -HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG -CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 -OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ -FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm -S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== -=ACB2 ------END PGP PUBLIC KEY BLOCK-----""" - - -def load_tfile_or_url(*args, **kwargs): - """load_tfile_or_url - load file and return content after decoding - """ - return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) - - -class TestAptSourceConfig(TestCase): - """TestAptSourceConfig - Main Class to test apt_source configs - """ - release = "fantastic" - - def setUp(self): - super(TestAptSourceConfig, self).setUp() - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - self.aptlistfile = os.path.join(self.tmp, "single-deb.list") - self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") - self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") - self.join = os.path.join - # mock fallback filename into writable tmp dir - self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") - - patcher = mock.patch("cloudinit.config.cc_apt_configure.get_release") - get_rel = patcher.start() - get_rel.return_value = self.release - self.addCleanup(patcher.stop) - - @staticmethod - def _get_default_params(): - """get_default_params - Get the most basic default mrror and release info to be used in tests - """ - params = {} - params['RELEASE'] = cc_apt_configure.get_release() - params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" - return params - - def myjoin(self, *args, **kwargs): - """myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" and - args[1] == "cloud_config_sources.list" and - len(args) == 2): - return self.join(self.tmp, args[0].lstrip("/"), args[1]) - else: - return self.join(*args, **kwargs) - - def apt_src_basic(self, filename, cfg): - """apt_src_basic - Test Fix deb source string, has to overwrite mirror conf in params - """ - params = self._get_default_params() - - cc_apt_configure.add_apt_sources(cfg, params) - - self.assertTrue(os.path.isfile(filename)) - - contents = load_tfile_or_url(filename) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "karmic-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_basic(self): - """Test deb source string, overwrite mirror and filename""" - cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted'), - 'filename': self.aptlistfile} - self.apt_src_basic(self.aptlistfile, [cfg]) - - def test_apt_src_basic_dict(self): - """Test deb source string, overwrite mirror and filename (dict)""" - cfg = {self.aptlistfile: {'source': - ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted')}} - self.apt_src_basic(self.aptlistfile, cfg) - - def apt_src_basic_tri(self, cfg): - """apt_src_basic_tri - Test Fix three deb source string, has to overwrite mirror conf in - params. Test with filenames provided in config. - generic part to check three files with different content - """ - self.apt_src_basic(self.aptlistfile, cfg) - - # extra verify on two extra files of this test - contents = load_tfile_or_url(self.aptlistfile2) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "precise-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "lucid-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_basic_tri(self): - """Test Fix three deb source string with filenames""" - cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted'), - 'filename': self.aptlistfile} - cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' precise-backports' - ' main universe multiverse restricted'), - 'filename': self.aptlistfile2} - cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' lucid-backports' - ' main universe multiverse restricted'), - 'filename': self.aptlistfile3} - self.apt_src_basic_tri([cfg1, cfg2, cfg3]) - - def test_apt_src_basic_dict_tri(self): - """Test Fix three deb source string with filenames (dict)""" - cfg = {self.aptlistfile: {'source': - ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted')}, - self.aptlistfile2: {'source': - ('deb http://archive.ubuntu.com/ubuntu' - ' precise-backports' - ' main universe multiverse restricted')}, - self.aptlistfile3: {'source': - ('deb http://archive.ubuntu.com/ubuntu' - ' lucid-backports' - ' main universe multiverse restricted')}} - self.apt_src_basic_tri(cfg) - - def test_apt_src_basic_nofn(self): - """Test Fix three deb source string without filenames (dict)""" - cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted')} - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_src_basic(self.fallbackfn, [cfg]) - - def apt_src_replacement(self, filename, cfg): - """apt_src_replace - Test Autoreplacement of MIRROR and RELEASE in source specs - """ - params = self._get_default_params() - cc_apt_configure.add_apt_sources(cfg, params) - - self.assertTrue(os.path.isfile(filename)) - - contents = load_tfile_or_url(filename) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", params['MIRROR'], params['RELEASE'], - "multiverse"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_replace(self): - """Test Autoreplacement of MIRROR and RELEASE in source specs""" - cfg = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - self.apt_src_replacement(self.aptlistfile, [cfg]) - - def apt_src_replace_tri(self, cfg): - """apt_src_replace_tri - Test three autoreplacements of MIRROR and RELEASE in source specs with - generic part - """ - self.apt_src_replacement(self.aptlistfile, cfg) - - # extra verify on two extra files of this test - params = self._get_default_params() - contents = load_tfile_or_url(self.aptlistfile2) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", params['MIRROR'], params['RELEASE'], - "main"), - contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", params['MIRROR'], params['RELEASE'], - "universe"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_replace_tri(self): - """Test triple Autoreplacement of MIRROR and RELEASE in source specs""" - cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - cfg2 = {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2} - cfg3 = {'source': 'deb $MIRROR $RELEASE universe', - 'filename': self.aptlistfile3} - self.apt_src_replace_tri([cfg1, cfg2, cfg3]) - - def test_apt_src_replace_dict_tri(self): - """Test triple Autoreplacement in source specs (dict)""" - cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, - 'notused': {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2}, - self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} - self.apt_src_replace_tri(cfg) - - def test_apt_src_replace_nofn(self): - """Test Autoreplacement of MIRROR and RELEASE in source specs nofile""" - cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_src_replacement(self.fallbackfn, [cfg]) - - def apt_src_keyid(self, filename, cfg, keynum): - """apt_src_keyid - Test specification of a source + keyid - """ - params = self._get_default_params() - - with mock.patch.object(util, 'subp', - return_value=('fakekey 1234', '')) as mockobj: - cc_apt_configure.add_apt_sources(cfg, params) - - # check if it added the right ammount of keys - calls = [] - for _ in range(keynum): - calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) - mockobj.assert_has_calls(calls, any_order=True) - - self.assertTrue(os.path.isfile(filename)) - - contents = load_tfile_or_url(filename) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", - ('http://ppa.launchpad.net/smoser/' - 'cloud-init-test/ubuntu'), - "xenial", "main"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_keyid(self): - """Test specification of a source + keyid with filename being set""" - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'keyid': "03683F77", - 'filename': self.aptlistfile} - self.apt_src_keyid(self.aptlistfile, [cfg], 1) - - def test_apt_src_keyid_tri(self): - """Test 3x specification of a source + keyid with filename being set""" - cfg1 = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'keyid': "03683F77", - 'filename': self.aptlistfile} - cfg2 = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial universe'), - 'keyid': "03683F77", - 'filename': self.aptlistfile2} - cfg3 = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial multiverse'), - 'keyid': "03683F77", - 'filename': self.aptlistfile3} - - self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) - contents = load_tfile_or_url(self.aptlistfile2) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", - ('http://ppa.launchpad.net/smoser/' - 'cloud-init-test/ubuntu'), - "xenial", "universe"), - contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", - ('http://ppa.launchpad.net/smoser/' - 'cloud-init-test/ubuntu'), - "xenial", "multiverse"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_keyid_nofn(self): - """Test specification of a source + keyid without filename being set""" - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'keyid': "03683F77"} - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_src_keyid(self.fallbackfn, [cfg], 1) - - def apt_src_key(self, filename, cfg): - """apt_src_key - Test specification of a source + key - """ - params = self._get_default_params() - - with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg], params) - - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') - - self.assertTrue(os.path.isfile(filename)) - - contents = load_tfile_or_url(filename) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", - ('http://ppa.launchpad.net/smoser/' - 'cloud-init-test/ubuntu'), - "xenial", "main"), - contents, flags=re.IGNORECASE)) - - def test_apt_src_key(self): - """Test specification of a source + key with filename being set""" - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'key': "fakekey 4321", - 'filename': self.aptlistfile} - self.apt_src_key(self.aptlistfile, cfg) - - def test_apt_src_key_nofn(self): - """Test specification of a source + key without filename being set""" - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'key': "fakekey 4321"} - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_src_key(self.fallbackfn, cfg) - - def test_apt_src_keyonly(self): - """Test specifying key without source""" - params = self._get_default_params() - cfg = {'key': "fakekey 4242", - 'filename': self.aptlistfile} - - with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg], params) - - mockobj.assert_called_once_with(('apt-key', 'add', '-'), - 'fakekey 4242') - - # filename should be ignored on key only - self.assertFalse(os.path.isfile(self.aptlistfile)) - - def test_apt_src_keyidonly(self): - """Test specification of a keyid without source""" - params = self._get_default_params() - cfg = {'keyid': "03683F77", - 'filename': self.aptlistfile} - - with mock.patch.object(util, 'subp', - return_value=('fakekey 1212', '')) as mockobj: - cc_apt_configure.add_apt_sources([cfg], params) - - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') - - # filename should be ignored on key only - self.assertFalse(os.path.isfile(self.aptlistfile)) - - def apt_src_keyid_real(self, cfg, expectedkey): - """apt_src_keyid_real - Test specification of a keyid without source including - up to addition of the key (add_apt_key_raw mocked to keep the - environment as is) - """ - params = self._get_default_params() - - with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: - with mock.patch.object(gpg, 'get_key_by_id', - return_value=expectedkey) as mockgetkey: - cc_apt_configure.add_apt_sources([cfg], params) - - mockgetkey.assert_called_with(cfg['keyid'], - cfg.get('keyserver', - 'keyserver.ubuntu.com')) - mockkey.assert_called_with(expectedkey) - - # filename should be ignored on key only - self.assertFalse(os.path.isfile(self.aptlistfile)) - - def test_apt_src_keyid_real(self): - """test_apt_src_keyid_real - Test keyid including key add""" - keyid = "03683F77" - cfg = {'keyid': keyid, - 'filename': self.aptlistfile} - - self.apt_src_keyid_real(cfg, EXPECTEDKEY) - - def test_apt_src_longkeyid_real(self): - """test_apt_src_longkeyid_real - Test long keyid including key add""" - keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" - cfg = {'keyid': keyid, - 'filename': self.aptlistfile} - - self.apt_src_keyid_real(cfg, EXPECTEDKEY) - - def test_apt_src_longkeyid_ks_real(self): - """test_apt_src_longkeyid_ks_real - Test long keyid from other ks""" - keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" - cfg = {'keyid': keyid, - 'keyserver': 'keys.gnupg.net', - 'filename': self.aptlistfile} - - self.apt_src_keyid_real(cfg, EXPECTEDKEY) - - def test_apt_src_ppa(self): - """Test adding a ppa""" - params = self._get_default_params() - cfg = {'source': 'ppa:smoser/cloud-init-test', - 'filename': self.aptlistfile} - - # default matcher needed for ppa - matcher = re.compile(r'^[\w-]+:\w').search - - with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg], params, - aa_repo_match=matcher) - mockobj.assert_called_once_with(['add-apt-repository', - 'ppa:smoser/cloud-init-test']) - - # adding ppa should ignore filename (uses add-apt-repository) - self.assertFalse(os.path.isfile(self.aptlistfile)) - - def test_apt_src_ppa_tri(self): - """Test adding three ppa's""" - params = self._get_default_params() - cfg1 = {'source': 'ppa:smoser/cloud-init-test', - 'filename': self.aptlistfile} - cfg2 = {'source': 'ppa:smoser/cloud-init-test2', - 'filename': self.aptlistfile2} - cfg3 = {'source': 'ppa:smoser/cloud-init-test3', - 'filename': self.aptlistfile3} - - # default matcher needed for ppa - matcher = re.compile(r'^[\w-]+:\w').search - - with mock.patch.object(util, 'subp') as mockobj: - cc_apt_configure.add_apt_sources([cfg1, cfg2, cfg3], params, - aa_repo_match=matcher) - calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), - call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), - call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] - mockobj.assert_has_calls(calls, any_order=True) - - # adding ppa should ignore all filenames (uses add-apt-repository) - self.assertFalse(os.path.isfile(self.aptlistfile)) - self.assertFalse(os.path.isfile(self.aptlistfile2)) - self.assertFalse(os.path.isfile(self.aptlistfile3)) - - def test_convert_to_new_format(self): - """Test the conversion of old to new format""" - cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - cfg2 = {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2} - cfg3 = {'source': 'deb $MIRROR $RELEASE universe', - 'filename': self.aptlistfile3} - checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, - 'source': 'deb $MIRROR $RELEASE ' - 'multiverse'}, - self.aptlistfile2: {'filename': self.aptlistfile2, - 'source': 'deb $MIRROR $RELEASE main'}, - self.aptlistfile3: {'filename': self.aptlistfile3, - 'source': 'deb $MIRROR $RELEASE ' - 'universe'}} - - newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) - self.assertEqual(newcfg, checkcfg) - - newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) - self.assertEqual(newcfg2, checkcfg) - - with self.assertRaises(ValueError): - cc_apt_configure.convert_to_new_format(5) - - -# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source_v1.py b/tests/unittests/test_handler/test_handler_apt_source_v1.py new file mode 100644 index 00000000..d96779c5 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source_v1.py @@ -0,0 +1,551 @@ +""" test_handler_apt_source_v1 +Testing various config variations of the apt_source config +This calls all things with v1 format to stress the conversion code on top of +the actually tested code. +""" +import os +import re +import shutil +import tempfile + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit.config import cc_apt_configure +from cloudinit import gpg +from cloudinit import util + +from ..helpers import TestCase + +EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" + +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + + +def load_tfile_or_url(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class FakeDistro(object): + """Fake Distro helper object""" + def update_package_sources(self): + """Fake update_package_sources helper method""" + return + + +class FakeCloud(object): + """Fake Cloud helper object""" + def __init__(self): + self.distro = FakeDistro() + + +class TestAptSourceConfig(TestCase): + """TestAptSourceConfig + Main Class to test apt_source configs + """ + release = "fantastic" + + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") + self.join = os.path.join + self.matcher = re.compile(ADD_APT_REPO_MATCH).search + # mock fallback filename into writable tmp dir + self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + self.fakecloud = FakeCloud() + + rpatcher = mock.patch("cloudinit.util.lsb_release") + get_rel = rpatcher.start() + get_rel.return_value = {'codename': self.release} + self.addCleanup(rpatcher.stop) + apatcher = mock.patch("cloudinit.util.get_architecture") + get_arch = apatcher.start() + get_arch.return_value = 'amd64' + self.addCleanup(apatcher.stop) + + def _get_default_params(self): + """get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = self.release + params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" + return params + + def wrapv1conf(self, cfg): + params = self._get_default_params() + # old v1 list format under old keys, but callabe to main handler + # disable source.list rendering and set mirror to avoid other code + return {'apt_preserve_sources_list': True, + 'apt_mirror': params['MIRROR'], + 'apt_sources': cfg} + + def myjoin(self, *args, **kwargs): + """myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + def apt_src_basic(self, filename, cfg): + """apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + cfg = self.wrapv1conf(cfg) + + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic(self): + """Test deb source string, overwrite mirror and filename""" + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + self.apt_src_basic(self.aptlistfile, [cfg]) + + def test_apt_src_basic_dict(self): + """Test deb source string, overwrite mirror and filename (dict)""" + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self.apt_src_basic(self.aptlistfile, cfg) + + def apt_src_basic_tri(self, cfg): + """apt_src_basic_tri + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + generic part to check three files with different content + """ + self.apt_src_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_basic_tri(self): + """Test Fix three deb source string with filenames""" + cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile3} + self.apt_src_basic_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_basic_dict_tri(self): + """Test Fix three deb source string with filenames (dict)""" + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self.apt_src_basic_tri(cfg) + + def test_apt_src_basic_nofn(self): + """Test Fix three deb source string without filenames (dict)""" + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_basic(self.fallbackfn, [cfg]) + + def apt_src_replacement(self, filename, cfg): + """apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + cfg = self.wrapv1conf(cfg) + params = self._get_default_params() + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace(self): + """Test Autoreplacement of MIRROR and RELEASE in source specs""" + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + self.apt_src_replacement(self.aptlistfile, [cfg]) + + def apt_src_replace_tri(self, cfg): + """apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + generic part + """ + self.apt_src_replacement(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_replace_tri(self): + """Test triple Autoreplacement of MIRROR and RELEASE in source specs""" + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_replace_dict_tri(self): + """Test triple Autoreplacement in source specs (dict)""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self.apt_src_replace_tri(cfg) + + def test_apt_src_replace_nofn(self): + """Test Autoreplacement of MIRROR and RELEASE in source specs nofile""" + cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_replacement(self.fallbackfn, [cfg]) + + def apt_src_keyid(self, filename, cfg, keynum): + """apt_src_keyid + Test specification of a source + keyid + """ + cfg = self.wrapv1conf(cfg) + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + + # check if it added the right ammount of keys + calls = [] + for _ in range(keynum): + calls.append(call(['apt-key', 'add', '-'], + data=b'fakekey 1234', + target=None)) + mockobj.assert_has_calls(calls, any_order=True) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid(self): + """Test specification of a source + keyid with filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + self.apt_src_keyid(self.aptlistfile, [cfg], 1) + + def test_apt_src_keyid_tri(self): + """Test 3x specification of a source + keyid with filename being set""" + cfg1 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77", + 'filename': self.aptlistfile3} + + self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_keyid_nofn(self): + """Test specification of a source + keyid without filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_keyid(self.fallbackfn, [cfg], 1) + + def apt_src_key(self, filename, cfg): + """apt_src_key + Test specification of a source + key + """ + cfg = self.wrapv1conf([cfg]) + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + + mockobj.assert_called_with(['apt-key', 'add', '-'], + data=b'fakekey 4321', target=None) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile_or_url(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_src_key(self): + """Test specification of a source + key with filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321", + 'filename': self.aptlistfile} + self.apt_src_key(self.aptlistfile, cfg) + + def test_apt_src_key_nofn(self): + """Test specification of a source + key without filename being set""" + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_src_key(self.fallbackfn, cfg) + + def test_apt_src_keyonly(self): + """Test specifying key without source""" + cfg = {'key': "fakekey 4242", + 'filename': self.aptlistfile} + cfg = self.wrapv1conf([cfg]) + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + + mockobj.assert_called_once_with(['apt-key', 'add', '-'], + data=b'fakekey 4242', target=None) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyidonly(self): + """Test specification of a keyid without source""" + cfg = {'keyid': "03683F77", + 'filename': self.aptlistfile} + cfg = self.wrapv1conf([cfg]) + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + + mockobj.assert_called_with(['apt-key', 'add', '-'], + data=b'fakekey 1212', target=None) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def apt_src_keyid_real(self, cfg, expectedkey): + """apt_src_keyid_real + Test specification of a keyid without source including + up to addition of the key (add_apt_key_raw mocked to keep the + environment as is) + """ + key = cfg['keyid'] + keyserver = cfg.get('keyserver', 'keyserver.ubuntu.com') + cfg = self.wrapv1conf([cfg]) + + with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: + with mock.patch.object(gpg, 'getkeybyid', + return_value=expectedkey) as mockgetkey: + cc_apt_configure.handle("test", cfg, self.fakecloud, + None, None) + + mockgetkey.assert_called_with(key, keyserver) + mockkey.assert_called_with(expectedkey, None) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_keyid_real(self): + """test_apt_src_keyid_real - Test keyid including key add""" + keyid = "03683F77" + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_longkeyid_real(self): + """test_apt_src_longkeyid_real - Test long keyid including key add""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_longkeyid_ks_real(self): + """test_apt_src_longkeyid_ks_real - Test long keyid from other ks""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {'keyid': keyid, + 'keyserver': 'keys.gnupg.net', + 'filename': self.aptlistfile} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_src_ppa(self): + """Test adding a ppa""" + cfg = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + cfg = self.wrapv1conf([cfg]) + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.handle("test", cfg, self.fakecloud, None, None) + mockobj.assert_called_once_with(['add-apt-repository', + 'ppa:smoser/cloud-init-test'], + target=None) + + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_src_ppa_tri(self): + """Test adding three ppa's""" + cfg1 = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + cfg2 = {'source': 'ppa:smoser/cloud-init-test2', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'ppa:smoser/cloud-init-test3', + 'filename': self.aptlistfile3} + cfg = self.wrapv1conf([cfg1, cfg2, cfg3]) + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.handle("test", cfg, self.fakecloud, + None, None) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], + target=None), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2'], + target=None), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'], + target=None)] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) + + def test_convert_to_new_format(self): + """Test the conversion of old to new format""" + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, + 'source': 'deb $MIRROR $RELEASE ' + 'multiverse'}, + self.aptlistfile2: {'filename': self.aptlistfile2, + 'source': 'deb $MIRROR $RELEASE main'}, + self.aptlistfile3: {'filename': self.aptlistfile3, + 'source': 'deb $MIRROR $RELEASE ' + 'universe'}} + + newcfg = cc_apt_configure.convert_v1_to_v2_apt_format([cfg1, cfg2, + cfg3]) + self.assertEqual(newcfg, checkcfg) + + newcfg2 = cc_apt_configure.convert_v1_to_v2_apt_format(newcfg) + self.assertEqual(newcfg2, checkcfg) + + with self.assertRaises(ValueError): + cc_apt_configure.convert_v1_to_v2_apt_format(5) + + +# vi: ts=4 expandtab diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py new file mode 100644 index 00000000..75556b6d --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py @@ -0,0 +1,1103 @@ +"""test_handler_apt_source_v3 +Testing various config variations of the apt_source custom config +This tries to call all in the new v3 format and cares about new features +""" +import glob +import os +import re +import shutil +import socket +import tempfile + +from unittest import TestCase + +try: + from unittest import mock +except ImportError: + import mock +from mock import call + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import gpg +from cloudinit import helpers +from cloudinit import util + +from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" + +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + +TARGET = None + + +def load_tfile(*args, **kwargs): + """load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfig(t_help.FilesystemMockingTestCase): + """TestAptSourceConfig + Main Class to test apt configs + """ + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") + self.join = os.path.join + self.matcher = re.compile(ADD_APT_REPO_MATCH).search + + @staticmethod + def _add_apt_sources(*args, **kwargs): + with mock.patch.object(cc_apt_configure, 'update_packages'): + cc_apt_configure.add_apt_sources(*args, **kwargs) + + @staticmethod + def _get_default_params(): + """get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = util.lsb_release()['codename'] + arch = 'amd64' + params['MIRROR'] = cc_apt_configure.\ + get_default_mirrors(arch)["PRIMARY"] + return params + + def _myjoin(self, *args, **kwargs): + """_myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + def _apt_src_basic(self, filename, cfg): + """_apt_src_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_basic(self): + """test_apt_v3_src_basic - Test fix deb source string""" + cfg = {self.aptlistfile: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self._apt_src_basic(self.aptlistfile, cfg) + + def test_apt_v3_src_basic_tri(self): + """test_apt_v3_src_basic_tri - Test multiple fix deb source strings""" + cfg = {self.aptlistfile: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://test.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self._apt_src_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://test.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def _apt_src_replacement(self, filename, cfg): + """apt_src_replace + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_replace(self): + """test_apt_v3_src_replace - Test replacement of MIRROR & RELEASE""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}} + self._apt_src_replacement(self.aptlistfile, cfg) + + def test_apt_v3_src_replace_fn(self): + """test_apt_v3_src_replace_fn - Test filename overwritten in dict""" + cfg = {'ignored': {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile}} + # second file should overwrite the dict key + self._apt_src_replacement(self.aptlistfile, cfg) + + def _apt_src_replace_tri(self, cfg): + """_apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + generic part + """ + self._apt_src_replacement(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_replace_tri(self): + """test_apt_v3_src_replace_tri - Test multiple replace/overwrites""" + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self._apt_src_replace_tri(cfg) + + def _apt_src_keyid(self, filename, cfg, keynum): + """_apt_src_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + + with mock.patch("cloudinit.util.subp", + return_value=('fakekey 1234', '')) as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + # check if it added the right ammount of keys + calls = [] + for _ in range(keynum): + calls.append(call(['apt-key', 'add', '-'], data=b'fakekey 1234', + target=TARGET)) + mockobj.assert_has_calls(calls, any_order=True) + + self.assertTrue(os.path.isfile(filename)) + + contents = load_tfile(filename) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_keyid(self): + """test_apt_v3_src_keyid - Test source + keyid with filename""" + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"}} + self._apt_src_keyid(self.aptlistfile, cfg, 1) + + def test_apt_v3_src_keyid_tri(self): + """test_apt_v3_src_keyid_tri - Test multiple src+key+filen writes""" + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"}, + 'ignored': {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77"}} + + self._apt_src_keyid(self.aptlistfile, cfg, 3) + contents = load_tfile(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_key(self): + """test_apt_v3_src_key - Test source + key""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4321', + target=TARGET) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + + def test_apt_v3_src_keyonly(self): + """test_apt_v3_src_keyonly - Test key without source""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'key': "fakekey 4242"}} + + with mock.patch.object(util, 'subp') as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 4242', + target=TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_keyidonly(self): + """test_apt_v3_src_keyidonly - Test keyid without source""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'keyid': "03683F77"}} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockobj.assert_any_call(['apt-key', 'add', '-'], data=b'fakekey 1212', + target=TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def apt_src_keyid_real(self, cfg, expectedkey): + """apt_src_keyid_real + Test specification of a keyid without source including + up to addition of the key (add_apt_key_raw mocked to keep the + environment as is) + """ + params = self._get_default_params() + + with mock.patch.object(cc_apt_configure, 'add_apt_key_raw') as mockkey: + with mock.patch.object(gpg, 'getkeybyid', + return_value=expectedkey) as mockgetkey: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + keycfg = cfg[self.aptlistfile] + mockgetkey.assert_called_with(keycfg['keyid'], + keycfg.get('keyserver', + 'keyserver.ubuntu.com')) + mockkey.assert_called_with(expectedkey, TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_keyid_real(self): + """test_apt_v3_src_keyid_real - Test keyid including key add""" + keyid = "03683F77" + cfg = {self.aptlistfile: {'keyid': keyid}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_v3_src_longkeyid_real(self): + """test_apt_v3_src_longkeyid_real Test long keyid including key add""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {self.aptlistfile: {'keyid': keyid}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_v3_src_longkeyid_ks_real(self): + """test_apt_v3_src_longkeyid_ks_real Test long keyid from other ks""" + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + cfg = {self.aptlistfile: {'keyid': keyid, + 'keyserver': 'keys.gnupg.net'}} + + self.apt_src_keyid_real(cfg, EXPECTEDKEY) + + def test_apt_v3_src_keyid_keyserver(self): + """test_apt_v3_src_keyid_keyserver - Test custom keyserver""" + keyid = "03683F77" + params = self._get_default_params() + cfg = {self.aptlistfile: {'keyid': keyid, + 'keyserver': 'test.random.com'}} + + # in some test environments only *.ubuntu.com is reachable + # so mock the call and check if the config got there + with mock.patch.object(gpg, 'getkeybyid', + return_value="fakekey") as mockgetkey: + with mock.patch.object(cc_apt_configure, + 'add_apt_key_raw') as mockadd: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + + mockgetkey.assert_called_with('03683F77', 'test.random.com') + mockadd.assert_called_with('fakekey', TARGET) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_ppa(self): + """test_apt_v3_src_ppa - Test specification of a ppa""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}} + + with mock.patch("cloudinit.util.subp") as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + mockobj.assert_any_call(['add-apt-repository', + 'ppa:smoser/cloud-init-test'], target=TARGET) + + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_v3_src_ppa_tri(self): + """test_apt_v3_src_ppa_tri - Test specification of multiple ppa's""" + params = self._get_default_params() + cfg = {self.aptlistfile: {'source': 'ppa:smoser/cloud-init-test'}, + self.aptlistfile2: {'source': 'ppa:smoser/cloud-init-test2'}, + self.aptlistfile3: {'source': 'ppa:smoser/cloud-init-test3'}} + + with mock.patch("cloudinit.util.subp") as mockobj: + self._add_apt_sources(cfg, TARGET, template_params=params, + aa_repo_match=self.matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test'], + target=TARGET), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2'], + target=TARGET), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'], + target=TARGET)] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) + + @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") + def test_apt_v3_list_rename(self, m_get_architecture): + """test_apt_v3_list_rename - Test find mirror and apt list renaming""" + pre = "/var/lib/apt/lists" + # filenames are archive dependent + + arch = 's390x' + m_get_architecture.return_value = arch + component = "ubuntu-ports" + archive = "ports.ubuntu.com" + + cfg = {'primary': [{'arches': ["default"], + 'uri': + 'http://test.ubuntu.com/%s/' % component}], + 'security': [{'arches': ["default"], + 'uri': + 'http://testsec.ubuntu.com/%s/' % component}]} + post = ("%s_dists_%s-updates_InRelease" % + (component, util.lsb_release()['codename'])) + fromfn = ("%s/%s_%s" % (pre, archive, post)) + tofn = ("%s/test.ubuntu.com_%s" % (pre, post)) + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) + + self.assertEqual(mirrors['MIRROR'], + "http://test.ubuntu.com/%s/" % component) + self.assertEqual(mirrors['PRIMARY'], + "http://test.ubuntu.com/%s/" % component) + self.assertEqual(mirrors['SECURITY'], + "http://testsec.ubuntu.com/%s/" % component) + + with mock.patch.object(os, 'rename') as mockren: + with mock.patch.object(glob, 'glob', + return_value=[fromfn]): + cc_apt_configure.rename_apt_lists(mirrors, TARGET) + + mockren.assert_any_call(fromfn, tofn) + + @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") + def test_apt_v3_list_rename_non_slash(self, m_get_architecture): + target = os.path.join(self.tmp, "rename_non_slash") + apt_lists_d = os.path.join(target, "./" + cc_apt_configure.APT_LISTS) + + m_get_architecture.return_value = 'amd64' + + mirror_path = "some/random/path/" + primary = "http://test.ubuntu.com/" + mirror_path + security = "http://test-security.ubuntu.com/" + mirror_path + mirrors = {'PRIMARY': primary, 'SECURITY': security} + + # these match default archive prefixes + opri_pre = "archive.ubuntu.com_ubuntu_dists_xenial" + osec_pre = "security.ubuntu.com_ubuntu_dists_xenial" + # this one won't match and should not be renamed defaults. + other_pre = "dl.google.com_linux_chrome_deb_dists_stable" + # these are our new expected prefixes + npri_pre = "test.ubuntu.com_some_random_path_dists_xenial" + nsec_pre = "test-security.ubuntu.com_some_random_path_dists_xenial" + + files = [ + # orig prefix, new prefix, suffix + (opri_pre, npri_pre, "_main_binary-amd64_Packages"), + (opri_pre, npri_pre, "_main_binary-amd64_InRelease"), + (opri_pre, npri_pre, "-updates_main_binary-amd64_Packages"), + (opri_pre, npri_pre, "-updates_main_binary-amd64_InRelease"), + (other_pre, other_pre, "_main_binary-amd64_Packages"), + (other_pre, other_pre, "_Release"), + (other_pre, other_pre, "_Release.gpg"), + (osec_pre, nsec_pre, "_InRelease"), + (osec_pre, nsec_pre, "_main_binary-amd64_Packages"), + (osec_pre, nsec_pre, "_universe_binary-amd64_Packages"), + ] + + expected = sorted([npre + suff for opre, npre, suff in files]) + # create files + for (opre, npre, suff) in files: + fpath = os.path.join(apt_lists_d, opre + suff) + util.write_file(fpath, content=fpath) + + cc_apt_configure.rename_apt_lists(mirrors, target) + found = sorted(os.listdir(apt_lists_d)) + self.assertEqual(expected, found) + + @staticmethod + def test_apt_v3_proxy(): + """test_apt_v3_proxy - Test apt_*proxy configuration""" + cfg = {"proxy": "foobar1", + "http_proxy": "foobar2", + "ftp_proxy": "foobar3", + "https_proxy": "foobar4"} + + with mock.patch.object(util, 'write_file') as mockobj: + cc_apt_configure.apply_apt_config(cfg, "proxyfn", "notused") + + mockobj.assert_called_with('proxyfn', + ('Acquire::http::Proxy "foobar1";\n' + 'Acquire::http::Proxy "foobar2";\n' + 'Acquire::ftp::Proxy "foobar3";\n' + 'Acquire::https::Proxy "foobar4";\n')) + + def test_apt_v3_mirror(self): + """test_apt_v3_mirror - Test defining a mirror""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir}], + "security": [{'arches': ["default"], + "uri": smir}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64') + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_mirror_default(self): + """test_apt_v3_mirror_default - Test without defining a mirror""" + arch = 'amd64' + default_mirrors = cc_apt_configure.get_default_mirrors(arch) + pmir = default_mirrors["PRIMARY"] + smir = default_mirrors["SECURITY"] + mycloud = self._get_cloud('ubuntu') + mirrors = cc_apt_configure.find_apt_mirror_info({}, mycloud, arch) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_mirror_arches(self): + """test_apt_v3_mirror_arches - Test arches selection of mirror""" + pmir = "http://my-primary.ubuntu.com/ubuntu/" + smir = "http://my-security.ubuntu.com/ubuntu/" + arch = 'ppc64el' + cfg = {"primary": [{'arches': ["default"], "uri": "notthis-primary"}, + {'arches': [arch], "uri": pmir}], + "security": [{'arches': ["default"], "uri": "nothis-security"}, + {'arches': [arch], "uri": smir}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) + + self.assertEqual(mirrors['PRIMARY'], pmir) + self.assertEqual(mirrors['MIRROR'], pmir) + self.assertEqual(mirrors['SECURITY'], smir) + + def test_apt_v3_mirror_arches_default(self): + """test_apt_v3_mirror_arches - Test falling back to default arch""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir}, + {'arches': ["thisarchdoesntexist"], + "uri": "notthis"}], + "security": [{'arches': ["thisarchdoesntexist"], + "uri": "nothat"}, + {'arches': ["default"], + "uri": smir}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, 'amd64') + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + @mock.patch("cloudinit.config.cc_apt_configure.util.get_architecture") + def test_apt_v3_get_def_mir_non_intel_no_arch(self, m_get_architecture): + arch = 'ppc64el' + m_get_architecture.return_value = arch + expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports', + 'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'} + self.assertEqual(expected, cc_apt_configure.get_default_mirrors()) + + def test_apt_v3_get_default_mirrors_non_intel_with_arch(self): + found = cc_apt_configure.get_default_mirrors('ppc64el') + + expected = {'PRIMARY': 'http://ports.ubuntu.com/ubuntu-ports', + 'SECURITY': 'http://ports.ubuntu.com/ubuntu-ports'} + self.assertEqual(expected, found) + + def test_apt_v3_mirror_arches_sysdefault(self): + """test_apt_v3_mirror_arches - Test arches fallback to sys default""" + arch = 'amd64' + default_mirrors = cc_apt_configure.get_default_mirrors(arch) + pmir = default_mirrors["PRIMARY"] + smir = default_mirrors["SECURITY"] + mycloud = self._get_cloud('ubuntu') + cfg = {"primary": [{'arches': ["thisarchdoesntexist_64"], + "uri": "notthis"}, + {'arches': ["thisarchdoesntexist"], + "uri": "notthiseither"}], + "security": [{'arches': ["thisarchdoesntexist"], + "uri": "nothat"}, + {'arches': ["thisarchdoesntexist_64"], + "uri": "nothateither"}]} + + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + + self.assertEqual(mirrors['MIRROR'], pmir) + self.assertEqual(mirrors['PRIMARY'], pmir) + self.assertEqual(mirrors['SECURITY'], smir) + + def test_apt_v3_mirror_search(self): + """test_apt_v3_mirror_search - Test searching mirrors in a list + mock checks to avoid relying on network connectivity""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "search": ["pfailme", pmir]}], + "security": [{'arches': ["default"], + "search": ["sfailme", smir]}]} + + with mock.patch.object(cc_apt_configure, 'search_for_mirror', + side_effect=[pmir, smir]) as mocksearch: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, + 'amd64') + + calls = [call(["pfailme", pmir]), + call(["sfailme", smir])] + mocksearch.assert_has_calls(calls) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_mirror_search_many2(self): + """test_apt_v3_mirror_search_many3 - Test both mirrors specs at once""" + pmir = "http://us.archive.ubuntu.com/ubuntu/" + smir = "http://security.ubuntu.com/ubuntu/" + cfg = {"primary": [{'arches': ["default"], + "uri": pmir, + "search": ["pfailme", "foo"]}], + "security": [{'arches': ["default"], + "uri": smir, + "search": ["sfailme", "bar"]}]} + + arch = 'amd64' + + # should be called only once per type, despite two mirror configs + mycloud = None + with mock.patch.object(cc_apt_configure, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + calls = [call(cfg, 'primary', arch, mycloud), + call(cfg, 'security', arch, mycloud)] + mockgm.assert_has_calls(calls) + + # should not be called, since primary is specified + with mock.patch.object(cc_apt_configure, + 'search_for_mirror') as mockse: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, None, arch) + mockse.assert_not_called() + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + def test_apt_v3_url_resolvable(self): + """test_apt_v3_url_resolvable - Test resolving urls""" + + with mock.patch.object(util, 'is_resolvable') as mockresolve: + util.is_resolvable_url("http://1.2.3.4/ubuntu") + mockresolve.assert_called_with("1.2.3.4") + + with mock.patch.object(util, 'is_resolvable') as mockresolve: + util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + mockresolve.assert_called_with("us.archive.ubuntu.com") + + # former tests can leave this set (or not if the test is ran directly) + # do a hard reset to ensure a stable result + util._DNS_REDIRECT_IP = None + bad = [(None, None, None, "badname", ["10.3.2.1"])] + good = [(None, None, None, "goodname", ["10.2.3.4"])] + with mock.patch.object(socket, 'getaddrinfo', + side_effect=[bad, bad, bad, good, + good]) as mocksock: + ret = util.is_resolvable_url("http://us.archive.ubuntu.com/ubuntu") + ret2 = util.is_resolvable_url("http://1.2.3.4/ubuntu") + mocksock.assert_any_call('does-not-exist.example.com.', None, + 0, 0, 1, 2) + mocksock.assert_any_call('example.invalid.', None, 0, 0, 1, 2) + mocksock.assert_any_call('us.archive.ubuntu.com', None) + mocksock.assert_any_call('1.2.3.4', None) + + self.assertTrue(ret) + self.assertTrue(ret2) + + # side effect need only bad ret after initial call + with mock.patch.object(socket, 'getaddrinfo', + side_effect=[bad]) as mocksock: + ret3 = util.is_resolvable_url("http://failme.com/ubuntu") + calls = [call('failme.com', None)] + mocksock.assert_has_calls(calls) + self.assertFalse(ret3) + + def test_apt_v3_disable_suites(self): + """test_disable_suites - disable_suites with many configurations""" + release = "xenial" + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + + # disable nothing + disabled = [] + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable release suite + disabled = ["$RELEASE"] + expect = """\ +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable other suite + disabled = ["$RELEASE-updates"] + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu""" + """ xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # multi disable + disabled = ["$RELEASE-updates", "$RELEASE-security"] + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # multi line disable (same suite multiple times in input) + disabled = ["$RELEASE-updates", "$RELEASE-security"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://UBUNTU.com//ubuntu xenial-updates main +deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +# suite disabled by cloud-init: deb http://UBUNTU.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://UBUNTU.COM//ubuntu """ + """xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # comment in input + disabled = ["$RELEASE-updates", "$RELEASE-security"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +#foo +#deb http://UBUNTU.com//ubuntu xenial-updates main +deb http://UBUNTU.COM//ubuntu xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +#foo +#deb http://UBUNTU.com//ubuntu xenial-updates main +# suite disabled by cloud-init: deb http://UBUNTU.COM//ubuntu """ + """xenial-updates main +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable custom suite + disabled = ["foobar"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ foobar main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +# suite disabled by cloud-init: deb http://ubuntu.com/ubuntu/ foobar main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable non existing suite + disabled = ["foobar"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ notfoobar main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +deb http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb http://ubuntu.com/ubuntu/ notfoobar main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable suite with option + disabled = ["$RELEASE-updates"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [a=b] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb [a=b] http://ubu.com//ubu """ + """xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable suite with more options and auto $RELEASE expansion + disabled = ["updates"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [a=b c=d] http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = """deb http://ubuntu.com//ubuntu xenial main +# suite disabled by cloud-init: deb [a=b c=d] \ +http://ubu.com//ubu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + # single disable suite while options at others + disabled = ["$RELEASE-security"] + orig = """deb http://ubuntu.com//ubuntu xenial main +deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main +deb http://ubuntu.com//ubuntu xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""" + expect = ("""deb http://ubuntu.com//ubuntu xenial main +deb [arch=foo] http://ubuntu.com//ubuntu xenial-updates main +# suite disabled by cloud-init: deb http://ubuntu.com//ubuntu """ + """xenial-security main +deb-src http://ubuntu.com//ubuntu universe multiverse +deb http://ubuntu.com/ubuntu/ xenial-proposed main""") + result = cc_apt_configure.disable_suites(disabled, orig, release) + self.assertEqual(expect, result) + + def test_disable_suites_blank_lines(self): + """test_disable_suites_blank_lines - ensure blank lines allowed""" + lines = ["deb %(repo)s %(rel)s main universe", + "", + "deb %(repo)s %(rel)s-updates main universe", + " # random comment", + "#comment here", + ""] + rel = "trusty" + repo = 'http://example.com/mirrors/ubuntu' + orig = "\n".join(lines) % {'repo': repo, 'rel': rel} + self.assertEqual( + orig, cc_apt_configure.disable_suites(["proposed"], orig, rel)) + + def test_apt_v3_mirror_search_dns(self): + """test_apt_v3_mirror_search_dns - Test searching dns patterns""" + pmir = "phit" + smir = "shit" + arch = 'amd64' + mycloud = self._get_cloud('ubuntu') + cfg = {"primary": [{'arches': ["default"], + "search_dns": True}], + "security": [{'arches': ["default"], + "search_dns": True}]} + + with mock.patch.object(cc_apt_configure, 'get_mirror', + return_value="http://mocked/foo") as mockgm: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + calls = [call(cfg, 'primary', arch, mycloud), + call(cfg, 'security', arch, mycloud)] + mockgm.assert_has_calls(calls) + + with mock.patch.object(cc_apt_configure, 'search_for_mirror_dns', + return_value="http://mocked/foo") as mocksdns: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + calls = [call(True, 'primary', cfg, mycloud), + call(True, 'security', cfg, mycloud)] + mocksdns.assert_has_calls(calls) + + # first return is for the non-dns call before + with mock.patch.object(cc_apt_configure, 'search_for_mirror', + side_effect=[None, pmir, None, smir]) as mockse: + mirrors = cc_apt_configure.find_apt_mirror_info(cfg, mycloud, arch) + + calls = [call(None), + call(['http://ubuntu-mirror.localdomain/ubuntu', + 'http://ubuntu-mirror/ubuntu']), + call(None), + call(['http://ubuntu-security-mirror.localdomain/ubuntu', + 'http://ubuntu-security-mirror/ubuntu'])] + mockse.assert_has_calls(calls) + + self.assertEqual(mirrors['MIRROR'], + pmir) + self.assertEqual(mirrors['PRIMARY'], + pmir) + self.assertEqual(mirrors['SECURITY'], + smir) + + +class TestDebconfSelections(TestCase): + + @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections") + def test_no_set_sel_if_none_to_set(self, m_set_sel): + cc_apt_configure.apply_debconf_selections({'foo': 'bar'}) + m_set_sel.assert_not_called() + + @mock.patch("cloudinit.config.cc_apt_configure." + "debconf_set_selections") + @mock.patch("cloudinit.config.cc_apt_configure." + "util.get_installed_packages") + def test_set_sel_call_has_expected_input(self, m_get_inst, m_set_sel): + data = { + 'set1': 'pkga pkga/q1 mybool false', + 'set2': ('pkgb\tpkgb/b1\tstr\tthis is a string\n' + 'pkgc\tpkgc/ip\tstring\t10.0.0.1')} + lines = '\n'.join(data.values()).split('\n') + + m_get_inst.return_value = ["adduser", "apparmor"] + m_set_sel.return_value = None + + cc_apt_configure.apply_debconf_selections({'debconf_selections': data}) + self.assertTrue(m_get_inst.called) + self.assertEqual(m_set_sel.call_count, 1) + + # assumes called with *args value. + selections = m_set_sel.call_args_list[0][0][0].decode() + + missing = [l for l in lines if l not in selections.splitlines()] + self.assertEqual([], missing) + + @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure") + @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections") + @mock.patch("cloudinit.config.cc_apt_configure." + "util.get_installed_packages") + def test_reconfigure_if_intersection(self, m_get_inst, m_set_sel, + m_dpkg_r): + data = { + 'set1': 'pkga pkga/q1 mybool false', + 'set2': ('pkgb\tpkgb/b1\tstr\tthis is a string\n' + 'pkgc\tpkgc/ip\tstring\t10.0.0.1'), + 'cloud-init': ('cloud-init cloud-init/datasources' + 'multiselect MAAS')} + + m_set_sel.return_value = None + m_get_inst.return_value = ["adduser", "apparmor", "pkgb", + "cloud-init", 'zdog'] + + cc_apt_configure.apply_debconf_selections({'debconf_selections': data}) + + # reconfigure should be called with the intersection + # of (packages in config, packages installed) + self.assertEqual(m_dpkg_r.call_count, 1) + # assumes called with *args (dpkg_reconfigure([a,b,c], target=)) + packages = m_dpkg_r.call_args_list[0][0][0] + self.assertEqual(set(['cloud-init', 'pkgb']), set(packages)) + + @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure") + @mock.patch("cloudinit.config.cc_apt_configure.debconf_set_selections") + @mock.patch("cloudinit.config.cc_apt_configure." + "util.get_installed_packages") + def test_reconfigure_if_no_intersection(self, m_get_inst, m_set_sel, + m_dpkg_r): + data = {'set1': 'pkga pkga/q1 mybool false'} + + m_get_inst.return_value = ["adduser", "apparmor", "pkgb", + "cloud-init", 'zdog'] + m_set_sel.return_value = None + + cc_apt_configure.apply_debconf_selections({'debconf_selections': data}) + + self.assertTrue(m_get_inst.called) + self.assertEqual(m_dpkg_r.call_count, 0) + + @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + def test_dpkg_reconfigure_does_reconfigure(self, m_subp): + target = "/foo-target" + + # due to the way the cleaners are called (via dictionary reference) + # mocking clean_cloud_init directly does not work. So we mock + # the CONFIG_CLEANERS dictionary and assert our cleaner is called. + ci_cleaner = mock.MagicMock() + with mock.patch.dict(("cloudinit.config.cc_apt_configure." + "CONFIG_CLEANERS"), + values={'cloud-init': ci_cleaner}, clear=True): + cc_apt_configure.dpkg_reconfigure(['pkga', 'cloud-init'], + target=target) + # cloud-init is actually the only package we have a cleaner for + # so for now, its the only one that should reconfigured + self.assertTrue(m_subp.called) + ci_cleaner.assert_called_with(target) + self.assertEqual(m_subp.call_count, 1) + found = m_subp.call_args_list[0][0][0] + expected = ['dpkg-reconfigure', '--frontend=noninteractive', + 'cloud-init'] + self.assertEqual(expected, found) + + @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + def test_dpkg_reconfigure_not_done_on_no_data(self, m_subp): + cc_apt_configure.dpkg_reconfigure([]) + m_subp.assert_not_called() + + @mock.patch("cloudinit.config.cc_apt_configure.util.subp") + def test_dpkg_reconfigure_not_done_if_no_cleaners(self, m_subp): + cc_apt_configure.dpkg_reconfigure(['pkgfoo', 'pkgbar']) + m_subp.assert_not_called() + +# +# vi: ts=4 expandtab diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index 73369cd3..d2031f59 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -508,4 +508,73 @@ class TestReadSeeded(helpers.TestCase): self.assertEqual(found_md, {'key1': 'val1'}) self.assertEqual(found_ud, ud) + +class TestSubp(helpers.TestCase): + + stdin2err = ['bash', '-c', 'cat >&2'] + stdin2out = ['cat'] + utf8_invalid = b'ab\xaadef' + utf8_valid = b'start \xc3\xa9 end' + utf8_valid_2 = b'd\xc3\xa9j\xc8\xa7' + + def printf_cmd(self, *args): + # bash's printf supports \xaa. So does /usr/bin/printf + # but by using bash, we remove dependency on another program. + return(['bash', '-c', 'printf "$@"', 'printf'] + list(args)) + + def test_subp_handles_utf8(self): + # The given bytes contain utf-8 accented characters as seen in e.g. + # the "deja dup" package in Ubuntu. + cmd = self.printf_cmd(self.utf8_valid_2) + (out, _err) = util.subp(cmd, capture=True) + self.assertEqual(out, self.utf8_valid_2.decode('utf-8')) + + def test_subp_respects_decode_false(self): + (out, err) = util.subp(self.stdin2out, capture=True, decode=False, + data=self.utf8_valid) + self.assertTrue(isinstance(out, bytes)) + self.assertTrue(isinstance(err, bytes)) + self.assertEqual(out, self.utf8_valid) + + def test_subp_decode_ignore(self): + # this executes a string that writes invalid utf-8 to stdout + (out, _err) = util.subp(self.printf_cmd('abc\\xaadef'), + capture=True, decode='ignore') + self.assertEqual(out, 'abcdef') + + def test_subp_decode_strict_valid_utf8(self): + (out, _err) = util.subp(self.stdin2out, capture=True, + decode='strict', data=self.utf8_valid) + self.assertEqual(out, self.utf8_valid.decode('utf-8')) + + def test_subp_decode_invalid_utf8_replaces(self): + (out, _err) = util.subp(self.stdin2out, capture=True, + data=self.utf8_invalid) + expected = self.utf8_invalid.decode('utf-8', errors='replace') + self.assertEqual(out, expected) + + def test_subp_decode_strict_raises(self): + args = [] + kwargs = {'args': self.stdin2out, 'capture': True, + 'decode': 'strict', 'data': self.utf8_invalid} + self.assertRaises(UnicodeDecodeError, util.subp, *args, **kwargs) + + def test_subp_capture_stderr(self): + data = b'hello world' + (out, err) = util.subp(self.stdin2err, capture=True, + decode=False, data=data) + self.assertEqual(err, data) + self.assertEqual(out, b'') + + def test_returns_none_if_no_capture(self): + (out, err) = util.subp(self.stdin2out, data=b'', capture=False) + self.assertEqual(err, None) + self.assertEqual(out, None) + + def test_bunch_of_slashes_in_path(self): + self.assertEqual("/target/my/path/", + util.target_path("/target/", "//my/path/")) + self.assertEqual("/target/my/path/", + util.target_path("/target/", "///my/path/")) + # vi: ts=4 expandtab -- cgit v1.2.3 From 41271bd8609550d02868b7681b49b8791678beaa Mon Sep 17 00:00:00 2001 From: Andrew Jorgensen Date: Thu, 18 Aug 2016 14:28:25 -0400 Subject: distros: fix get_primary_arch method use of os.uname os.uname is a method, not a property. --- cloudinit/distros/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 40af8802..b1192e84 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -112,7 +112,7 @@ class Distro(object): raise NotImplementedError() def get_primary_arch(self): - arch = os.uname[4] + arch = os.uname()[4] if arch in ("i386", "i486", "i586", "i686"): return "i386" return arch -- cgit v1.2.3