From 0b3e3f898f70cc98c6e694c7b7a11e654fff9967 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sun, 1 May 2016 13:03:23 -0400 Subject: initial commit of rework --- cloudinit/sources/DataSourceSmartOS.py | 450 +++++++++++++++++++++------------ 1 file changed, 286 insertions(+), 164 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6cbd8dfa..46cf117a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -32,8 +32,10 @@ # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html # Comments with "@datadictionary" are snippets of the definition +import base64 import binascii import contextlib +import json import os import random import re @@ -66,12 +68,26 @@ SMARTOS_ATTRIB_MAP = { DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] +NO_BASE64_DECODE = [ + 'iptables_disable', + 'motd_sys_info', + 'root_authorized_keys', + 'sdc:datacenter_name', + 'sdc:uuid' + 'user-data', + 'user-script', +] + +METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' +SERIAL_DEVICE = '/dev/ttyS1' +SERIAL_TIMEOUT = 60 + # BUILT-IN DATASOURCE CONFIGURATION # The following is the built-in configuration. If the values # are not set via the system configuration, then these default # will be used: # serial_device: which serial device to use for the meta-data -# seed_timeout: how long to wait on the device +# serial_timeout: how long to wait on the device # no_base64_decode: values which are not base64 encoded and # are fetched directly from SmartOS, not meta-data values # base64_keys: meta-data keys that are delivered in base64 @@ -81,16 +97,10 @@ DS_CFG_PATH = ['datasource', DS_NAME] # fs_setup: describes how to format the ephemeral drive # BUILTIN_DS_CONFIG = { - 'serial_device': '/dev/ttyS1', - 'metadata_sockfile': '/native/.zonecontrol/metadata.sock', - 'seed_timeout': 60, - 'no_base64_decode': ['root_authorized_keys', - 'motd_sys_info', - 'iptables_disable', - 'user-data', - 'user-script', - 'sdc:datacenter_name', - 'sdc:uuid'], + 'serial_device': SERIAL_DEVICE, + 'serial_timeout': SERIAL_TIMEOUT, + 'metadata_sockfile': METADATA_SOCKFILE, + 'no_base64_decode': NO_BASE64_DECODE, 'base64_keys': [], 'base64_all': False, 'disk_aliases': {'ephemeral0': '/dev/vdb'}, @@ -154,9 +164,12 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): + _unset = "_unset" + smartos_environ = _unset + md_client = _unset + def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.is_smartdc = None self.ds_cfg = util.mergemanydict([ self.ds_cfg, util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), @@ -164,49 +177,24 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = {} - # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but - # report 'BrandZ virtual linux' as the kernel version - if os.uname()[3].lower() == 'brandz virtual linux': - LOG.debug("Host is SmartOS, guest in Zone") - self.is_smartdc = True - self.smartos_type = 'lx-brand' - self.cfg = {} - self.seed = self.ds_cfg.get("metadata_sockfile") - else: - self.is_smartdc = True - self.smartos_type = 'kvm' - self.seed = self.ds_cfg.get("serial_device") - self.cfg = BUILTIN_CLOUD_CONFIG - self.seed_timeout = self.ds_cfg.get("serial_timeout") - self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode') - self.b64_keys = self.ds_cfg.get('base64_keys') - self.b64_all = self.ds_cfg.get('base64_all') self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self._init() + def __str__(self): root = sources.DataSource.__str__(self) - return "%s [seed=%s]" % (root, self.seed) - - def _get_seed_file_object(self): - if not self.seed: - raise AttributeError("seed device is not set") - - if self.smartos_type == 'lx-brand': - if not stat.S_ISSOCK(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a socket", self.seed) - return None - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.seed) - return sock.makefile('rwb') - else: - if not stat.S_ISCHR(os.stat(self.seed).st_mode): - LOG.debug("Seed %s is not a character device") - return None - ser = serial.Serial(self.seed, timeout=self.seed_timeout) - if not ser.isOpen(): - raise SystemError("Unable to open %s" % self.seed) - return ser - return None + return "%s [client=%s]" % (root, self.md_client) + + def _init(self): + if self.smartos_environ == self._unset: + self.smartos_env = get_smartos_environ() + + if self.md_client == self._unset: + self.md_client = jmc_client_factory( + metadata_sockfile=self.ds_cfg['metadata_sockfile'], + serial_device=self.ds_cfg['serial_device'], + serial_timeout=self.ds_cfg['serial_timeout']) + def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -225,50 +213,23 @@ class DataSourceSmartOS(sources.DataSource): '/'.join([svc_path, 'provision_success'])) def get_data(self): + self._init() + md = {} ud = "" - if not device_exists(self.seed): - LOG.debug("No metadata device '%s' found for SmartOS datasource", - self.seed) - return False - - uname_arch = os.uname()[4] - if uname_arch.startswith("arm") or uname_arch == "aarch64": - # Disabling because dmidcode in dmi_data() crashes kvm process - LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)") + if not self.smartos_env: + LOG.debug("Not running on smartos") return False - - # SDC KVM instances will provide dmi data, LX-brand does not - if self.smartos_type == 'kvm': - dmi_info = dmi_data() - if dmi_info is None: - LOG.debug("No dmidata utility found") - return False - - system_type = dmi_info - if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS. system_type=%s", - system_type) - return False - LOG.debug("Host is SmartOS, guest in KVM") - - seed_obj = self._get_seed_file_object() - if seed_obj is None: - LOG.debug('Seed file object not found.') + + if not self.md_client.exists(): + LOG.debug("No metadata device '%r' found for SmartOS datasource", + self.md_client) return False - with contextlib.closing(seed_obj) as seed: - b64_keys = self.query('base64_keys', seed, strip=True, b64=False) - if b64_keys is not None: - self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] - b64_all = self.query('base64_all', seed, strip=True, b64=False) - if b64_all is not None: - self.b64_all = util.is_true(b64_all) - - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): - smartos_noun, strip = attribute - md[ci_noun] = self.query(smartos_noun, seed, strip=strip) + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): + smartos_noun, strip = attribute + md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then @@ -331,65 +292,6 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def query(self, noun, seed_file, strip=False, default=None, b64=None): - if b64 is None: - if noun in self.smartos_no_base64: - b64 = False - elif self.b64_all or noun in self.b64_keys: - b64 = True - - return self._query_data(noun, seed_file, strip=strip, - default=default, b64=b64) - - def _query_data(self, noun, seed_file, strip=False, - default=None, b64=None): - """Makes a request via "GET " - - In the response, the first line is the status, while subsequent - lines are is the value. A blank line with a "." is used to - indicate end of response. - - If the response is expected to be base64 encoded, then set - b64encoded to true. Unfortantely, there is no way to know if - something is 100% encoded, so this method relies on being told - if the data is base64 or not. - """ - - if not noun: - return False - - response = JoyentMetadataClient(seed_file).get_metadata(noun) - - if response is None: - return default - - if b64 is None: - b64 = self._query_data('b64-%s' % noun, seed_file, b64=False, - default=False, strip=True) - b64 = util.is_true(b64) - - resp = None - if b64 or strip: - resp = "".join(response).rstrip() - else: - resp = "".join(response) - - if b64: - try: - return util.b64d(resp) - # Bogus input produces different errors in Python 2 and 3; - # catch both. - except (TypeError, binascii.Error): - LOG.warn("Failed base64 decoding key '%s'", noun) - return resp - - return resp - - -def device_exists(device): - """Symplistic method to determine if the device exists or not""" - return os.path.exists(device) - class JoyentMetadataFetchException(Exception): pass @@ -407,8 +309,11 @@ class JoyentMetadataClient(object): r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' r'( (?P.+))?)') - def __init__(self, metasource): - self.metasource = metasource + def __init__(self, smartos_type=None): + if smartos_type is None: + smartos_type = get_smartos_environ() + self.smartos_type = smartos_type + self.fp = None def _checksum(self, body): return '{0:08x}'.format( @@ -436,37 +341,227 @@ class JoyentMetadataClient(object): LOG.debug('Value "%s" found.', value) return value - def get_metadata(self, metadata_key): - LOG.debug('Fetching metadata key "%s"...', metadata_key) + def request(self, rtype, param=None): request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) - message_body = '{0} GET {1}'.format(request_id, - util.b64e(metadata_key)) + message_body = ' '.join((request_id, rtype,)) + if param: + message_body += ' ' + base64.b64encode(param.encode()).decode() msg = 'V2 {0} {1} {2}\n'.format( len(message_body), self._checksum(message_body), message_body) LOG.debug('Writing "%s" to metadata transport.', msg) - self.metasource.write(msg.encode('ascii')) - self.metasource.flush() + + need_close = False + if not self.fp: + self.open_transport() + need_close = True + + self.fp.write(msg.encode('ascii')) + self.fp.flush() response = bytearray() - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) while response[-1:] != b'\n': - response.extend(self.metasource.read(1)) + response.extend(self.fp.read(1)) + + if need_close: + self.close_transport() + response = response.rstrip().decode('ascii') LOG.debug('Read "%s" from metadata transport.', response) if 'SUCCESS' not in response: return None - return self._get_value_from_frame(request_id, response) + value = self._get_value_from_frame(request_id, response) + if value is None: + return default + return value -def dmi_data(): - sys_type = util.read_dmi_data("system-product-name") + def get(self, key, default=None, strip=False): + result = self.request(rtype='GET', param=key) + if result is None: + return default + if result and strip: + result = result.strip() + return result + + def get_json(self, key, default=None): + result = self.get(key) + if result is None: + return default + return json.loads(result) + + def list(self): + result = self.request(rtype='KEYS') + if result: + result = result.split('\n') + return result + + def put(self, key, val): + param = b' '.join([base64.b64encode(i.encode()) + for i in (key, val)]).decode() + return self.request(rtype='PUT', param=param) + + def delete(self, key): + return self.request(rtype='DELETE', param=key) + + def close_transport(self): + if self.fp: + self.fp.close() + self.fp = None + + def __enter__(self): + if self.fp: + return self + self.open_transport() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close_transport() + return - if not sys_type: - return None - return sys_type +class JoyentMetadataSocketClient(JoyentMetadataClient): + def __init__(self, socketpath): + self.socketpath = metadata_socketfile + + def open_transport(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + self.fp = sock.makefile('rwb') + + def exists(self): + return os.path.exists(self.socketpath) + + def __repr__(self): + return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) + + +class JoyentMetadataSerialClient(JoyentMetadataClient): + def __init__(self, device, timeout=10, smartos_type=None): + super(JoyentMetadataSerialClient, self).__init__(smartos_type) + self.device = device + self.timeout = timeout + + def exists(self): + return os.path.exists(self.device) + + def open_transport(self): + ser = serial.Serial(self.device, timeout=self.timeout) + if not ser.isOpen(): + raise SystemError("Unable to open %s" % self.device) + self.fp = ser + + def __repr__(self): + return "%s(device=%s, timeout=%s)" % ( + self.__class__.__name__, self.device, self.timeout) + + +class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): + """V1 of the protocol was not safe for all values. + Thus, we allowed the user to pass values in as base64 encoded. + Users may still reasonably expect to be able to send base64 data + and have it transparently decoded. So even though the V2 format is + now used, and is safe (using base64 itself), we keep legacy support. + + The way for a user to do this was: + a.) specify 'base64_keys' key whose value is a comma delimited + list of keys that were base64 encoded. + b.) base64_all: string interpreted as a boolean that indicates + if all keys are base64 encoded. + c.) set a key named b64- with a boolean indicating that + is base64 encoded.""" + + def __init__(self, device, timeout=10, smartos_type=None): + s = super(JoyentMetadataLegacySerialClient, self) + s.__init__(device, timeout, smartos_type) + self.base64_keys = None + self.base64_all = None + + def _init_base64_keys(self, reset=False): + if reset: + self.base64_keys = None + self.base64_all = None + + keys = None + if self.base64_all is None: + keys = self.list() + if 'base64_all' in keys: + self.base64_all = util.is_true(self._get("base64_all")) + else: + self.base64_all = False + + if self.base64_all: + # short circuit if base64_all is true + return + + if self.base64_keys is None: + if keys is None: + keys = self.list() + b64_keys = set() + if 'base64_keys' in keys: + b64_keys = set(self._get("base64_keys").split(",")) + + # now add any b64- that has a true value + for key in [k[3:] for k in keys if k.startswith("b64-")]: + if util.is_true(self._get(key)): + b64_keys.append(key) + else: + if key in b64_keys: + b64_keys.remove(key) + + self.base64_keys = b64_keys + + def _get(self, key, default=None, strip=False): + return (super(JoyentMetadataLegacySerialClient, self). + get(key, default=default, strip=strip)) + + def is_b64_encoded(self, key, reset=False): + if key in NO_BASE64_DECODE: + return False + + self._init_base64_keys(reset=reset) + if self.base64_all: + return True + + return key in self.base64_keys + + def get(self, key, default=None, strip=False): + mdefault = object() + val = self._get(key, strip=False, default=mdefault) + if val is mdefault: + return default + + if self.is_b64_encoded(key): + try: + val = base64.b64decode(val.encode()).decode() + # Bogus input produces different errors in Python 2 and 3 + except (TypeError, binascii.Error): + LOG.warn("Failed base64 decoding key '%s': %s", key, val) + + if strip: + val = val.strip() + + return val + + +def jmc_client_factory( + smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, + serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, + uname_version=None): + + if smartos_type is None: + smartos_type = get_smartos_environ(uname_version) + + if smartos_type == 'kvm': + return JoyentMetadataLegacySerialClient( + device=serial_device, timeout=serial_timeout, + smartos_type=smartos_type) + elif smartos_type == 'lx-brand': + return JoyentMetadataSerialClient(socketpath=metadata_socketfile) + + raise ValueError("Unknown value for smartos_type: %s" % smartos_type) def write_boot_content(content, content_f, link=None, shebang=False, @@ -525,6 +620,29 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.logexc(LOG, "failed establishing content link", e) +def get_smartos_environ(uname_version=None, product_name=None, + uname_arch=None): + uname = os.uname() + if uname_arch is None: + uname_arch = uname[4] + + if uname_arch.startswith("arm") or uname_arch == "aarch64": + return None + + # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but + # report 'BrandZ virtual linux' as the kernel version + if uname_version is None: + uname_version = uname[3] + if uname_version.lower() == 'brandz virtual linux': + return 'lx-brand' + + system_type = util.read_dmi_data("system-product-name") + if system_type and 'smartdc' in system_type.lower(): + return 'kvm' + + return None + + # Used to match classes to dependencies datasources = [ (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), @@ -534,3 +652,7 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +if __name__ == "__main__": + jmc = JoyentMetadataClient(seed_file).get_metadata(noun) -- cgit v1.2.3 From eeabe2ed7480dc653c9d65d218738a8ed5a21579 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 14:42:19 +0200 Subject: basic apt_source test --- .../test_handler/test_handler_apt_source.py | 87 ++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/unittests/test_handler/test_handler_apt_source.py diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py new file mode 100644 index 00000000..b45fc1d1 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -0,0 +1,87 @@ +""" test_handler_apt_source +Testing various config variations of the apt_source config +""" +import os +import shutil +import tempfile +import re + +from cloudinit import distros +from cloudinit import util +from cloudinit.config import cc_apt_configure + +from ..helpers import TestCase + +UNKNOWN_ARCH_INFO = { + 'arches': ['default'], + 'failsafe': {'primary': 'http://fs-primary-default', + 'security': 'http://fs-security-default'} +} + +PACKAGE_MIRRORS = [ + {'arches': ['i386', 'amd64'], + 'failsafe': {'primary': 'http://fs-primary-intel', + 'security': 'http://fs-security-intel'}, + 'search': { + 'primary': ['http://%(ec2_region)s.ec2/', + 'http://%(availability_zone)s.clouds/'], + 'security': ['http://security-mirror1-intel', + 'http://security-mirror2-intel']}}, + {'arches': ['armhf', 'armel'], + 'failsafe': {'primary': 'http://fs-primary-arm', + 'security': 'http://fs-security-arm'}}, + UNKNOWN_ARCH_INFO +] + +GAPMI = distros._get_arch_package_mirror_info + +def load_tfile_or_url(*args, **kwargs): + """ load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + +class TestAptSourceConfig(TestCase): + """ TestAptSourceConfig + Main Class to test apt_source configs + """ + def setUp(self): + super(TestAptSourceConfig, self).setUp() + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + + + @staticmethod + def _get_default_params(): + """ get_default_params + Get the most basic default mrror and release info to be used in tests + """ + params = {} + params['RELEASE'] = cc_apt_configure.get_release() + params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" + return params + + @staticmethod + def _search_apt_source(contents, params, pre, post): + return re.search(r"%s %s %s %s\n" % + (pre, params['MIRROR'], params['RELEASE'], post), + contents, flags=re.IGNORECASE) + + def test_apt_source_release(self): + """ test_apt_source_release + Test Autoreplacement of MIRROR and RELEASE in source specs + """ + params = self._get_default_params() + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(self._search_apt_source(contents, params, + "deb", "multiverse")) + +# vi: ts=4 expandtab -- cgit v1.2.3 From b0494addabfe1d07947427ade99a00d2c7588f12 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 14:59:57 +0200 Subject: split into basic and replacement test --- .../test_handler/test_handler_apt_source.py | 37 ++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index b45fc1d1..a9647156 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -62,14 +62,31 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params - @staticmethod - def _search_apt_source(contents, params, pre, post): - return re.search(r"%s %s %s %s\n" % - (pre, params['MIRROR'], params['RELEASE'], post), - contents, flags=re.IGNORECASE) - def test_apt_source_release(self): - """ test_apt_source_release + def test_apt_source_basic(self): + """ test_apt_source_basic + Test Fix deb source string, has to overwrite mirror conf in params + """ + params = self._get_default_params() + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "karmic-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + + def test_apt_source_replacement(self): + """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -81,7 +98,9 @@ class TestAptSourceConfig(TestCase): self.assertTrue(os.path.isfile(self.aptlistfile)) contents = load_tfile_or_url(self.aptlistfile) - self.assertTrue(self._search_apt_source(contents, params, - "deb", "multiverse")) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "multiverse"), + contents, flags=re.IGNORECASE)) # vi: ts=4 expandtab -- cgit v1.2.3 From 8cb8502cc1c99ec787e23504cf4e9f60c01bf0fe Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 15:30:49 +0200 Subject: add test_apt_source_ppa (failing for now) --- .../test_handler/test_handler_apt_source.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a9647156..22a4accf 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -103,4 +103,25 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) + + def test_apt_source_ppa(self): + """ test_apt_source_ppa + Test specification of a ppa + """ + params = self._get_default_params() + cfg = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + # report content before making regex + # FAIL ? goes in "untranslated" + # should become e.g. deb http://ppa.launchpad.net/smoser/cloud-init-test/ubuntu xenial main + contents = load_tfile_or_url(self.aptlistfile) + print(contents) + self.assertTrue(1 == 2) + + # vi: ts=4 expandtab -- cgit v1.2.3 From 66f847f4cba2215490986ffede8e03299dbd5b90 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 17:14:26 +0200 Subject: add test_apt_source_key for sources with a keyid to import --- .../test_handler/test_handler_apt_source.py | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 22a4accf..d9942901 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -104,6 +104,38 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) + def test_apt_source_key(self): + """ test_apt_source_key + Test specification of a source + key + """ + params = self._get_default_params() + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid:': "03683F77", + 'filename': self.aptlistfile} + + cc_apt_configure.add_sources([cfg], params) + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + # report content before making regex + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + # check if key was imported + try: + util.subp(('apt-key', 'list', '03683F77')) + except util.ProcessExecutionError as err: + print("apt-key failed. " + str(err)) + self.assertTrue(1 == 2) + + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From 904aeedf343af17ef88bbbaef9896d425eefa778 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 20:57:30 +0200 Subject: provide valid matcher for ppa so that apt-add-repository is triggered --- .../unittests/test_handler/test_handler_apt_source.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index d9942901..10a03a8d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -144,14 +144,21 @@ class TestAptSourceConfig(TestCase): cfg = {'source': 'ppa:smoser/cloud-init-test', 'filename': self.aptlistfile} - cc_apt_configure.add_sources([cfg], params) + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search - self.assertTrue(os.path.isfile(self.aptlistfile)) + cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) - # report content before making regex - # FAIL ? goes in "untranslated" - # should become e.g. deb http://ppa.launchpad.net/smoser/cloud-init-test/ubuntu xenial main - contents = load_tfile_or_url(self.aptlistfile) + # adding ppa should ignore filename (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + expected_sources_fn=('/etc/apt/sources.list.d/' + 'smoser-ubuntu-cloud-init-test-%s.list' + % params['RELEASE']) + print("filename: %s" % expected_sources_fn) + self.assertTrue(os.path.isfile(expected_sources_fn)) + + # file gets not created, might be permission or env detail + contents = load_tfile_or_url(expected_sources_fn) print(contents) self.assertTrue(1 == 2) -- cgit v1.2.3 From 86c59ffa50a74a1d0001c5ef6ccc78bd6f656fdc Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 10 May 2016 21:03:07 +0200 Subject: use proper asserRaises for try catch --- tests/unittests/test_handler/test_handler_apt_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 10a03a8d..a4d359a5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -132,8 +132,7 @@ class TestAptSourceConfig(TestCase): try: util.subp(('apt-key', 'list', '03683F77')) except util.ProcessExecutionError as err: - print("apt-key failed. " + str(err)) - self.assertTrue(1 == 2) + self.assertRaises(err, "apt-key failed failed") def test_apt_source_ppa(self): @@ -160,7 +159,8 @@ class TestAptSourceConfig(TestCase): # file gets not created, might be permission or env detail contents = load_tfile_or_url(expected_sources_fn) print(contents) - self.assertTrue(1 == 2) + # intentional debug exit + self.assertRaises(ValueError) # vi: ts=4 expandtab -- cgit v1.2.3 From 4c5d1966ead445dc6d110e9677902b95dfef2dc5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 10:57:14 +0200 Subject: test test_apt_source_key with mocked util.subp --- tests/unittests/test_handler/test_handler_apt_source.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a4d359a5..29535bee 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -6,6 +6,11 @@ import shutil import tempfile import re +try: + from unittest import mock +except ImportError: + import mock + from cloudinit import distros from cloudinit import util from cloudinit.config import cc_apt_configure @@ -116,11 +121,13 @@ class TestAptSourceConfig(TestCase): 'keyid:': "03683F77", 'filename': self.aptlistfile} - cc_apt_configure.add_sources([cfg], params) + with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') self.assertTrue(os.path.isfile(self.aptlistfile)) - # report content before making regex contents = load_tfile_or_url(self.aptlistfile) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", @@ -128,11 +135,6 @@ class TestAptSourceConfig(TestCase): 'cloud-init-test/ubuntu'), "xenial", "main"), contents, flags=re.IGNORECASE)) - # check if key was imported - try: - util.subp(('apt-key', 'list', '03683F77')) - except util.ProcessExecutionError as err: - self.assertRaises(err, "apt-key failed failed") def test_apt_source_ppa(self): -- cgit v1.2.3 From fef11c6a98ea74774aa84b3b14007b246a1c615e Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 10:57:33 +0200 Subject: fix specification of keyid --- tests/unittests/test_handler/test_handler_apt_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 29535bee..601504bd 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -118,7 +118,7 @@ class TestAptSourceConfig(TestCase): 'http://ppa.launchpad.net/' 'smoser/cloud-init-test/ubuntu' ' xenial main'), - 'keyid:': "03683F77", + 'keyid': "03683F77", 'filename': self.aptlistfile} with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: -- cgit v1.2.3 From 0e299c89e10a6ccb583588ff31ac783421e57501 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 11:03:51 +0200 Subject: convert test_apt_source_ppa to use a mocked util.subp --- tests/unittests/test_handler/test_handler_apt_source.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 601504bd..e73a72c6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -148,21 +148,13 @@ class TestAptSourceConfig(TestCase): # default matcher needed for ppa matcher = re.compile(r'^[\w-]+:\w').search - cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) + mockobj.assert_called_once_with(['add-apt-repository', + 'ppa:smoser/cloud-init-test']) # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) - expected_sources_fn=('/etc/apt/sources.list.d/' - 'smoser-ubuntu-cloud-init-test-%s.list' - % params['RELEASE']) - print("filename: %s" % expected_sources_fn) - self.assertTrue(os.path.isfile(expected_sources_fn)) - - # file gets not created, might be permission or env detail - contents = load_tfile_or_url(expected_sources_fn) - print(contents) - # intentional debug exit - self.assertRaises(ValueError) # vi: ts=4 expandtab -- cgit v1.2.3 From 0a35ba5b8d85077a865a137887aa8cfd76405eb2 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 11:06:08 +0200 Subject: fix a few forgotten pep8 warnings in test_handler_apt_source.py --- tests/unittests/test_handler/test_handler_apt_source.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index e73a72c6..849f23c9 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -40,12 +40,14 @@ PACKAGE_MIRRORS = [ GAPMI = distros._get_arch_package_mirror_info + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + class TestAptSourceConfig(TestCase): """ TestAptSourceConfig Main Class to test apt_source configs @@ -56,7 +58,6 @@ class TestAptSourceConfig(TestCase): self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") - @staticmethod def _get_default_params(): """ get_default_params @@ -67,7 +68,6 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params - def test_apt_source_basic(self): """ test_apt_source_basic Test Fix deb source string, has to overwrite mirror conf in params @@ -89,7 +89,6 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_source_replacement(self): """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs @@ -108,7 +107,6 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) - def test_apt_source_key(self): """ test_apt_source_key Test specification of a source + key @@ -121,7 +119,8 @@ class TestAptSourceConfig(TestCase): 'keyid': "03683F77", 'filename': self.aptlistfile} - with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: cc_apt_configure.add_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') @@ -136,7 +135,6 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) - def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa @@ -151,7 +149,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params, aa_repo_match=matcher) mockobj.assert_called_once_with(['add-apt-repository', - 'ppa:smoser/cloud-init-test']) + 'ppa:smoser/cloud-init-test']) # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) -- cgit v1.2.3 From a9cd544d7de1af90f6c5cf9df43135b530acb308 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:52:17 +0200 Subject: split test_apt_source_key into one for key and one for keyid --- .../test_handler/test_handler_apt_source.py | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 849f23c9..a7db0fa6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -107,6 +107,34 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) + def test_apt_source_keyid(self): + """ test_apt_source_keyid + Test specification of a source + keyid + """ + params = self._get_default_params() + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1234', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') + + self.assertTrue(os.path.isfile(self.aptlistfile)) + + contents = load_tfile_or_url(self.aptlistfile) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "main"), + contents, flags=re.IGNORECASE)) + def test_apt_source_key(self): """ test_apt_source_key Test specification of a source + key @@ -116,14 +144,14 @@ class TestAptSourceConfig(TestCase): 'http://ppa.launchpad.net/' 'smoser/cloud-init-test/ubuntu' ' xenial main'), - 'keyid': "03683F77", + 'key': "fakekey 4321", 'filename': self.aptlistfile} with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: cc_apt_configure.add_sources([cfg], params) - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') self.assertTrue(os.path.isfile(self.aptlistfile)) -- cgit v1.2.3 From d82f6c1ecd255ed3a76bd8ef7b76163408f0b398 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:55:37 +0200 Subject: add test_apt_source_keyonly (not yet supported) --- tests/unittests/test_handler/test_handler_apt_source.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index a7db0fa6..92a92406 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -163,6 +163,22 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) + def test_apt_source_keyonly(self): + """ test_apt_source_keyonly + Test specification key without source (not yet supported) + """ + params = self._get_default_params() + cfg = {'key': "fakekey 4242", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_once_with(('apt-key', 'add', '-'), 'fakekey 4242') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From b4f76a0a855d792acc05807a3a62cc8c72d80792 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:56:12 +0200 Subject: apt_apt_source_key doesn't need a mocked retval --- tests/unittests/test_handler/test_handler_apt_source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 92a92406..091b07da 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -147,8 +147,7 @@ class TestAptSourceConfig(TestCase): 'key': "fakekey 4321", 'filename': self.aptlistfile} - with mock.patch.object(util, 'subp', - return_value=('fakekey 1234', '')) as mockobj: + with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') -- cgit v1.2.3 From d5bc051cdc418efa0a910f8b61790523b89e8f91 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 16:59:15 +0200 Subject: add test_apt_source_keyidonly (not yet supported) --- tests/unittests/test_handler/test_handler_apt_source.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 091b07da..de009174 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -178,6 +178,23 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_keyidonly(self): + """ test_apt_source_keyidonly + Test specification of a keyid without source (not yet supported) + """ + params = self._get_default_params() + cfg = {'keyid': "03683F77", + 'filename': self.aptlistfile} + + with mock.patch.object(util, 'subp', + return_value=('fakekey 1212', '')) as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1212') + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From da2640951d2d87f38dd539e53115af98f12c11ac Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 11 May 2016 18:44:52 +0200 Subject: fix pep8 warning --- tests/unittests/test_handler/test_handler_apt_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index de009174..01d56559 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -173,7 +173,8 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params) - mockobj.assert_called_once_with(('apt-key', 'add', '-'), 'fakekey 4242') + mockobj.assert_called_once_with(('apt-key', 'add', '-'), + 'fakekey 4242') # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) -- cgit v1.2.3 From cca640d332eb8a6b068033a28b0b319873c7fbf6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:21:50 +0200 Subject: allow to add keys without specifying a source --- cloudinit/config/cc_apt_configure.py | 42 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 702977cb..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -164,6 +164,29 @@ def generate_sources_list(codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key(ent, errorlist): + """ + add key to the system as defiend in entry (if any) + suppords raw keys or keyid's + The latter will as a first step fetched to get the raw key + """ + if ('keyid' in ent and 'key' not in ent): + keyserver = "keyserver.ubuntu.com" + if 'keyserver' in ent: + keyserver = ent['keyserver'] + try: + ent['key'] = getkeybyid(ent['keyid'], keyserver) + except: + errorlist.append([ent, "failed to get key from %s" % keyserver]) + return + + if 'key' in ent: + try: + util.subp(('apt-key', 'add', '-'), ent['key']) + except: + errorlist.append([ent, "failed add key"]) + + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated @@ -179,6 +202,9 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: + # keys can be added without specifying a source + add_key(ent, errorlist) + if 'source' not in ent: errorlist.append(["", "missing source"]) continue @@ -201,22 +227,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) - if ('keyid' in ent and 'key' not in ent): - ks = "keyserver.ubuntu.com" - if 'keyserver' in ent: - ks = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], ks) - except: - errorlist.append([source, "failed to get key from %s" % ks]) - continue - - if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - errorlist.append([source, "failed add key"]) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") -- cgit v1.2.3 From a6282380814750851c0dc2ac2d4d3386eb6fcce4 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:26:27 +0200 Subject: update doc/examples/cloud-config.txt with new key-without-source cases --- doc/examples/cloud-config.txt | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 1236796c..f84d526d 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -111,12 +111,14 @@ apt_sources: keyid: F430BBA5 # GPG key ID published on a key server filename: byobu-ppa.list + # this would only import the key without adding a ppa or other source spec + - keyid: F430BBA5 # GPG key ID published on a key server + # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block # * Providing the PBG key here is the most robust method for # specifying a key, as it removes dependency on a remote key server - - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- @@ -132,6 +134,24 @@ apt_sources: =Y2oI -----END PGP PUBLIC KEY BLOCK----- + # Custom gpg key: + # * As the keyid also a key can be specified withut a related source + # * all other facts mentioned above still apply + - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: SKS 1.0.10 + + mI0ESpA3UQEEALdZKVIMq0j6qWAXAyxSlF63SvPVIgxHPb9Nk0DZUixn+akqytxG4zKCONz6 + qLjoBBfHnynyVLfT4ihg9an1PqxRnTO+JKQxl8NgKGz6Pon569GtAOdWNKw15XKinJTDLjnj + 9y96ljJqRcpV9t/WsIcdJPcKFR5voHTEoABE2aEXABEBAAG0GUxhdW5jaHBhZCBQUEEgZm9y + IEFsZXN0aWOItgQTAQIAIAUCSpA3UQIbAwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJEA7H + 5Qi+CcVxWZ8D/1MyYvfj3FJPZUm2Yo1zZsQ657vHI9+pPouqflWOayRR9jbiyUFIn0VdQBrP + t0FwvnOFArUovUWoKAEdqR8hPy3M3APUZjl5K4cMZR/xaMQeQRZ5CHpS4DBKURKAHC0ltS5o + uBJKQOZm5iltJp15cgyIkBkGe8Mx18VFyVglAZey + =Y2oI + -----END PGP PUBLIC KEY BLOCK----- + + ## apt config via system_info: # under the 'system_info', you can further customize cloud-init's interaction # with apt. -- cgit v1.2.3 From 48a50dabf482a8ae029775a94be2d1b84763f98d Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 09:52:02 +0200 Subject: remove unused test parts in test_handler_apt_source --- .../test_handler/test_handler_apt_source.py | 23 ---------------------- 1 file changed, 23 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 01d56559..38c93e0e 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -17,29 +17,6 @@ from cloudinit.config import cc_apt_configure from ..helpers import TestCase -UNKNOWN_ARCH_INFO = { - 'arches': ['default'], - 'failsafe': {'primary': 'http://fs-primary-default', - 'security': 'http://fs-security-default'} -} - -PACKAGE_MIRRORS = [ - {'arches': ['i386', 'amd64'], - 'failsafe': {'primary': 'http://fs-primary-intel', - 'security': 'http://fs-security-intel'}, - 'search': { - 'primary': ['http://%(ec2_region)s.ec2/', - 'http://%(availability_zone)s.clouds/'], - 'security': ['http://security-mirror1-intel', - 'http://security-mirror2-intel']}}, - {'arches': ['armhf', 'armel'], - 'failsafe': {'primary': 'http://fs-primary-arm', - 'security': 'http://fs-security-arm'}}, - UNKNOWN_ARCH_INFO -] - -GAPMI = distros._get_arch_package_mirror_info - def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url -- cgit v1.2.3 From 1b4a06cd7933f876b16576567bbc0dab46b83b6b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:21:29 +0200 Subject: add testcase for rendering of templates into source.list --- .../test_handler_apt_configure_sources_list.py | 89 ++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/unittests/test_handler/test_handler_apt_configure_sources_list.py diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py new file mode 100644 index 00000000..46edb628 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -0,0 +1,89 @@ +""" test_handler_apt_configure_sources_list +Test templating of sources list +""" +import os +import shutil +import tempfile +import re + +import logging + +try: + from unittest import mock +except ImportError: + import mock + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import util +from cloudinit import helpers +from cloudinit import templater + +from cloudinit.sources import DataSourceNone +from cloudinit.config import cc_apt_configure + +from .. import helpers as t_help + +LOG = logging.getLogger(__name__) + + +def load_tfile_or_url(*args, **kwargs): + """ load_tfile_or_url + load file and return content after decoding + """ + return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) + + +class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): + """ TestAptSourceConfigSourceList + Main Class to test sources list rendering + """ + def setUp(self): + super(TestAptSourceConfigSourceList, self).setUp() + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + +# TODO - Ubuntu template +# TODO - Debian template +# TODO Later - custom template filename +# TODO Later - custom template raw + + def test_apt_source_list_ubuntu(self): + """ test_apt_source_list + Test rendering of a source.list from template for ubuntu + """ + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + + cfg = {'apt_mirror': 'http://archive.ubuntu.com/ubuntu/'} + mycloud = self._get_cloud('ubuntu') + + with mock.patch.object(templater, 'render_to_file') as mocktmpl: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call(('/etc/cloud/templates/' + 'sources.list.ubuntu.tmpl')) + mocktmpl.assert_called_once_with(('/etc/cloud/templates/' + 'sources.list.ubuntu.tmpl'), + '/etc/apt/sources.list', + {'codename': '', + 'primary': + 'http://archive.ubuntu.com/ubuntu/', + 'mirror': + 'http://archive.ubuntu.com/ubuntu/'}) + + +# vi: ts=4 expandtab -- cgit v1.2.3 From 83f2351bba273273d04759ebacd2a25df9045090 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:28:36 +0200 Subject: test debian and ubuntu source.list templating --- .../test_handler_apt_configure_sources_list.py | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 46edb628..aff272a3 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,15 +58,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # TODO Later - custom template filename # TODO Later - custom template raw - def test_apt_source_list_ubuntu(self): - """ test_apt_source_list - Test rendering of a source.list from template for ubuntu + def apt_source_list(self, distro, mirror): + """ apt_source_list + Test rendering of a source.list from template for a given distro """ self.patchOS(self.new_root) self.patchUtils(self.new_root) - cfg = {'apt_mirror': 'http://archive.ubuntu.com/ubuntu/'} - mycloud = self._get_cloud('ubuntu') + cfg = {'apt_mirror': mirror} + mycloud = self._get_cloud(distro) with mock.patch.object(templater, 'render_to_file') as mocktmpl: with mock.patch.object(os.path, 'isfile', @@ -75,15 +75,29 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): LOG, None) mockisfile.assert_any_call(('/etc/cloud/templates/' - 'sources.list.ubuntu.tmpl')) + 'sources.list.%s.tmpl' % distro)) mocktmpl.assert_called_once_with(('/etc/cloud/templates/' - 'sources.list.ubuntu.tmpl'), + 'sources.list.%s.tmpl' % distro), '/etc/apt/sources.list', {'codename': '', 'primary': - 'http://archive.ubuntu.com/ubuntu/', + mirror, 'mirror': - 'http://archive.ubuntu.com/ubuntu/'}) + mirror}) + + + def test_apt_source_list_ubuntu(self): + """ test_apt_source_list_ubuntu + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + + def test_apt_source_list_debian(self): + """ test_apt_source_list_debian + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', 'ftp.us.debian.org') # vi: ts=4 expandtab -- cgit v1.2.3 From 6fc583a4bcb49f7dbecdac095bc63e90dd6edaf3 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:43:59 +0200 Subject: test mirror list with failing mirror --- cloudinit/config/cc_apt_configure.py | 1 + .../test_handler_apt_configure_sources_list.py | 23 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..ccbdcbc1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,6 +57,7 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) + print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index aff272a3..bac2da24 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,14 +58,20 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # TODO Later - custom template filename # TODO Later - custom template raw - def apt_source_list(self, distro, mirror): + def apt_source_list(self, distro, mirror, mirrorcheck=None): """ apt_source_list Test rendering of a source.list from template for a given distro """ self.patchOS(self.new_root) self.patchUtils(self.new_root) - cfg = {'apt_mirror': mirror} + if mirrorcheck is None: + mirrorcheck = mirror + + if isinstance(mirror, list): + cfg = {'apt_mirror_search': mirror} + else: + cfg = {'apt_mirror': mirror} mycloud = self._get_cloud(distro) with mock.patch.object(templater, 'render_to_file') as mocktmpl: @@ -81,9 +87,9 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): '/etc/apt/sources.list', {'codename': '', 'primary': - mirror, + mirrorcheck, 'mirror': - mirror}) + mirrorcheck}) def test_apt_source_list_ubuntu(self): @@ -100,4 +106,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('debian', 'ftp.us.debian.org') + def test_apt_srcl_ubuntu_mirrorfail(self): + """ test_apt_source_list_ubuntu_mirrorfail + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', ['http://does.not.exist', + 'http://archive.ubuntu.com/ubuntu/'], + 'http://archive.ubuntu.com/ubuntu/') + + # vi: ts=4 expandtab -- cgit v1.2.3 From d2af10ab5d3fef34934c04d4590ea611204f44c5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:44:29 +0200 Subject: order code in test order --- .../test_handler_apt_configure_sources_list.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index bac2da24..d48167c9 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -92,13 +92,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mirrorcheck}) - def test_apt_source_list_ubuntu(self): - """ test_apt_source_list_ubuntu - Test rendering of a source.list from template for ubuntu - """ - self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') - - def test_apt_source_list_debian(self): """ test_apt_source_list_debian Test rendering of a source.list from template for debian @@ -106,6 +99,13 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('debian', 'ftp.us.debian.org') + def test_apt_source_list_ubuntu(self): + """ test_apt_source_list_ubuntu + Test rendering of a source.list from template for ubuntu + """ + self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + + def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu -- cgit v1.2.3 From 18864f8e7331da359399decb1b080e36fa343f5a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 13:45:15 +0200 Subject: remove missed test print --- cloudinit/config/cc_apt_configure.py | 1 - .../test_handler/test_handler_apt_configure_sources_list.py | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ccbdcbc1..1d3eddff 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -57,7 +57,6 @@ def handle(name, cfg, cloud, log, _args): release = get_release() mirrors = find_apt_mirror_info(cloud, cfg) - print(mirrors) if not mirrors or "primary" not in mirrors: log.debug(("Skipping module named %s," " no package 'mirror' located"), name) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index d48167c9..b8fe03ae 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -106,6 +106,15 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') + def test_apt_srcl_debian_mirrorfail(self): + """ test_apt_source_list_debian_mirrorfail + Test rendering of a source.list from template for debian + """ + self.apt_source_list('debian', ['http://does.not.exist', + 'ftp.us.debian.org'], + 'ftp.us.debian.org') + + def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu -- cgit v1.2.3 From dc2b81f3f6922c678806ce9120e9ce4c590243dd Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 14:09:17 +0200 Subject: use recommended http mirror redirection for debian --- .../test_handler/test_handler_apt_configure_sources_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index b8fe03ae..5255c5b9 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -96,7 +96,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ test_apt_source_list_debian Test rendering of a source.list from template for debian """ - self.apt_source_list('debian', 'ftp.us.debian.org') + self.apt_source_list('debian', 'http://httpredir.debian.org/debian') def test_apt_source_list_ubuntu(self): @@ -111,8 +111,8 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): Test rendering of a source.list from template for debian """ self.apt_source_list('debian', ['http://does.not.exist', - 'ftp.us.debian.org'], - 'ftp.us.debian.org') + 'http://httpredir.debian.org/debian'], + 'http://httpredir.debian.org/debian') def test_apt_srcl_ubuntu_mirrorfail(self): -- cgit v1.2.3 From 9a75e200402410e6f0ae841cdadebb20840c6a8e Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 14:30:11 +0200 Subject: initial version of a test for a custom source.list template --- .../test_handler_apt_configure_sources_list.py | 49 +++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 5255c5b9..8777d2a9 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -26,6 +26,24 @@ from .. import helpers as t_help LOG = logging.getLogger(__name__) +YAML_TEXT_CUSTOM_SL = """ +apt_mirror: http://archive.ubuntu.com/ubuntu/ +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + + # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to + # newer versions of the distribution. + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + # FIND_SOMETHING_SPECIAL +""" def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url @@ -53,11 +71,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): myds.metadata.update(metadata) return cloud.Cloud(myds, paths, {}, mydist, None) -# TODO - Ubuntu template -# TODO - Debian template -# TODO Later - custom template filename -# TODO Later - custom template raw - def apt_source_list(self, distro, mirror, mirrorcheck=None): """ apt_source_list Test rendering of a source.list from template for a given distro @@ -124,4 +137,30 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/') + def test_apt_srcl_custom(self): + """ test_apt_srcl_custom + Test rendering from a custom source.list template + """ + self.patchOS(self.new_root) + self.patchUtils(self.new_root) + + cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) + mycloud = self._get_cloud('ubuntu') + mirrorcheck = 'http://archive.ubuntu.com/ubuntu/' + + with mock.patch.object(templater, 'render_to_file') as mocktmpl: + with mock.patch.object(os.path, 'isfile', + return_value=True) as mockisfile: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockisfile.assert_any_call(('/etc/cloud/templates/sources.list.ubuntu.tmpl')) + mocktmpl.assert_called_once_with(('/etc/cloud/templates/sources.list.ubuntu.tmpl'), + '/etc/apt/sources.list', + {'codename': '', + 'primary': + mirrorcheck, + 'mirror': + mirrorcheck}) + # vi: ts=4 expandtab -- cgit v1.2.3 From c6cbba7184f0864f34b90532a4b0d3d61fdd6bc9 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:42:31 +0200 Subject: Finalize test_apt_srcl_custom Adding: - known content after full templating - restore subp from mocking for proper execution - drop all kind of superfluous mocking of util and OS --- .../test_handler_apt_configure_sources_list.py | 41 +++++++++++----------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 8777d2a9..c1ca71af 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -58,6 +58,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ def setUp(self): super(TestAptSourceConfigSourceList, self).setUp() + self.subp = util.subp self.new_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.new_root) @@ -75,9 +76,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ apt_source_list Test rendering of a source.list from template for a given distro """ - self.patchOS(self.new_root) - self.patchUtils(self.new_root) - if mirrorcheck is None: mirrorcheck = mirror @@ -141,26 +139,29 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): """ test_apt_srcl_custom Test rendering from a custom source.list template """ - self.patchOS(self.new_root) - self.patchUtils(self.new_root) - cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) mycloud = self._get_cloud('ubuntu') - mirrorcheck = 'http://archive.ubuntu.com/ubuntu/' - with mock.patch.object(templater, 'render_to_file') as mocktmpl: - with mock.patch.object(os.path, 'isfile', - return_value=True) as mockisfile: - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + # the second mock restores the original subp + with mock.patch.object(util, 'write_file') as mockwrite, \ + mock.patch.object(util, 'subp', self.subp) as mocksubp: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) + + mockwrite.assert_called_once_with( + '/etc/apt/sources.list', + ("## Note, this file is written by cloud-init on first boot of an" + " instance\n## modifications made here will not survive a re-bun" + "dle.\n## if you wish to make changes you can:\n## a.) add 'apt_" + "preserve_sources_list: true' to /etc/cloud/cloud.cfg\n## or" + " do the same in user-data\n## b.) add sources in /etc/apt/sourc" + "es.list.d\n## c.) make changes to template file /etc/cloud/temp" + "lates/sources.list.tmpl\n\n# See http://help.ubuntu.com/communi" + "ty/UpgradeNotes for how to upgrade to\n# newer versions of the " + "distribution.\ndeb http://archive.ubuntu.com/ubuntu/ xenial mai" + "n restricted\ndeb-src http://archive.ubuntu.com/ubuntu/ xenial " + "main restricted\n# FIND_SOMETHING_SPECIAL\n"), + mode=420) - mockisfile.assert_any_call(('/etc/cloud/templates/sources.list.ubuntu.tmpl')) - mocktmpl.assert_called_once_with(('/etc/cloud/templates/sources.list.ubuntu.tmpl'), - '/etc/apt/sources.list', - {'codename': '', - 'primary': - mirrorcheck, - 'mirror': - mirrorcheck}) # vi: ts=4 expandtab -- cgit v1.2.3 From cdee96acdde84071d54764e2554e6ee27702166c Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:51:00 +0200 Subject: use old style nested context to make pep8 happy --- .../test_handler_apt_configure_sources_list.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index c1ca71af..16d6a5d1 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -45,6 +45,7 @@ apt_custom_sources_list: | # FIND_SOMETHING_SPECIAL """ + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding @@ -102,21 +103,18 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'mirror': mirrorcheck}) - def test_apt_source_list_debian(self): """ test_apt_source_list_debian Test rendering of a source.list from template for debian """ self.apt_source_list('debian', 'http://httpredir.debian.org/debian') - def test_apt_source_list_ubuntu(self): """ test_apt_source_list_ubuntu Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') - def test_apt_srcl_debian_mirrorfail(self): """ test_apt_source_list_debian_mirrorfail Test rendering of a source.list from template for debian @@ -125,7 +123,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://httpredir.debian.org/debian'], 'http://httpredir.debian.org/debian') - def test_apt_srcl_ubuntu_mirrorfail(self): """ test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu @@ -134,7 +131,6 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/'], 'http://archive.ubuntu.com/ubuntu/') - def test_apt_srcl_custom(self): """ test_apt_srcl_custom Test rendering from a custom source.list template @@ -143,10 +139,10 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mycloud = self._get_cloud('ubuntu') # the second mock restores the original subp - with mock.patch.object(util, 'write_file') as mockwrite, \ - mock.patch.object(util, 'subp', self.subp) as mocksubp: - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + with mock.patch.object(util, 'write_file') as mockwrite: + with mock.patch.object(util, 'subp', self.subp) as mocksubp: + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) mockwrite.assert_called_once_with( '/etc/apt/sources.list', -- cgit v1.2.3 From 0d8bf6f2c1464b5f4dab735841a50d02016d2caf Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 15:51:21 +0200 Subject: add feature to allow a custom template for source list --- cloudinit/config/cc_apt_configure.py | 18 +++++++++++++----- cloudinit/templater.py | 5 +++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 1d3eddff..2ab5e86c 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,6 +21,7 @@ import glob import os import re +import tempfile from cloudinit import templater from cloudinit import util @@ -70,7 +71,7 @@ def handle(name, cfg, cloud, log, _args): if not util.get_cfg_option_bool(cfg, 'apt_preserve_sources_list', False): - generate_sources_list(release, mirrors, cloud, log) + generate_sources_list(cfg, release, mirrors, cloud, log) old_mirrors = cfg.get('apt_old_mirrors', {"primary": "archive.ubuntu.com/ubuntu", "security": "security.ubuntu.com/ubuntu"}) @@ -149,7 +150,17 @@ def get_release(): return stdout.strip() -def generate_sources_list(codename, mirrors, cloud, log): +def generate_sources_list(cfg, codename, mirrors, cloud, log): + params = {'codename': codename} + for k in mirrors: + params[k] = mirrors[k] + + custtmpl = cfg.get('apt_custom_sources_list', None) + if custtmpl is not None: + templater.render_string_to_file(custtmpl, + '/etc/apt/sources.list', params) + return + template_fn = cloud.get_template_filename('sources.list.%s' % (cloud.distro.name)) if not template_fn: @@ -158,9 +169,6 @@ def generate_sources_list(codename, mirrors, cloud, log): log.warn("No template found, not rendering /etc/apt/sources.list") return - params = {'codename': codename} - for k in mirrors: - params[k] = mirrors[k] templater.render_to_file(template_fn, '/etc/apt/sources.list', params) diff --git a/cloudinit/templater.py b/cloudinit/templater.py index a9231482..8a6ad417 100644 --- a/cloudinit/templater.py +++ b/cloudinit/templater.py @@ -142,6 +142,11 @@ def render_to_file(fn, outfn, params, mode=0o644): util.write_file(outfn, contents, mode=mode) +def render_string_to_file(content, outfn, params, mode=0o644): + contents = render_string(content, params) + util.write_file(outfn, contents, mode=mode) + + def render_string(content, params): if not params: params = {} -- cgit v1.2.3 From 3be3e7452410d97ef9f9d4b525fa828de1f57bc0 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 16:27:50 +0200 Subject: Document apt_custom_sources_list in examples --- doc/examples/cloud-config.txt | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index f84d526d..75a4b6d4 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -72,6 +72,36 @@ apt_pipelining: False # then apt_mirror above will have no effect apt_preserve_sources_list: true +# Provide a custom template for rednering sources.list +# Default: a default template for Ubuntu/Debain will be used as packaged in +# Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl +# Debian: /etc/cloud/templates/sources.list.debian.tmpl +# Others: n/a +# This will follow the normal mirror/codename replacement rules before +# being written to disk. +apt_custom_sources_list: | + ## template:jinja + ## Note, this file is written by cloud-init on first boot of an instance + ## modifications made here will not survive a re-bundle. + ## if you wish to make changes you can: + ## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg + ## or do the same in user-data + ## b.) add sources in /etc/apt/sources.list.d + ## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + deb {{mirror}} {{codename}} main restricted + deb-src {{mirror}} {{codename}} main restricted + + # could drop some of the usually used entries + + # could refer to other mirrors + deb http://ddebs.ubuntu.com {{codename}} main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-updates main restricted universe multiverse + deb http://ddebs.ubuntu.com {{codename}}-proposed main restricted universe multiverse + + # or even more uncommon examples like local or NFS mounted repos, + # eventually whatever is compatible with sources.list syntax + deb file:/home/apt/debian unstable main contrib non-free + # 'source' entries in apt-sources that match this python regex # expression will be passed to add-apt-repository add_apt_repo_match: '^[\w-]+:\w' -- cgit v1.2.3 From 4f6b14ea0f21f015ce73a28d985ecded981b931d Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 16:54:20 +0200 Subject: remove no more applicable "not supported" statements --- tests/unittests/test_handler/test_handler_apt_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 38c93e0e..23f458eb 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -141,7 +141,7 @@ class TestAptSourceConfig(TestCase): def test_apt_source_keyonly(self): """ test_apt_source_keyonly - Test specification key without source (not yet supported) + Test specification key without source """ params = self._get_default_params() cfg = {'key': "fakekey 4242", @@ -158,7 +158,7 @@ class TestAptSourceConfig(TestCase): def test_apt_source_keyidonly(self): """ test_apt_source_keyidonly - Test specification of a keyid without source (not yet supported) + Test specification of a keyid without source """ params = self._get_default_params() cfg = {'keyid': "03683F77", -- cgit v1.2.3 From 9c6b5f54ad5b83131de6d997930bd9f4031e6a83 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:01 +0200 Subject: move errorlist.append out of add_key --- cloudinit/config/cc_apt_configure.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2ab5e86c..91d02815 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -172,7 +172,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) -def add_key(ent, errorlist): +def add_key(ent): """ add key to the system as defiend in entry (if any) suppords raw keys or keyid's @@ -185,14 +185,13 @@ def add_key(ent, errorlist): try: ent['key'] = getkeybyid(ent['keyid'], keyserver) except: - errorlist.append([ent, "failed to get key from %s" % keyserver]) - return + raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: try: util.subp(('apt-key', 'add', '-'), ent['key']) except: - errorlist.append([ent, "failed add key"]) + raise Exception('failed add key') def add_sources(srclist, template_params=None, aa_repo_match=None): @@ -211,7 +210,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] for ent in srclist: # keys can be added without specifying a source - add_key(ent, errorlist) + try: + add_key(ent) + except Exception as detail: + errorlist.append([ent, detail]) if 'source' not in ent: errorlist.append(["", "missing source"]) -- cgit v1.2.3 From e55ccfa5670e16aa7431a193d0838aa7d04db4d5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 19:50:34 +0200 Subject: remove Unnecessary parens in add_key --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 91d02815..492c3c00 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -178,7 +178,7 @@ def add_key(ent): suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ - if ('keyid' in ent and 'key' not in ent): + if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] -- cgit v1.2.3 From 53834934e4c520b2fb8b5acffca641213ddd688a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:29:33 +0200 Subject: fix EXPORT_GPG_KEYID for long key fingerprints --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 492c3c00..28f20939 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -43,7 +43,7 @@ EXPORT_GPG_KEYID = """ [ -n "$k" ] || exit 1; armour=$(gpg --list-keys --armour "${k}") if [ -z "${armour}" ]; then - gpg --keyserver ${ks} --recv $k >/dev/null && + gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && gpg --batch --yes --delete-keys "${k}" fi -- cgit v1.2.3 From 2202494b72cae19cbf9d34a8f3176d7021becb13 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 20:53:12 +0200 Subject: split add_key and add_key_raw fior better testability --- cloudinit/config/cc_apt_configure.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 28f20939..e7b8a9b3 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,10 +171,19 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) +def add_key_raw(key): + """ + actual adding of a key as defined in key argument + to the system + """ + try: + util.subp(('apt-key', 'add', '-'), key) + except: + raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in entry (if any) + add key to the system as defiend in ent (if any) suppords raw keys or keyid's The latter will as a first step fetched to get the raw key """ @@ -188,10 +197,7 @@ def add_key(ent): raise Exception('failed to get key from %s' % keyserver) if 'key' in ent: - try: - util.subp(('apt-key', 'add', '-'), ent['key']) - except: - raise Exception('failed add key') + add_key_raw(ent['key']) def add_sources(srclist, template_params=None, aa_repo_match=None): -- cgit v1.2.3 From 46f930a718a89c514ba7caa22096201e7c6ca0a8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:00:30 +0200 Subject: Adding test_apt_source_keyid_real and test_apt_source_longkeyid_real This now ensures that the stack of fetching IDs from keyservers and adding them really works by comparing against known good keys that are expected. --- .../test_handler/test_handler_apt_source.py | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 23f458eb..e50c7468 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -17,6 +17,19 @@ from cloudinit.config import cc_apt_configure from ..helpers import TestCase +EXPECTEDKEY = """-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0ESuZLUgEEAKkqq3idtFP7g9hzOu1a8+v8ImawQN4TrvlygfScMU1TIS1eC7UQ +NUA8Qqgr9iUaGnejb0VciqftLrU9D6WYHSKz+EITefgdyJ6SoQxjoJdsCpJ7o9Jy +8PQnpRttiFm4qHu6BVnKnBNxw/z3ST9YMqW5kbMQpfxbGe+obRox59NpABEBAAG0 +HUxhdW5jaHBhZCBQUEEgZm9yIFNjb3R0IE1vc2VyiLYEEwECACAFAkrmS1ICGwMG +CwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAGILvPA2g/d3aEA/9tVjc10HOZwV29 +OatVuTeERjjrIbxflO586GLA8cp0C9RQCwgod/R+cKYdQcHjbqVcP0HqxveLg0RZ +FJpWLmWKamwkABErwQLGlM/Hwhjfade8VvEQutH5/0JgKHmzRsoqfR+LMO6OS+Sm +S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== +=ACB2 +-----END PGP PUBLIC KEY BLOCK-----""" def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url @@ -173,6 +186,42 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_keyid_real(self): + """ test_apt_source_keyid_real + Test specification of a keyid without source incl + up to addition of the key (nothing but add_key_raw mocked) + """ + keyid = "03683F77" + params = self._get_default_params() + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(EXPECTEDKEY) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + + def test_apt_source_longkeyid_real(self): + """ test_apt_source_keyid_real + Test specification of a long key fingerprint without source incl + up to addition of the key (nothing but add_key_raw mocked) + """ + keyid = "B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77" + params = self._get_default_params() + cfg = {'keyid': keyid, + 'filename': self.aptlistfile} + + with mock.patch.object(cc_apt_configure, 'add_key_raw') as mockobj: + cc_apt_configure.add_sources([cfg], params) + + mockobj.assert_called_with(EXPECTEDKEY) + + # filename should be ignored on key only + self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa(self): """ test_apt_source_ppa Test specification of a ppa -- cgit v1.2.3 From 9c098751b8065da609566572a7badd0d66c2b6ba Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:01:30 +0200 Subject: remove superfluous import --- tests/unittests/test_handler/test_handler_apt_source.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index e50c7468..88b4ccc6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -11,7 +11,6 @@ try: except ImportError: import mock -from cloudinit import distros from cloudinit import util from cloudinit.config import cc_apt_configure -- cgit v1.2.3 From 3ece03a2df95bdb40851b9d629b39c43a233868b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:03:51 +0200 Subject: alphabetical import order --- doc/examples/cloud-config.txt | 3 +++ tests/unittests/test_handler/test_handler_apt_source.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 75a4b6d4..8adc5a96 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -144,6 +144,9 @@ apt_sources: # this would only import the key without adding a ppa or other source spec - keyid: F430BBA5 # GPG key ID published on a key server + # In general keyid's can also be specified via their long fingerprints + - keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 88b4ccc6..439bd038 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -2,17 +2,17 @@ Testing various config variations of the apt_source config """ import os +import re import shutil import tempfile -import re try: from unittest import mock except ImportError: import mock -from cloudinit import util from cloudinit.config import cc_apt_configure +from cloudinit import util from ..helpers import TestCase -- cgit v1.2.3 From 338198c5a802207cabd7c13422fcf0e03c667ce7 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:04:34 +0200 Subject: alphabetical order on imports --- .../test_handler/test_handler_apt_configure_sources_list.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 16d6a5d1..d4fdee21 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -1,12 +1,11 @@ """ test_handler_apt_configure_sources_list Test templating of sources list """ +import logging import os +import re import shutil import tempfile -import re - -import logging try: from unittest import mock @@ -15,12 +14,12 @@ except ImportError: from cloudinit import cloud from cloudinit import distros -from cloudinit import util from cloudinit import helpers from cloudinit import templater +from cloudinit import util -from cloudinit.sources import DataSourceNone from cloudinit.config import cc_apt_configure +from cloudinit.sources import DataSourceNone from .. import helpers as t_help -- cgit v1.2.3 From 3c2c85a7c152c5026ff3122a7e134c5db4cfdce7 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:09:34 +0200 Subject: fix old typo in example --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 8adc5a96..31b791b6 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -150,7 +150,7 @@ apt_sources: # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block - # * Providing the PBG key here is the most robust method for + # * Providing the PGP key here is the most robust method for # specifying a key, as it removes dependency on a remote key server - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -- cgit v1.2.3 From 2a54898f8009f1511862a75467e84a77bef1943a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:13:57 +0200 Subject: improve spacing in apt_source_list test --- .../test_handler_apt_configure_sources_list.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index d4fdee21..3bcd93cf 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -91,16 +91,12 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): cc_apt_configure.handle("notimportant", cfg, mycloud, LOG, None) - mockisfile.assert_any_call(('/etc/cloud/templates/' - 'sources.list.%s.tmpl' % distro)) - mocktmpl.assert_called_once_with(('/etc/cloud/templates/' - 'sources.list.%s.tmpl' % distro), - '/etc/apt/sources.list', - {'codename': '', - 'primary': - mirrorcheck, - 'mirror': - mirrorcheck}) + mockisfile.assert_any_call( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro)) + mocktmpl.assert_called_once_with( + ('/etc/cloud/templates/sources.list.%s.tmpl' % distro), + '/etc/apt/sources.list', + {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) def test_apt_source_list_debian(self): """ test_apt_source_list_debian -- cgit v1.2.3 From 0a407b63f61f48d310d205f833c1a2792273ec80 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:20:29 +0200 Subject: streamline code and sanitize expected result string definition --- .../test_handler_apt_configure_sources_list.py | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 3bcd93cf..1aa4f553 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -44,6 +44,22 @@ apt_custom_sources_list: | # FIND_SOMETHING_SPECIAL """ +EXPECTED_CONVERTED_CONTENT = ( + """## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +## if you wish to make changes you can: +## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg +## or do the same in user-data +## b.) add sources in /etc/apt/sources.list.d +## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl + +# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to +# newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ xenial main restricted +deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted +# FIND_SOMETHING_SPECIAL +""") + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url @@ -141,17 +157,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): mockwrite.assert_called_once_with( '/etc/apt/sources.list', - ("## Note, this file is written by cloud-init on first boot of an" - " instance\n## modifications made here will not survive a re-bun" - "dle.\n## if you wish to make changes you can:\n## a.) add 'apt_" - "preserve_sources_list: true' to /etc/cloud/cloud.cfg\n## or" - " do the same in user-data\n## b.) add sources in /etc/apt/sourc" - "es.list.d\n## c.) make changes to template file /etc/cloud/temp" - "lates/sources.list.tmpl\n\n# See http://help.ubuntu.com/communi" - "ty/UpgradeNotes for how to upgrade to\n# newer versions of the " - "distribution.\ndeb http://archive.ubuntu.com/ubuntu/ xenial mai" - "n restricted\ndeb-src http://archive.ubuntu.com/ubuntu/ xenial " - "main restricted\n# FIND_SOMETHING_SPECIAL\n"), + EXPECTED_CONVERTED_CONTENT, mode=420) -- cgit v1.2.3 From 454de24c7d457b980c91849b128efe4faee62032 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Thu, 12 May 2016 21:21:36 +0200 Subject: make pep8 happy with a few spaces --- cloudinit/config/cc_apt_configure.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 1 + 2 files changed, 3 insertions(+) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e7b8a9b3..e5a962ac 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -171,6 +171,7 @@ def generate_sources_list(cfg, codename, mirrors, cloud, log): templater.render_to_file(template_fn, '/etc/apt/sources.list', params) + def add_key_raw(key): """ actual adding of a key as defined in key argument @@ -181,6 +182,7 @@ def add_key_raw(key): except: raise Exception('failed add key') + def add_key(ent): """ add key to the system as defiend in ent (if any) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 439bd038..e130392c 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -30,6 +30,7 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== =ACB2 -----END PGP PUBLIC KEY BLOCK-----""" + def load_tfile_or_url(*args, **kwargs): """ load_tfile_or_url load file and return content after decoding -- cgit v1.2.3 From 75211bdfd58b3209455c88ab00c96e9e7c7acecc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 12 May 2016 14:29:50 -0700 Subject: Enable nose-timer and always show top 10 slow tests --- test-requirements.txt | 1 + tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 2ab53707..88a67a26 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ httpretty>=0.7.1 mock nose +nose-timer # Only really needed on older versions of python contextlib2 diff --git a/tox.ini b/tox.ini index dafaaf6d..d051f2ac 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py27,py3,flake8 recreate = True [testenv] -commands = python -m nose {posargs:tests} +commands = python -m nose --with-timer --timer-top-n 10 {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt @@ -19,7 +19,7 @@ setenv = LC_ALL = en_US.utf-8 [testenv:py26] -commands = nosetests {posargs:tests} +commands = nosetests --with-timer --timer-top-n 10 {posargs:tests} deps = contextlib2 httpretty>=0.7.1 -- cgit v1.2.3 From f2665c246a3e6dec55064eced09919d912ae0e52 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 16 May 2016 16:08:19 -0700 Subject: Fix slow tests Timeouts and retries were triggering so make it so that tests do not use the typical timesouts and retries so that the tests finish faster. --- cloudinit/sources/DataSourceOpenStack.py | 12 ++++++--- tests/unittests/test_datasource/test_openstack.py | 32 +++++++++++------------ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 3af17b10..dfd96035 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -103,7 +103,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address = url2base.get(avail_url) return bool(avail_url) - def get_data(self): + def get_data(self, retries=5, timeout=5): try: if not self.wait_for_metadata_service(): return False @@ -115,7 +115,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): 'Crawl of openstack metadata service', read_metadata_service, args=[self.metadata_address], - kwargs={'ssl_details': self.ssl_details}) + kwargs={'ssl_details': self.ssl_details, + 'retries': retries, + 'timeout': timeout}) except openstack.NonReadable: return False except (openstack.BrokenMetadata, IOError): @@ -153,8 +155,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): return sources.instance_id_matches_system_uuid(self.get_instance_id()) -def read_metadata_service(base_url, ssl_details=None): - reader = openstack.MetadataReader(base_url, ssl_details=ssl_details) +def read_metadata_service(base_url, ssl_details=None, + timeout=5, retries=5): + reader = openstack.MetadataReader(base_url, ssl_details=ssl_details, + timeout=timeout, retries=retries) return reader.read_v2() diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 4140d054..5c8592c5 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -135,13 +135,17 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): body=get_request_callback) +def _read_metadata_service(): + return ds.read_metadata_service(BASE_URL, retries=0, timeout=0.1) + + class TestOpenStackDataSource(test_helpers.HttprettyTestCase): VERSION = 'latest' @hp.activate def test_successful(self): _register_uris(self.VERSION, EC2_FILES, EC2_META, OS_FILES) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -163,7 +167,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): @hp.activate def test_no_ec2(self): _register_uris(self.VERSION, {}, {}, OS_FILES) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -178,8 +182,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.NonReadable, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.NonReadable, _read_metadata_service) @hp.activate def test_bad_uuid(self): @@ -190,8 +193,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = json.dumps(os_meta) _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_userdata_empty(self): @@ -200,7 +202,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('user_data'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(VENDOR_DATA, f.get('vendordata')) self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) @@ -213,7 +215,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files.pop(k, None) _register_uris(self.VERSION, {}, {}, os_files) - f = ds.read_metadata_service(BASE_URL) + f = _read_metadata_service() self.assertEqual(CONTENT_0, f['files']['/etc/foo.cfg']) self.assertEqual(CONTENT_1, f['files']['/etc/bar/bar.cfg']) self.assertFalse(f.get('vendordata')) @@ -225,8 +227,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('vendor_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_metadata_invalid(self): @@ -235,8 +236,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): if k.endswith('meta_data.json'): os_files[k] = '{' # some invalid json _register_uris(self.VERSION, {}, {}, os_files) - self.assertRaises(openstack.BrokenMetadata, ds.read_metadata_service, - BASE_URL) + self.assertRaises(openstack.BrokenMetadata, _read_metadata_service) @hp.activate def test_datasource(self): @@ -245,7 +245,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertTrue(found) self.assertEqual(2, ds_os.version) md = dict(ds_os.metadata) @@ -269,7 +269,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): None, helpers.Paths({})) self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -288,7 +288,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) @@ -311,7 +311,7 @@ class TestOpenStackDataSource(test_helpers.HttprettyTestCase): 'timeout': 0, } self.assertIsNone(ds_os.version) - found = ds_os.get_data() + found = ds_os.get_data(timeout=0.1, retries=0) self.assertFalse(found) self.assertIsNone(ds_os.version) -- cgit v1.2.3 From 1df451e76d98371cf54e7a474c4d46fa3a96361a Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 20 May 2016 13:03:42 +0100 Subject: Clean up merging.rst formatting --- doc/merging.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/merging.rst b/doc/merging.rst index d4d5cd05..4046338b 100644 --- a/doc/merging.rst +++ b/doc/merging.rst @@ -19,13 +19,13 @@ For example. run_cmd: - bash1 - bash2 - + #cloud-config (2) run_cmd: - bash3 - bash4 -The previous way of merging the following 2 objects would result in a final +The previous way of merging the following 2 objects would result in a final cloud-config object that contains the following. .. code-block:: yaml @@ -56,7 +56,7 @@ Customizability Since the above merging algorithm may not always be the desired merging algorithm (like how the previous merging algorithm was not always the preferred one) the concept of customizing how merging can be done was introduced through -a new concept call 'merge classes'. +a new concept call 'merge classes'. A merge class is a class defintion which provides functions that can be used to merge a given type with another given type. @@ -69,7 +69,7 @@ An example of one of these merging classes is the following: def __init__(self, merger, opts): self._merger = merger self._overwrite = 'overwrite' in opts - + # This merging algorithm will attempt to merge with # another dictionary, on encountering any other type of object # it will not merge with said object, but will instead return @@ -129,12 +129,12 @@ for your own usage. definition are the following (in order), 'merge_how', 'merge_type'. String format -******** +************* The string format that is expected is the following. :: - + classname1(option1,option2)+classname2(option3,option4).... The class name there will be connected to class names used when looking for the @@ -144,11 +144,11 @@ on construction of that class. For example, the default string that is used when none is provided is the following: :: - + list()+dict()+str() Dictionary format -******** +***************** In cases where a dictionary can be used to specify the same information as the string format (ie option #2 of above) it can be used, for example. @@ -171,7 +171,7 @@ for every cloud-config that I provide, what exactly happens? The answer is that when merging, a stack of 'merging classes' is kept, the first one on that stack is the default merging classes, this set of mergers will be used when the first cloud-config is merged with the initial empty -cloud-config dictionary. If the cloud-config that was just merged provided a +cloud-config dictionary. If the cloud-config that was just merged provided a set of merging classes (via the above formats) then those merging classes will be pushed onto the stack. Now if there is a second cloud-config to be merged then the merging classes from the cloud-config before the first will be used (not the -- cgit v1.2.3 From b699df642dec996c517a94ea44f5e763c08570d0 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Fri, 20 May 2016 13:13:08 +0100 Subject: Update merging documentation to make its application clearer Currently, some users think that user-data and conf.d configuration will be merged together[0]. This has never been the case, and we should make it clear that this is not currently supported to avoid user confusion. [0] https://bugs.launchpad.net/cloud-init/+bug/1532234 --- doc/merging.rst | 24 +++++++++++++++--------- doc/rtd/topics/merging.rst | 6 +++--- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/doc/merging.rst b/doc/merging.rst index 4046338b..2967ca9d 100644 --- a/doc/merging.rst +++ b/doc/merging.rst @@ -1,15 +1,16 @@ Overview -------- -This was done because it has been a common feature request that there be a -way to specify how cloud-config yaml "dictionaries" are merged together when -there are multiple yamls to merge together (say when performing an #include). +This was implemented because it has been a common feature request that there be +a way to specify how cloud-config yaml "dictionaries" provided as user-data are +merged together when there are multiple yamls to merge together (say when +performing an #include). Since previously the merging algorithm was very simple and would only overwrite and not append lists, or strings, and so on it was decided to create a new and improved way to merge dictionaries (and there contained objects) together in a -way that is customizable, thus allowing for users who provide cloud-config data -to determine exactly how there objects will be merged. +way that is customizable, thus allowing for users who provide cloud-config +user-data to determine exactly how there objects will be merged. For example. @@ -181,8 +182,13 @@ cloud-config dictionary coming after it. Other uses ---------- -The default merging algorithm for merging 'conf.d' yaml files (which form a initial -yaml config for cloud-init) was also changed to use this mechanism so its full +In addition to being used for merging user-data sections, the default merging +algorithm for merging 'conf.d' yaml files (which form an initial yaml config +for cloud-init) was also changed to use this mechanism so its full benefits (and customization) can also be used there as well. Other places that -used the previous merging are also similar now extensible (metadata merging for -example). +used the previous merging are also, similarly, now extensible (metadata +merging, for example). + +Note, however, that merge algorithms are not used *across* types of +configuration. As was previously the case, merged user-data will still +overwrite conf.d configuration. diff --git a/doc/rtd/topics/merging.rst b/doc/rtd/topics/merging.rst index 8a03f3c7..2bd87b16 100644 --- a/doc/rtd/topics/merging.rst +++ b/doc/rtd/topics/merging.rst @@ -1,5 +1,5 @@ -========= -Merging -========= +========================== +Merging User-Data Sections +========================== .. include:: ../../merging.rst -- cgit v1.2.3 From 8575c4f7045c7074370bba73198ea36571224ece Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 08:21:59 +0200 Subject: generalize test_apt_source_basic to be reusable across more testcases --- .../test_handler/test_handler_apt_source.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index e130392c..1b294431 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -58,27 +58,34 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params - def test_apt_source_basic(self): - """ test_apt_source_basic + def apt_source_basic(self, filename, cfg): + """ apt_source_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() - cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' - ' karmic-backports' - ' main universe multiverse restricted'), - 'filename': self.aptlistfile} cc_apt_configure.add_sources([cfg], params) - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", "http://archive.ubuntu.com/ubuntu", "karmic-backports", "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) + def test_apt_source_basic(self): + """ test_apt_source_basic + Test Fix deb source string, has to overwrite mirror conf in params. + Test with a filename provided in config. + """ + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + self.apt_source_basic(self.aptlistfile, cfg) + def test_apt_source_replacement(self): """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs -- cgit v1.2.3 From 29065e4a52a747b4f4cf30092ddcc3744e6aa350 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 09:24:44 +0200 Subject: test_apt_source_basic_nofn check for non-specified filename Cloud-inint uses a default fallback, we want to ensure no code change modfies this behaviour. --- .../test_handler/test_handler_apt_source.py | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 1b294431..ce356fc6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -47,6 +47,7 @@ class TestAptSourceConfig(TestCase): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.join = os.path.join @staticmethod def _get_default_params(): @@ -86,6 +87,30 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} self.apt_source_basic(self.aptlistfile, cfg) + def test_apt_source_basic_nofn(self): + """ test_apt_source_basic_nofn + Test Fix deb source string, has to overwrite mirror conf in params. + Test without a filename provided in config and test for known fallback. + """ + cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')} + # mock into writable tmp dir and check path/content there + filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + def myjoin(*args, **kwargs): + """ myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" + and args[1] == "cloud_config_sources.list" + and len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + + with mock.patch.object(os.path, 'join', side_effect=myjoin): + self.apt_source_basic(filename, cfg) + def test_apt_source_replacement(self): """ test_apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs -- cgit v1.2.3 From f06dd57907caa648743a73566b2b6e62b96be2fb Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 09:34:23 +0200 Subject: drop unused mockappsubp --- tests/unittests/test_handler/test_handler_apt_configure_sources_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 1aa4f553..353422a2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -151,7 +151,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: - with mock.patch.object(util, 'subp', self.subp) as mocksubp: + with mock.patch.object(util, 'subp', self.subp): cc_apt_configure.handle("notimportant", cfg, mycloud, LOG, None) -- cgit v1.2.3 From cf37b78e9bada97a7cc223bd824fb0e8cd8c4c7c Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:48:03 +0200 Subject: extend test_apt_source_replace by a no-filename case --- .../test_handler/test_handler_apt_source.py | 53 +++++++++++++++------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index ce356fc6..050ee78f 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -59,6 +59,15 @@ class TestAptSourceConfig(TestCase): params['MIRROR'] = "http://archive.ubuntu.com/ubuntu" return params + def myjoin(self, *args, **kwargs): + """ myjoin - redir into writable tmpdir""" + if (args[0] == "/etc/apt/sources.list.d/" + and args[1] == "cloud_config_sources.list" + and len(args) == 2): + return self.join(self.tmp, args[0].lstrip("/"), args[1]) + else: + return self.join(*args, **kwargs) + def apt_source_basic(self, filename, cfg): """ apt_source_basic Test Fix deb source string, has to overwrite mirror conf in params @@ -99,36 +108,46 @@ class TestAptSourceConfig(TestCase): filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - def myjoin(*args, **kwargs): - """ myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" - and args[1] == "cloud_config_sources.list" - and len(args) == 2): - return self.join(self.tmp, args[0].lstrip("/"), args[1]) - else: - return self.join(*args, **kwargs) - - with mock.patch.object(os.path, 'join', side_effect=myjoin): + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_basic(filename, cfg) - def test_apt_source_replacement(self): - """ test_apt_source_replace + def apt_source_replacement(self, filename, cfg): + """ apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() - cfg = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - cc_apt_configure.add_sources([cfg], params) - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", params['MIRROR'], params['RELEASE'], "multiverse"), contents, flags=re.IGNORECASE)) + def test_apt_source_replace(self): + """ test_apt_source_replace + Test Autoreplacement of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + self.apt_source_replacement(self.aptlistfile, cfg) + + def test_apt_source_replace_nofn(self): + """ test_apt_source_replace_nofn + Test Autoreplacement of MIRROR and RELEASE in source specs with + No filename being set + """ + cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} + # mock into writable tmp dir and check path/content there + filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_source_replacement(filename, cfg) + def test_apt_source_keyid(self): """ test_apt_source_keyid Test specification of a source + keyid -- cgit v1.2.3 From 6b7711dbb75cd3fd31a638f3b530e863eb907708 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:53:16 +0200 Subject: extend test_apt_source_keyid by no filename case --- .../test_handler/test_handler_apt_source.py | 41 ++++++++++++++++------ 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 050ee78f..c4ba6d00 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -148,17 +148,11 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_replacement(filename, cfg) - def test_apt_source_keyid(self): - """ test_apt_source_keyid + def apt_source_keyid(self, filename, cfg): + """ apt_source_keyid Test specification of a source + keyid """ params = self._get_default_params() - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'keyid': "03683F77", - 'filename': self.aptlistfile} with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: @@ -166,9 +160,9 @@ class TestAptSourceConfig(TestCase): mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -176,6 +170,33 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) + def test_apt_source_keyid(self): + """ test_apt_source_keyid + Test specification of a source + keyid with filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + self.apt_source_keyid(self.aptlistfile, cfg) + + def test_apt_source_keyid_nofn(self): + """ test_apt_source_keyid + Test specification of a source + keyid without filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77"} + # mock into writable tmp dir and check path/content there + filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_source_keyid(filename, cfg) + def test_apt_source_key(self): """ test_apt_source_key Test specification of a source + key -- cgit v1.2.3 From 0b2cbda06a5622bc2b6dcc4f2d4190824528f1f6 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:55:37 +0200 Subject: put fallbackfn to init This was now used by multiple methods, no need to duplicate code. --- .../test_handler/test_handler_apt_source.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index c4ba6d00..79c33b4e 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -48,6 +48,10 @@ class TestAptSourceConfig(TestCase): self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") self.join = os.path.join + # mock fallback filename into writable tmp dir + self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", + "cloud_config_sources.list") + @staticmethod def _get_default_params(): @@ -104,12 +108,8 @@ class TestAptSourceConfig(TestCase): cfg = {'source': ('deb http://archive.ubuntu.com/ubuntu' ' karmic-backports' ' main universe multiverse restricted')} - # mock into writable tmp dir and check path/content there - filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_basic(filename, cfg) + self.apt_source_basic(self.fallbackfn, cfg) def apt_source_replacement(self, filename, cfg): """ apt_source_replace @@ -141,12 +141,8 @@ class TestAptSourceConfig(TestCase): No filename being set """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} - # mock into writable tmp dir and check path/content there - filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") - with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_replacement(filename, cfg) + self.apt_source_replacement(self.fallbackfn, cfg) def apt_source_keyid(self, filename, cfg): """ apt_source_keyid @@ -191,11 +187,8 @@ class TestAptSourceConfig(TestCase): 'smoser/cloud-init-test/ubuntu' ' xenial main'), 'keyid': "03683F77"} - # mock into writable tmp dir and check path/content there - filename = os.path.join(self.tmp, "etc/apt/sources.list.d/", - "cloud_config_sources.list") with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_keyid(filename, cfg) + self.apt_source_keyid(self.fallbackfn, cfg) def test_apt_source_key(self): """ test_apt_source_key -- cgit v1.2.3 From 1926eb2476f9e1fda3356c7828479231dccc309b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 10:59:45 +0200 Subject: extend test_apt_source_key by nofn case --- .../test_handler/test_handler_apt_source.py | 38 ++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 79c33b4e..516dc694 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -190,26 +190,20 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_keyid(self.fallbackfn, cfg) - def test_apt_source_key(self): - """ test_apt_source_key + def apt_source_key(self, filename, cfg): + """ apt_source_key Test specification of a source + key """ params = self._get_default_params() - cfg = {'source': ('deb ' - 'http://ppa.launchpad.net/' - 'smoser/cloud-init-test/ubuntu' - ' xenial main'), - 'key': "fakekey 4321", - 'filename': self.aptlistfile} with mock.patch.object(util, 'subp') as mockobj: cc_apt_configure.add_sources([cfg], params) mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 4321') - self.assertTrue(os.path.isfile(self.aptlistfile)) + self.assertTrue(os.path.isfile(filename)) - contents = load_tfile_or_url(self.aptlistfile) + contents = load_tfile_or_url(filename) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", ('http://ppa.launchpad.net/smoser/' @@ -217,6 +211,30 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) + def test_apt_source_key(self): + """ test_apt_source_key + Test specification of a source + key with filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321", + 'filename': self.aptlistfile} + self.apt_source_key(self.aptlistfile, cfg) + + def test_apt_source_key_nofn(self): + """ test_apt_source_key + Test specification of a source + key without filename being set + """ + cfg = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'key': "fakekey 4321"} + with mock.patch.object(os.path, 'join', side_effect=self.myjoin): + self.apt_source_key(self.fallbackfn, cfg) + def test_apt_source_keyonly(self): """ test_apt_source_keyonly Test specification key without source -- cgit v1.2.3 From ee239517c5342cbd62c9fdeaf735d78d6fd1fbb8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 11:59:14 +0200 Subject: support apt_sources to be a dictionary key is the filename, and "old" input shall be handled as it was all the time. For compatibility this will (continue to) overwrite the file of multiple options that did not specify an output file (they all get the same default). Yet it will process them all - as it always did - e.g. to add the keys of all of them. Any users of the new format won't have these issues, as they will always have a key. --- cloudinit/config/cc_apt_configure.py | 34 +++++++++++++++++++++++++--------- cloudinit/util.py | 8 ++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index e5a962ac..a46ebb3e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,8 +215,28 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + # convert old list format to new dict based format + if isinstance(srclist, list): + srcdict = {} + for srcent in srclist: + if 'filename' not in srcent: + # file collides for multiple !filename cases for compatibility + # yet we need them all processed, so not same dictionary key + srcent['filename'] = "cloud_config_sources.list" + key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + else: + # all with filename use that as key (matching new format) + key = srcent['filename'] + srcdict[key] = srcent + else: + srcdict = srclist + errorlist = [] - for ent in srclist: + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent[filename] = filename + # keys can be added without specifying a source try: add_key(ent) @@ -226,10 +246,13 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): if 'source' not in ent: errorlist.append(["", "missing source"]) continue - source = ent['source'] source = templater.render_string(source, template_params) + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if aa_repo_match(source): try: util.subp(["add-apt-repository", source]) @@ -238,13 +261,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): ("add-apt-repository failed. " + str(e))]) continue - if 'filename' not in ent: - ent['filename'] = 'cloud_config_sources.list' - - if not ent['filename'].startswith("/"): - ent['filename'] = os.path.join("/etc/apt/sources.list.d/", - ent['filename']) - try: contents = "%s\n" % (source) util.write_file(ent['filename'], contents, omode="ab") diff --git a/cloudinit/util.py b/cloudinit/util.py index 0d21e11b..2931efbd 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,14 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) +def rand_dict_key(dictionary, postfix=None): + if not postfix: + postfix = "" + while True: + newkey = rand_str(strlen=8) + "_" + postfix + if newkey not in dictionary: + break + return newkey def read_conf(fname): try: -- cgit v1.2.3 From a33f8c09863381006f708a1e9d49997ed9f7befa Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 12:12:26 +0200 Subject: warn about multiple colliding apt_source without filenames --- cloudinit/config/cc_apt_configure.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a46ebb3e..327b543e 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -215,15 +215,22 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): def aa_repo_match(x): return False + errorlist = [] # convert old list format to new dict based format if isinstance(srclist, list): srcdict = {} + fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") + if fnfallbackused is not None: + errorlist.append(["multiple apt_source entries without", + "filename will conflict: %s vs %s" % + (srcent, fnfallbackused)]) + fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -231,7 +238,6 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: srcdict = srclist - errorlist = [] for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: -- cgit v1.2.3 From 3560afc50569169ceddf95ff755d231fd5858143 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 13:34:27 +0200 Subject: fix function names in inline doc --- tests/unittests/test_handler/test_handler_apt_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 516dc694..6441374d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -179,7 +179,7 @@ class TestAptSourceConfig(TestCase): self.apt_source_keyid(self.aptlistfile, cfg) def test_apt_source_keyid_nofn(self): - """ test_apt_source_keyid + """ test_apt_source_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -224,7 +224,7 @@ class TestAptSourceConfig(TestCase): self.apt_source_key(self.aptlistfile, cfg) def test_apt_source_key_nofn(self): - """ test_apt_source_key + """ test_apt_source_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' -- cgit v1.2.3 From db5188ca3b9eac1097735ce42807881266d29543 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 13:57:17 +0200 Subject: testcases with multiple source list entries --- .../test_handler/test_handler_apt_source.py | 74 ++++++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 6441374d..dcbd51e6 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -47,6 +47,8 @@ class TestAptSourceConfig(TestCase): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp) self.aptlistfile = os.path.join(self.tmp, "single-deb.list") + self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list") + self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list") self.join = os.path.join # mock fallback filename into writable tmp dir self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", @@ -78,7 +80,7 @@ class TestAptSourceConfig(TestCase): """ params = self._get_default_params() - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -98,7 +100,40 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile} - self.apt_source_basic(self.aptlistfile, cfg) + self.apt_source_basic(self.aptlistfile, [cfg]) + + def test_apt_source_basic_triple(self): + """ test_apt_source_basic_triple + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + """ + cfg1 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted'), + 'filename': self.aptlistfile3} + self.apt_source_basic(self.aptlistfile, [cfg1, cfg2, cfg3]) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) def test_apt_source_basic_nofn(self): """ test_apt_source_basic_nofn @@ -109,14 +144,14 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted')} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_basic(self.fallbackfn, cfg) + self.apt_source_basic(self.fallbackfn, [cfg]) def apt_source_replacement(self, filename, cfg): """ apt_source_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_sources(cfg, params) self.assertTrue(os.path.isfile(filename)) @@ -133,7 +168,34 @@ class TestAptSourceConfig(TestCase): """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse', 'filename': self.aptlistfile} - self.apt_source_replacement(self.aptlistfile, cfg) + self.apt_source_replacement(self.aptlistfile, [cfg]) + + def test_apt_source_replace_triple(self): + """ test_apt_source_replace_triple + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_source_replacement(self.aptlistfile, [cfg1, cfg2, cfg3]) + + # extra verify on two extra files of this test + params = self._get_default_params() + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "main"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", params['MIRROR'], params['RELEASE'], + "universe"), + contents, flags=re.IGNORECASE)) + def test_apt_source_replace_nofn(self): """ test_apt_source_replace_nofn @@ -142,7 +204,7 @@ class TestAptSourceConfig(TestCase): """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_replacement(self.fallbackfn, cfg) + self.apt_source_replacement(self.fallbackfn, [cfg]) def apt_source_keyid(self, filename, cfg): """ apt_source_keyid -- cgit v1.2.3 From a62d3ddcf6ec06ccf0f3c2dd0fd969c83304d89b Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 14:13:18 +0200 Subject: add triple case for test_apt_source_keyid_triple incl triple key check --- .../test_handler/test_handler_apt_source.py | 55 ++++++++++++++++++++-- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index dcbd51e6..9edd3df5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -10,6 +10,7 @@ try: from unittest import mock except ImportError: import mock +from mock import call from cloudinit.config import cc_apt_configure from cloudinit import util @@ -206,7 +207,7 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(os.path, 'join', side_effect=self.myjoin): self.apt_source_replacement(self.fallbackfn, [cfg]) - def apt_source_keyid(self, filename, cfg): + def apt_source_keyid(self, filename, cfg, keynum): """ apt_source_keyid Test specification of a source + keyid """ @@ -214,9 +215,13 @@ class TestAptSourceConfig(TestCase): with mock.patch.object(util, 'subp', return_value=('fakekey 1234', '')) as mockobj: - cc_apt_configure.add_sources([cfg], params) + cc_apt_configure.add_sources(cfg, params) - mockobj.assert_called_with(('apt-key', 'add', '-'), 'fakekey 1234') + # check if it added the right ammount of keys + calls = [] + for i in range(keynum): + calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) + mockobj.assert_has_calls(calls, any_order=True) self.assertTrue(os.path.isfile(filename)) @@ -238,7 +243,47 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77", 'filename': self.aptlistfile} - self.apt_source_keyid(self.aptlistfile, cfg) + self.apt_source_keyid(self.aptlistfile, [cfg], 1) + + def test_apt_source_keyid_triple(self): + """ test_apt_source_keyid_triple + Test specification of a source + keyid with filename being set + Setting three of such, check for content and keys + """ + cfg1 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial main'), + 'keyid': "03683F77", + 'filename': self.aptlistfile} + cfg2 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial universe'), + 'keyid': "03683F77", + 'filename': self.aptlistfile2} + cfg3 = {'source': ('deb ' + 'http://ppa.launchpad.net/' + 'smoser/cloud-init-test/ubuntu' + ' xenial multiverse'), + 'keyid': "03683F77", + 'filename': self.aptlistfile3} + + self.apt_source_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "universe"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", + ('http://ppa.launchpad.net/smoser/' + 'cloud-init-test/ubuntu'), + "xenial", "multiverse"), + contents, flags=re.IGNORECASE)) def test_apt_source_keyid_nofn(self): """ test_apt_source_keyid_nofn @@ -250,7 +295,7 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77"} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_keyid(self.fallbackfn, cfg) + self.apt_source_keyid(self.fallbackfn, [cfg], 1) def apt_source_key(self, filename, cfg): """ apt_source_key -- cgit v1.2.3 From 7a141721c5bb0ba2e65191c514a15ff01220ebca Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 14:14:44 +0200 Subject: make checkers happy about unused loop index --- tests/unittests/test_handler/test_handler_apt_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 9edd3df5..3056e9d0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -219,7 +219,7 @@ class TestAptSourceConfig(TestCase): # check if it added the right ammount of keys calls = [] - for i in range(keynum): + for _ in range(keynum): calls.append(call(('apt-key', 'add', '-'), 'fakekey 1234')) mockobj.assert_has_calls(calls, any_order=True) -- cgit v1.2.3 From 72ff44b4517eacb4f525e1bf7db6787607ff306a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 14:21:23 +0200 Subject: add triple test for ppa adding --- .../test_handler/test_handler_apt_source.py | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 3056e9d0..76099035 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -431,5 +431,32 @@ class TestAptSourceConfig(TestCase): # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) + def test_apt_source_ppa_triple(self): + """ test_apt_source_ppa_triple + Test specification of a ppa + """ + params = self._get_default_params() + cfg1 = {'source': 'ppa:smoser/cloud-init-test', + 'filename': self.aptlistfile} + cfg2 = {'source': 'ppa:smoser/cloud-init-test2', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'ppa:smoser/cloud-init-test3', + 'filename': self.aptlistfile3} + + # default matcher needed for ppa + matcher = re.compile(r'^[\w-]+:\w').search + + with mock.patch.object(util, 'subp') as mockobj: + cc_apt_configure.add_sources([cfg1, cfg2, cfg3], params, + aa_repo_match=matcher) + calls = [call(['add-apt-repository', 'ppa:smoser/cloud-init-test']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test2']), + call(['add-apt-repository', 'ppa:smoser/cloud-init-test3'])] + mockobj.assert_has_calls(calls, any_order=True) + + # adding ppa should ignore all filenames (uses add-apt-repository) + self.assertFalse(os.path.isfile(self.aptlistfile)) + self.assertFalse(os.path.isfile(self.aptlistfile2)) + self.assertFalse(os.path.isfile(self.aptlistfile3)) # vi: ts=4 expandtab -- cgit v1.2.3 From 4ed5251d17ee7a44ce12d38d9b3d4fa554279419 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:06:48 +0200 Subject: fix issue with dictionary style apt_sources handling filenames --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 327b543e..a25d6af1 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -241,7 +241,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: - ent[filename] = filename + ent['filename'] = filename # keys can be added without specifying a source try: -- cgit v1.2.3 From d1887b6fb38d378bb35298fadd6bea729952f3a0 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:07:37 +0200 Subject: add test_apt_source_basic_dict This is the basic testcase but in the new dictionary format --- tests/unittests/test_handler/test_handler_apt_source.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 76099035..d2370e93 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -103,6 +103,18 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile} self.apt_source_basic(self.aptlistfile, [cfg]) + def test_apt_source_basic_dict(self): + """ test_apt_source_basic_dict + Test Fix deb source string, has to overwrite mirror conf in params. + Test with a filename provided in config. + Provided in a dictionary with filename being the key (new format) + """ + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}} + self.apt_source_basic(self.aptlistfile, cfg) + def test_apt_source_basic_triple(self): """ test_apt_source_basic_triple Test Fix three deb source string, has to overwrite mirror conf in -- cgit v1.2.3 From e4968621c802062910d964c2d9c897d7124861b0 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 15:21:55 +0200 Subject: unify basic triple check and add test_apt_src_basic_dict_triple based on it --- .../test_handler/test_handler_apt_source.py | 57 ++++++++++++++++------ 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index d2370e93..f19a78b1 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -115,6 +115,29 @@ class TestAptSourceConfig(TestCase): ' main universe multiverse restricted')}} self.apt_source_basic(self.aptlistfile, cfg) + def apt_source_basic_triple(self, cfg): + """ apt_source_basic_triple + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + generic part to check three files with different content + """ + self.apt_source_basic(self.aptlistfile, cfg) + + # extra verify on two extra files of this test + contents = load_tfile_or_url(self.aptlistfile2) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "precise-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + contents = load_tfile_or_url(self.aptlistfile3) + self.assertTrue(re.search(r"%s %s %s %s\n" % + ("deb", "http://archive.ubuntu.com/ubuntu", + "lucid-backports", + "main universe multiverse restricted"), + contents, flags=re.IGNORECASE)) + + def test_apt_source_basic_triple(self): """ test_apt_source_basic_triple Test Fix three deb source string, has to overwrite mirror conf in @@ -132,21 +155,27 @@ class TestAptSourceConfig(TestCase): ' lucid-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile3} - self.apt_source_basic(self.aptlistfile, [cfg1, cfg2, cfg3]) + self.apt_source_basic_triple([cfg1, cfg2, cfg3]) - # extra verify on two extra files of this test - contents = load_tfile_or_url(self.aptlistfile2) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "precise-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) - contents = load_tfile_or_url(self.aptlistfile3) - self.assertTrue(re.search(r"%s %s %s %s\n" % - ("deb", "http://archive.ubuntu.com/ubuntu", - "lucid-backports", - "main universe multiverse restricted"), - contents, flags=re.IGNORECASE)) + def test_apt_src_basic_dict_triple(self): + """ test_apt_src_basic_dict_triple + Test Fix three deb source string, has to overwrite mirror conf in + params. Test with filenames provided in config. + Provided in a dictionary with filename being the key (new format) + """ + cfg = {self.aptlistfile: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' karmic-backports' + ' main universe multiverse restricted')}, + self.aptlistfile2: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' precise-backports' + ' main universe multiverse restricted')}, + self.aptlistfile3: {'source': + ('deb http://archive.ubuntu.com/ubuntu' + ' lucid-backports' + ' main universe multiverse restricted')}} + self.apt_source_basic_triple(cfg) def test_apt_source_basic_nofn(self): """ test_apt_source_basic_nofn -- cgit v1.2.3 From 2945e028477ddb031d9a51ada16d5b992380242a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:12:18 +0200 Subject: make sure we only handle list or dict apt_sources and bail out for others --- cloudinit/config/cc_apt_configure.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index a25d6af1..dd199471 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -217,8 +217,8 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): errorlist = [] # convert old list format to new dict based format + srcdict = {} if isinstance(srclist, list): - srcdict = {} fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: @@ -235,8 +235,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): # all with filename use that as key (matching new format) key = srcent['filename'] srcdict[key] = srcent - else: + elif isinstance(srclist, dict): srcdict = srclist + else: + errorlist.append(["srclist", "unknown apt_sources format"]) for filename in srcdict: ent = srcdict[filename] -- cgit v1.2.3 From 299e3dd9b3769101fa694d7e96895bb89e0375ca Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:13:45 +0200 Subject: shorten method names to follow python rules --- .../test_handler/test_handler_apt_source.py | 145 +++++++++++---------- 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index f19a78b1..9a2cc268 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -75,8 +75,8 @@ class TestAptSourceConfig(TestCase): else: return self.join(*args, **kwargs) - def apt_source_basic(self, filename, cfg): - """ apt_source_basic + def apt_src_basic(self, filename, cfg): + """ apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() @@ -92,8 +92,8 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_source_basic(self): - """ test_apt_source_basic + def test_apt_src_basic(self): + """ test_apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. """ @@ -101,10 +101,10 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile} - self.apt_source_basic(self.aptlistfile, [cfg]) + self.apt_src_basic(self.aptlistfile, [cfg]) - def test_apt_source_basic_dict(self): - """ test_apt_source_basic_dict + def test_apt_src_basic_dict(self): + """ test_apt_src_basic_dict Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. Provided in a dictionary with filename being the key (new format) @@ -113,15 +113,15 @@ class TestAptSourceConfig(TestCase): ('deb http://archive.ubuntu.com/ubuntu' ' karmic-backports' ' main universe multiverse restricted')}} - self.apt_source_basic(self.aptlistfile, cfg) + self.apt_src_basic(self.aptlistfile, cfg) - def apt_source_basic_triple(self, cfg): - """ apt_source_basic_triple + def apt_src_basic_tri(self, cfg): + """ apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. generic part to check three files with different content """ - self.apt_source_basic(self.aptlistfile, cfg) + self.apt_src_basic(self.aptlistfile, cfg) # extra verify on two extra files of this test contents = load_tfile_or_url(self.aptlistfile2) @@ -138,8 +138,8 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) - def test_apt_source_basic_triple(self): - """ test_apt_source_basic_triple + def test_apt_src_basic_tri(self): + """ test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. """ @@ -155,10 +155,10 @@ class TestAptSourceConfig(TestCase): ' lucid-backports' ' main universe multiverse restricted'), 'filename': self.aptlistfile3} - self.apt_source_basic_triple([cfg1, cfg2, cfg3]) + self.apt_src_basic_tri([cfg1, cfg2, cfg3]) - def test_apt_src_basic_dict_triple(self): - """ test_apt_src_basic_dict_triple + def test_apt_src_basic_dict_tri(self): + """ test_apt_src_basic_dict_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. Provided in a dictionary with filename being the key (new format) @@ -175,10 +175,10 @@ class TestAptSourceConfig(TestCase): ('deb http://archive.ubuntu.com/ubuntu' ' lucid-backports' ' main universe multiverse restricted')}} - self.apt_source_basic_triple(cfg) + self.apt_src_basic_tri(cfg) - def test_apt_source_basic_nofn(self): - """ test_apt_source_basic_nofn + def test_apt_src_basic_nofn(self): + """ test_apt_src_basic_nofn Test Fix deb source string, has to overwrite mirror conf in params. Test without a filename provided in config and test for known fallback. """ @@ -186,10 +186,10 @@ class TestAptSourceConfig(TestCase): ' karmic-backports' ' main universe multiverse restricted')} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_basic(self.fallbackfn, [cfg]) + self.apt_src_basic(self.fallbackfn, [cfg]) - def apt_source_replacement(self, filename, cfg): - """ apt_source_replace + def apt_src_replacement(self, filename, cfg): + """ apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -203,27 +203,21 @@ class TestAptSourceConfig(TestCase): "multiverse"), contents, flags=re.IGNORECASE)) - def test_apt_source_replace(self): - """ test_apt_source_replace + def test_apt_src_replace(self): + """ test_apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs with Filename being set """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse', 'filename': self.aptlistfile} - self.apt_source_replacement(self.aptlistfile, [cfg]) + self.apt_src_replacement(self.aptlistfile, [cfg]) - def test_apt_source_replace_triple(self): - """ test_apt_source_replace_triple + def apt_src_replace_tri(self, cfg): + """ apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with - Filename being set + generic part """ - cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', - 'filename': self.aptlistfile} - cfg2 = {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2} - cfg3 = {'source': 'deb $MIRROR $RELEASE universe', - 'filename': self.aptlistfile3} - self.apt_source_replacement(self.aptlistfile, [cfg1, cfg2, cfg3]) + self.apt_src_replacement(self.aptlistfile, cfg) # extra verify on two extra files of this test params = self._get_default_params() @@ -239,17 +233,30 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) - def test_apt_source_replace_nofn(self): - """ test_apt_source_replace_nofn + def test_apt_src_replace_tri(self): + """ test_apt_src_replace_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + + def test_apt_src_replace_nofn(self): + """ test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with No filename being set """ cfg = {'source': 'deb $MIRROR $RELEASE multiverse'} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_replacement(self.fallbackfn, [cfg]) + self.apt_src_replacement(self.fallbackfn, [cfg]) - def apt_source_keyid(self, filename, cfg, keynum): - """ apt_source_keyid + def apt_src_keyid(self, filename, cfg, keynum): + """ apt_src_keyid Test specification of a source + keyid """ params = self._get_default_params() @@ -274,8 +281,8 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) - def test_apt_source_keyid(self): - """ test_apt_source_keyid + def test_apt_src_keyid(self): + """ test_apt_src_keyid Test specification of a source + keyid with filename being set """ cfg = {'source': ('deb ' @@ -284,10 +291,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77", 'filename': self.aptlistfile} - self.apt_source_keyid(self.aptlistfile, [cfg], 1) + self.apt_src_keyid(self.aptlistfile, [cfg], 1) - def test_apt_source_keyid_triple(self): - """ test_apt_source_keyid_triple + def test_apt_src_keyid_tri(self): + """ test_apt_src_keyid_tri Test specification of a source + keyid with filename being set Setting three of such, check for content and keys """ @@ -310,7 +317,7 @@ class TestAptSourceConfig(TestCase): 'keyid': "03683F77", 'filename': self.aptlistfile3} - self.apt_source_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) + self.apt_src_keyid(self.aptlistfile, [cfg1, cfg2, cfg3], 3) contents = load_tfile_or_url(self.aptlistfile2) self.assertTrue(re.search(r"%s %s %s %s\n" % ("deb", @@ -326,8 +333,8 @@ class TestAptSourceConfig(TestCase): "xenial", "multiverse"), contents, flags=re.IGNORECASE)) - def test_apt_source_keyid_nofn(self): - """ test_apt_source_keyid_nofn + def test_apt_src_keyid_nofn(self): + """ test_apt_src_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -336,10 +343,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'keyid': "03683F77"} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_keyid(self.fallbackfn, [cfg], 1) + self.apt_src_keyid(self.fallbackfn, [cfg], 1) - def apt_source_key(self, filename, cfg): - """ apt_source_key + def apt_src_key(self, filename, cfg): + """ apt_src_key Test specification of a source + key """ params = self._get_default_params() @@ -359,8 +366,8 @@ class TestAptSourceConfig(TestCase): "xenial", "main"), contents, flags=re.IGNORECASE)) - def test_apt_source_key(self): - """ test_apt_source_key + def test_apt_src_key(self): + """ test_apt_src_key Test specification of a source + key with filename being set """ cfg = {'source': ('deb ' @@ -369,10 +376,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'key': "fakekey 4321", 'filename': self.aptlistfile} - self.apt_source_key(self.aptlistfile, cfg) + self.apt_src_key(self.aptlistfile, cfg) - def test_apt_source_key_nofn(self): - """ test_apt_source_key_nofn + def test_apt_src_key_nofn(self): + """ test_apt_src_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' @@ -381,10 +388,10 @@ class TestAptSourceConfig(TestCase): ' xenial main'), 'key': "fakekey 4321"} with mock.patch.object(os.path, 'join', side_effect=self.myjoin): - self.apt_source_key(self.fallbackfn, cfg) + self.apt_src_key(self.fallbackfn, cfg) - def test_apt_source_keyonly(self): - """ test_apt_source_keyonly + def test_apt_src_keyonly(self): + """ test_apt_src_keyonly Test specification key without source """ params = self._get_default_params() @@ -400,8 +407,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_keyidonly(self): - """ test_apt_source_keyidonly + def test_apt_src_keyidonly(self): + """ test_apt_src_keyidonly Test specification of a keyid without source """ params = self._get_default_params() @@ -417,8 +424,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_keyid_real(self): - """ test_apt_source_keyid_real + def test_apt_src_keyid_real(self): + """ test_apt_src_keyid_real Test specification of a keyid without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -435,8 +442,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_longkeyid_real(self): - """ test_apt_source_keyid_real + def test_apt_src_longkeyid_real(self): + """ test_apt_src_keyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -453,8 +460,8 @@ class TestAptSourceConfig(TestCase): # filename should be ignored on key only self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_ppa(self): - """ test_apt_source_ppa + def test_apt_src_ppa(self): + """ test_apt_src_ppa Test specification of a ppa """ params = self._get_default_params() @@ -472,8 +479,8 @@ class TestAptSourceConfig(TestCase): # adding ppa should ignore filename (uses add-apt-repository) self.assertFalse(os.path.isfile(self.aptlistfile)) - def test_apt_source_ppa_triple(self): - """ test_apt_source_ppa_triple + def test_apt_src_ppa_tri(self): + """ test_apt_src_ppa_tri Test specification of a ppa """ params = self._get_default_params() -- cgit v1.2.3 From 23a19bde96d895e97c5fb6dbbe50620fb1130553 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:14:04 +0200 Subject: add test_apt_src_replace_dict_tri This includes a test for the weird but valid case in the new dictionary syntax that one sets a key (which is the filename) but overwrites the filename value inside of it. --- tests/unittests/test_handler/test_handler_apt_source.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 9a2cc268..237cf14d 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -246,6 +246,20 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile3} self.apt_src_replace_tri([cfg1, cfg2, cfg3]) + def test_apt_src_replace_dict_tri(self): + """ test_apt_src_replace_dict_tri + Test three autoreplacements of MIRROR and RELEASE in source specs with + Filename being set + Provided in a dictionary with filename being the key (new format) + We also test a new special conditions of the new format that allows + filenames to be overwritten inside the directory entry. + """ + cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, + self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} + self.apt_src_replace_tri(cfg) + def test_apt_src_replace_nofn(self): """ test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with -- cgit v1.2.3 From 65ad82bec66ea3379a20785b1932ed1dc3c17b67 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:46:30 +0200 Subject: modify cloud-config examples to match the new apt_source format --- doc/examples/cloud-config.txt | 76 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 31b791b6..50c6d282 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -106,10 +106,52 @@ apt_custom_sources_list: | # expression will be passed to add-apt-repository add_apt_repo_match: '^[\w-]+:\w' +# 'apt_sources' is a dictionary +# The key is the filename and will be prepended by /etc/apt/sources.list.d/ if +# it doesn't start with a '/'. +# There are certain cases - where no content is written into a source.list file +# where the filename will be ignored - yet it can still be used as index for +# merging. +# The value it maps to is a dictionary with the following optional entries: +# 'source': a sources.list entry (some variable replacements apply) +# 'keyid': providing a key to import via shortid or fingerprint +# 'key': providing a raw PGP key +# 'filename': for compatibility with the older format (now the key to this +# dictionary is the filename). If specified this overwrites the +# filename given as key. + +# the new "filename: {specification-dictionary}, filename2: ..." format allows +# better merging between multiple input files than a list like: +# cloud-config1 +# sources: + s1: {'key': 'key1', 'source': 'source1'} +# cloud-config2 +# sources: + s2: {'key': 'key2'} + s1: {filename: 'foo'} +# this would be merged to +#sources: +# s1: +# filename: foo +# key: key1 +# source: source1 +# s2: +# key: key2 +# Be aware that this style of merging is not the default (for backward +# compatibility reasons). You should specify the following merge_how to get +# this more complete and modern merging behaviour: +# merge_how: "list()+dict()+str()" +# This would then also be equivalent to the config merging used in curtin +# (https://launchpad.net/curtin). + +# for more details see below in the various examples + apt_sources: - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + byobu-ppa.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server - filename: byobu-ppa.list + # adding a source.list line, importing a gpg key for a given key id and + # storing it in the file /etc/apt/sources.list.d/byobu-ppa.list # PPA shortcut: # * Setup correct apt sources.list line @@ -117,7 +159,9 @@ apt_sources: # # See https://help.launchpad.net/Packaging/PPA for more information # this requires 'add-apt-repository' - - source: "ppa:smoser/ppa" # Quote the string + # due to that the filename key is ignored in this case + ignored1: + source: "ppa:smoser/ppa" # Quote the string # Custom apt repository: # * all that is required is 'source' @@ -128,31 +172,39 @@ apt_sources: # + filename: cloud_config_sources.list # # See sources.list man page for more information about the format - - source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted + my-repo.list: + source: deb http://archive.ubuntu.com/ubuntu karmic-backports main universe multiverse restricted # sources can use $MIRROR and $RELEASE and they will be replaced # with the local mirror for this cloud, and the running release # the entry below would be possibly turned into: - # - source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse - - source: deb $MIRROR $RELEASE multiverse + # source: deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu natty multiverse + my-repo.list: + source: deb $MIRROR $RELEASE multiverse # this would have the same end effect as 'ppa:byobu/ppa' - - source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" + my-repo.list: + source: "deb http://ppa.launchpad.net/byobu/ppa/ubuntu karmic main" keyid: F430BBA5 # GPG key ID published on a key server filename: byobu-ppa.list # this would only import the key without adding a ppa or other source spec - - keyid: F430BBA5 # GPG key ID published on a key server + # since this doesn't generate a source.list file the filename key is ignored + ignored2: + keyid: F430BBA5 # GPG key ID published on a key server # In general keyid's can also be specified via their long fingerprints - - keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 + # since this doesn't generate a source.list file the filename key is ignored + ignored3: + keyid: B59D 5F15 97A5 04B7 E230 6DCA 0620 BBCF 0368 3F77 # Custom apt repository: # * The apt signing key can also be specified # by providing a pgp public key block # * Providing the PGP key here is the most robust method for # specifying a key, as it removes dependency on a remote key server - - source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main + my-repo.list: + source: deb http://ppa.launchpad.net/alestic/ppa/ubuntu karmic main key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.0.10 @@ -170,7 +222,9 @@ apt_sources: # Custom gpg key: # * As the keyid also a key can be specified withut a related source # * all other facts mentioned above still apply - - key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- + # since this doesn't generate a source.list file the filename key is ignored + ignored4: + key: | # The value needs to start with -----BEGIN PGP PUBLIC KEY BLOCK----- -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.0.10 -- cgit v1.2.3 From a63a64a70def97730d2ab544b0df9f87f3484333 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 23 May 2016 16:53:03 +0200 Subject: final pep8 check fixups --- cloudinit/util.py | 2 ++ tests/unittests/test_handler/test_handler_apt_source.py | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 2931efbd..0773af69 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -335,6 +335,7 @@ def rand_str(strlen=32, select_from=None): select_from = string.ascii_letters + string.digits return "".join([random.choice(select_from) for _x in range(0, strlen)]) + def rand_dict_key(dictionary, postfix=None): if not postfix: postfix = "" @@ -344,6 +345,7 @@ def rand_dict_key(dictionary, postfix=None): break return newkey + def read_conf(fname): try: return load_yaml(load_file(fname), default={}) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 237cf14d..c19904fb 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -55,7 +55,6 @@ class TestAptSourceConfig(TestCase): self.fallbackfn = os.path.join(self.tmp, "etc/apt/sources.list.d/", "cloud_config_sources.list") - @staticmethod def _get_default_params(): """ get_default_params @@ -68,9 +67,9 @@ class TestAptSourceConfig(TestCase): def myjoin(self, *args, **kwargs): """ myjoin - redir into writable tmpdir""" - if (args[0] == "/etc/apt/sources.list.d/" - and args[1] == "cloud_config_sources.list" - and len(args) == 2): + if (args[0] == "/etc/apt/sources.list.d/" and + args[1] == "cloud_config_sources.list" and + len(args) == 2): return self.join(self.tmp, args[0].lstrip("/"), args[1]) else: return self.join(*args, **kwargs) @@ -137,7 +136,6 @@ class TestAptSourceConfig(TestCase): "main universe multiverse restricted"), contents, flags=re.IGNORECASE)) - def test_apt_src_basic_tri(self): """ test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in @@ -232,7 +230,6 @@ class TestAptSourceConfig(TestCase): "universe"), contents, flags=re.IGNORECASE)) - def test_apt_src_replace_tri(self): """ test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with -- cgit v1.2.3 From 1b418ef2db337e9f8bea7462f18f58ee41dea3b2 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 16:27:24 +0200 Subject: fix typo in examples doc --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 50c6d282..60457093 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -72,7 +72,7 @@ apt_pipelining: False # then apt_mirror above will have no effect apt_preserve_sources_list: true -# Provide a custom template for rednering sources.list +# Provide a custom template for rendering sources.list # Default: a default template for Ubuntu/Debain will be used as packaged in # Ubuntu: /etc/cloud/templates/sources.list.ubuntu.tmpl # Debian: /etc/cloud/templates/sources.list.debian.tmpl -- cgit v1.2.3 From 14040a9c8df6e8406acb79fd653873bb05cb4d40 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 16:29:31 +0200 Subject: improve examples of ap_source --- doc/examples/cloud-config.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index 60457093..df59ff57 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -113,10 +113,11 @@ add_apt_repo_match: '^[\w-]+:\w' # where the filename will be ignored - yet it can still be used as index for # merging. # The value it maps to is a dictionary with the following optional entries: -# 'source': a sources.list entry (some variable replacements apply) -# 'keyid': providing a key to import via shortid or fingerprint -# 'key': providing a raw PGP key -# 'filename': for compatibility with the older format (now the key to this +# source: a sources.list entry (some variable replacements apply) +# keyid: providing a key to import via shortid or fingerprint +# key: providing a raw PGP key +# keyserver: keyserver to fetch keys from, default is keyserver.ubuntu.com +# filename: for compatibility with the older format (now the key to this # dictionary is the filename). If specified this overwrites the # filename given as key. @@ -169,7 +170,6 @@ apt_sources: # * [optional] Import the apt signing key from the keyserver # * Defaults: # + keyserver: keyserver.ubuntu.com - # + filename: cloud_config_sources.list # # See sources.list man page for more information about the format my-repo.list: -- cgit v1.2.3 From 2a20382249c35f368f931dc2bcd8225ea1fc4e84 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 18:55:35 +0200 Subject: integrate further smaller review feedback --- cloudinit/config/cc_apt_configure.py | 52 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 8f22c446..02b16336 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -21,7 +21,6 @@ import glob import os import re -import tempfile from cloudinit import templater from cloudinit import util @@ -179,44 +178,29 @@ def add_key_raw(key): """ try: util.subp(('apt-key', 'add', '-'), key) - except: + except util.ProcessExecutionError: raise Exception('failed add key') def add_key(ent): """ - add key to the system as defiend in ent (if any) - suppords raw keys or keyid's - The latter will as a first step fetched to get the raw key + add key to the system as defined in ent (if any) + supports raw keys or keyid's + The latter will as a first step fetch the raw key from a keyserver """ if 'keyid' in ent and 'key' not in ent: keyserver = "keyserver.ubuntu.com" if 'keyserver' in ent: keyserver = ent['keyserver'] - try: - ent['key'] = getkeybyid(ent['keyid'], keyserver) - except: - raise Exception('failed to get key from %s' % keyserver) + ent['key'] = getkeybyid(ent['keyid'], keyserver) if 'key' in ent: add_key_raw(ent['key']) - -def add_sources(srclist, template_params=None, aa_repo_match=None): - """ - add entries in /etc/apt/sources.list.d for each abbreviated - sources.list entry in 'srclist'. When rendering template, also - include the values in dictionary searchList +def convert_to_new_format(srclist, errorlist): + """ convert_to_new_format + convert the old list based format to the new dict based one """ - if template_params is None: - template_params = {} - - if aa_repo_match is None: - def aa_repo_match(x): - return False - - errorlist = [] - # convert old list format to new dict based format srcdict = {} if isinstance(srclist, list): fnfallbackused = None @@ -240,6 +224,24 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): else: errorlist.append(["srclist", "unknown apt_sources format"]) + return srcdict + +def add_sources(srclist, template_params=None, aa_repo_match=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srclist'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + if aa_repo_match is None: + def aa_repo_match(x): + return False + + errorlist = [] + srcdict = convert_to_new_format(srclist, errorlist) + for filename in srcdict: ent = srcdict[filename] if 'filename' not in ent: @@ -257,7 +259,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): source = ent['source'] source = templater.render_string(source, template_params) - if not ent['filename'].startswith("/"): + if not ent['filename'].startswith(os.path.sep): ent['filename'] = os.path.join("/etc/apt/sources.list.d/", ent['filename']) -- cgit v1.2.3 From c2cfbd6831912e6dfd6b16ed21afb929592ce51a Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 19:06:55 +0200 Subject: pacify pep8 regarding the new changes --- cloudinit/config/cc_apt_configure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 02b16336..ffbf7513 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -197,6 +197,7 @@ def add_key(ent): if 'key' in ent: add_key_raw(ent['key']) + def convert_to_new_format(srclist, errorlist): """ convert_to_new_format convert the old list based format to the new dict based one @@ -226,6 +227,7 @@ def convert_to_new_format(srclist, errorlist): return srcdict + def add_sources(srclist, template_params=None, aa_repo_match=None): """ add entries in /etc/apt/sources.list.d for each abbreviated -- cgit v1.2.3 From 150abceed3ec8d1d5099e855ebc4fce0361be65d Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Tue, 24 May 2016 19:07:15 +0200 Subject: add test for the now isolated convert_to_new_format function --- .../test_handler/test_handler_apt_source.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index c19904fb..1b2bc58f 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -518,4 +518,33 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile2)) self.assertFalse(os.path.isfile(self.aptlistfile3)) + def test_convert_to_new_format(self): + """ test_convert_to_new_format + Test the conversion of old to new format + And the noop conversion of new to new format as well + """ + cfg1 = {'source': 'deb $MIRROR $RELEASE multiverse', + 'filename': self.aptlistfile} + cfg2 = {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2} + cfg3 = {'source': 'deb $MIRROR $RELEASE universe', + 'filename': self.aptlistfile3} + errorlist = [] + checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, + 'source': 'deb $MIRROR $RELEASE ' + 'multiverse'}, + self.aptlistfile2: {'filename': self.aptlistfile2, + 'source': 'deb $MIRROR $RELEASE main'}, + self.aptlistfile3: {'filename': self.aptlistfile3, + 'source': 'deb $MIRROR $RELEASE ' + 'universe'}} + + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3], + errorlist) + self.assertEqual(newcfg, checkcfg) + + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist) + self.assertEqual(newcfg2, checkcfg) + + # vi: ts=4 expandtab -- cgit v1.2.3 From 70ee3c4e6806996401d747e4f8b25855e85d86eb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 24 May 2016 14:05:20 -0700 Subject: Make the usage of 'nose-timer' optional --- test-requirements.txt | 4 +++- tox.ini | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 88a67a26..aa01fa88 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,9 @@ httpretty>=0.7.1 mock nose -nose-timer + +# Only needed if u want to know the test times +# nose-timer # Only really needed on older versions of python contextlib2 diff --git a/tox.ini b/tox.ini index d051f2ac..dafaaf6d 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py27,py3,flake8 recreate = True [testenv] -commands = python -m nose --with-timer --timer-top-n 10 {posargs:tests} +commands = python -m nose {posargs:tests} deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt @@ -19,7 +19,7 @@ setenv = LC_ALL = en_US.utf-8 [testenv:py26] -commands = nosetests --with-timer --timer-top-n 10 {posargs:tests} +commands = nosetests {posargs:tests} deps = contextlib2 httpretty>=0.7.1 -- cgit v1.2.3 From 2ec72c91b7ba60b260ba0a50097df16f82960dd8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 24 May 2016 19:27:08 -0400 Subject: fix logic error in ec2 get_instance_userdata and slow tests The change to get_instance_userdata is to fix an issue that was causing retry in the test when it was not desired. if user_data returned 404 it means "there was no user-data", so dont bother retrying. However, _skip_retry_on_codes was returning False indicating that readurl should retry. test_merging was creating 2500 random tests, shrink that down to 100. test_seed_runs is still on my system the slowest test, but taking < .5 seconds where it was taking > 3. --- cloudinit/ec2_utils.py | 7 ++++--- tests/unittests/test_merging.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 37b92a83..90b34eff 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,10 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - if cause.code in status_codes: - return False - return True + print("status_codes=%s" % status_codes) + print("_request_args=%s" % _request_args) + print("cause=%s" % cause) + return cause.code in status_codes def get_instance_userdata(api_version='latest', diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index 681f3780..a33ec184 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -125,9 +125,9 @@ class TestSimpleRun(helpers.ResourceUsingTestCase): def test_seed_runs(self): test_dicts = [] - for i in range(1, 50): + for i in range(1, 10): base_dicts = [] - for j in range(1, 50): + for j in range(1, 10): base_dicts.append(make_dict(5, i * j)) test_dicts.append(base_dicts) for test in test_dicts: -- cgit v1.2.3 From 8bd380c5b17b012bcbf00013ab095ea7c6d8b533 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 24 May 2016 19:34:28 -0400 Subject: update changelog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index e06578fc..6748e8fa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -109,6 +109,7 @@ [Wido den Hollander] - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) [Robert Jennings] + - Ec2: do not retry requests for user-data path on 404. 0.7.6: - open 0.7.6 -- cgit v1.2.3 From a84747a8b7a6b527e8ee294e6fc04d70a7090bc5 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 25 May 2016 09:32:39 +0200 Subject: make test_apt_srcl_custom independent to where it is executed --- .../test_handler/test_handler_apt_configure_sources_list.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 353422a2..5cf386f8 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -55,8 +55,8 @@ EXPECTED_CONVERTED_CONTENT = ( # See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to # newer versions of the distribution. -deb http://archive.ubuntu.com/ubuntu/ xenial main restricted -deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted +deb http://archive.ubuntu.com/ubuntu/ fakerelease main restricted +deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted # FIND_SOMETHING_SPECIAL """) @@ -152,8 +152,10 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): # the second mock restores the original subp with mock.patch.object(util, 'write_file') as mockwrite: with mock.patch.object(util, 'subp', self.subp): - cc_apt_configure.handle("notimportant", cfg, mycloud, - LOG, None) + with mock.patch.object(cc_apt_configure, 'get_release', + return_value='fakerelease'): + cc_apt_configure.handle("notimportant", cfg, mycloud, + LOG, None) mockwrite.assert_called_once_with( '/etc/apt/sources.list', -- cgit v1.2.3 From 12cf5ce64f0252a67c6a9dcbd9d3c93a1a07ccad Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Wed, 25 May 2016 13:18:50 +0200 Subject: fix inline doc of test_apt_src_longkeyid_real --- tests/unittests/test_handler/test_handler_apt_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 1b2bc58f..753e86a8 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_longkeyid_real(self): - """ test_apt_src_keyid_real + """ test_apt_src_longkeyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ -- cgit v1.2.3 From db54fca319954d72f3fd48eabd27aad8be31e7b3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 09:08:29 -0400 Subject: remove debug print statements --- cloudinit/ec2_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 90b34eff..76dda042 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -144,9 +144,6 @@ def _skip_retry_on_codes(status_codes, _request_args, cause): """Returns if a request should retry based on a given set of codes that case retrying to be stopped/skipped. """ - print("status_codes=%s" % status_codes) - print("_request_args=%s" % _request_args) - print("cause=%s" % cause) return cause.code in status_codes -- cgit v1.2.3 From 7f2e99f5345c227d07849da68acdf8562b44c3e1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 17:05:09 -0400 Subject: commit to push for fear of loss. == background == DataSource Mode (dsmode) is present in many datasources in cloud-init. dsmode was originally added to cloud-init to specify when this datasource should be 'realized'. cloud-init has 4 stages of boot. a.) cloud-init --local . network is guaranteed not present. b.) cloud-init (--network). network is guaranteed present. c.) cloud-config d.) cloud-init final 'init_modules' [1] are run "as early as possible". And as such, are executed in either 'a' or 'b' based on the datasource. However, executing them means that user-data has been fully consumed. User-data and vendor-data may have '#include http://...' which then rely on the network being present. boothooks are an example of the things run in init_modules. The 'dsmode' was a way for a user to indicate that init_modules should run at 'a' (dsmode=local) or 'b' (dsmode=net) directly. Things were further confused when a datasource could provide networking configuration. Then, we needed to apply the networking config at 'a' but if the user had provided boothooks that expected networking, then the init_modules would need to be executed at 'b'. The config drive datasource hacked its way through this and applies networking if *it* detects it is a new instance. == Suggested Change == The plan is to 1. incorporate 'dsmode' into DataSource superclass 2. make all existing datasources default to network 3. apply any networking configuration from a datasource on first boot only apply_networking will always rename network devices when it runs. for bug 1579130. 4. run init_modules at cloud-init (network) time frame unless datasource is 'local'. 5. Datasources can provide a 'first_boot' method that will be called when a new instance_id is found. This will allow the config drive's write_files to be applied once. Over all, this will very much simplify things. We'll no longer have 2 sources like DataSourceNoCloud and DataSourceNoCloudNet, but would just have one source with a dsmode. == Concerns == Some things have odd reliance on dsmode. For example, OpenNebula's get_hostname uses it to determine if it should do a lookup of an ip address. == Bugs to fix here == http://pad.lv/1577982 ConfigDrive: cloud-init fails to configure network from network_data.json http://pad.lv/1579130 need to support systemd.link renaming of devices in container http://pad.lv/1577844 Drop unnecessary blocking of all net udev rules --- bin/cloud-init | 20 ++++--- cloudinit/distros/__init__.py | 2 + cloudinit/helpers.py | 23 ++++---- cloudinit/net/__init__.py | 45 +++++++++++++++ cloudinit/sources/DataSourceCloudSigma.py | 18 +----- cloudinit/sources/DataSourceConfigDrive.py | 90 ++++++++---------------------- cloudinit/sources/DataSourceNoCloud.py | 78 +++++++++++--------------- cloudinit/sources/DataSourceOpenNebula.py | 39 ++----------- cloudinit/sources/DataSourceOpenStack.py | 9 +-- cloudinit/sources/__init__.py | 33 +++++++++++ cloudinit/stages.py | 66 ++++++++++++++++------ tox.ini | 3 +- 12 files changed, 223 insertions(+), 203 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 5857af32..482b8402 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -236,6 +236,7 @@ def main_init(name, args): else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") + else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): @@ -265,17 +266,20 @@ def main_init(name, args): else: return (None, ["No instance datasource found."]) - if args.local: - if not init.ds_restored: - # if local mode and the datasource was not restored from cache - # (this is not first boot) then apply networking. - init.apply_network_config() - else: - LOG.debug("skipping networking config from restored datasource.") - # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) + + if init.is_new_instance(): + # on new instance, apply network config. if not in local mode, + # then we just bring up new networking as the OS has already + # brought up the configured networking. + init.apply_network_config(bringup=not args.local) + + if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: + return (init.datasource, []) + + # update fully realizes user-data (pulling in #include if necessary) init.update() # Stage 7 try: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 0f222c8c..3bfbc484 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -128,6 +128,8 @@ class Distro(object): mirror_info=arch_info) def apply_network(self, settings, bring_up=True): + # this applies network where 'settings' is interfaces(5) style + # it is obsolete compared to apply_network_config # Write it out dev_names = self._write_network(settings) # Now try to bring them up diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 09d75e65..abfb0cbb 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -328,6 +328,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud') + self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init') self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") self.upstart_conf_d = path_cfgs.get('upstart_dir') @@ -349,26 +350,19 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", + "instance_id": "instance-id", } # Set when a datasource becomes active self.datasource = ds # get_ipath_cur: get the current instance path for an item def get_ipath_cur(self, name=None): - ipath = self.instance_link - add_on = self.lookups.get(name) - if add_on: - ipath = os.path.join(ipath, add_on) - return ipath + return self._get_path(self.instance_link, name) # get_cpath : get the "clouddir" (/var/lib/cloud/) # for a name in dirmap def get_cpath(self, name=None): - cpath = self.cloud_dir - add_on = self.lookups.get(name) - if add_on: - cpath = os.path.join(cpath, add_on) - return cpath + return self._get_path(self.cloud_dir, name) # _get_ipath : get the instance path for a name in pathmap # (/var/lib/cloud/instances//) @@ -397,6 +391,15 @@ class Paths(object): else: return ipath + def _get_path(self, base, name=None): + add_on = self.lookups.get(name) + if not add_on: + return base + return os.path.join(base, add_on) + + def get_runpath(self, name=None): + return self._get_path(self.run_dir, name) + # This config parser will not throw when sections don't exist # and you are setting values on those sections which is useful diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 91e36aca..40d330b5 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -768,4 +768,49 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None): return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs) +def convert_eni_data(eni_data): + # return a network config representation of what is in eni_data + ifaces = {} + parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None) + return _ifaces_to_net_config_data(ifaces) + + +def _ifaces_to_net_config_data(ifaces): + """Return network config that represents the ifaces data provided. + ifaces = parse_deb_config("/etc/network/interfaces") + config = ifaces_to_net_config_data(ifaces) + state = parse_net_config_data(config).""" + devs = {} + for name, data in ifaces.items(): + # devname is 'eth0' for name='eth0:1' + devname = name.partition(":")[0] + if devname not in devs: + devs[devname] = {'type': 'physical', 'name': devname, + 'subnets': []} + # this isnt strictly correct, but some might specify + # hwaddress on a nic for matching / declaring name. + if 'hwaddress' in data: + devs[devname]['mac_address'] = data['hwaddress'] + subnet = {'_orig_eni_name': name, 'type': data['method']} + if data.get('auto'): + subnet['control'] = 'auto' + else: + subnet['control'] = 'manual' + + if data.get('method') == 'static': + subnet['address'] = data['address'] + + if 'gateway' in data: + subnet['gateway'] = data['gateway'] + + if 'dns' in data: + for n in ('nameservers', 'search'): + if n in data['dns'] and data['dns'][n]: + subnet['dns_' + n] = data['dns'][n] + devs[devname]['subnets'].append(subnet) + + return {'version': 1, + 'config': [devs[d] for d in sorted(devs)]} + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 33fe78b9..07e8ae11 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -27,8 +27,6 @@ from cloudinit import util LOG = logging.getLogger(__name__) -VALID_DSMODES = ("local", "net", "disabled") - class DataSourceCloudSigma(sources.DataSource): """ @@ -38,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource): http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html """ def __init__(self, sys_cfg, distro, paths): - self.dsmode = 'local' self.cepko = Cepko() self.ssh_public_key = '' sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -84,11 +81,9 @@ class DataSourceCloudSigma(sources.DataSource): LOG.debug("CloudSigma: Unable to read from serial port") return False - dsmode = server_meta.get('cloudinit-dsmode', self.dsmode) - if dsmode not in VALID_DSMODES: - LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode) - dsmode = 'net' - if dsmode == "disabled" or dsmode != self.dsmode: + self.dsmode = self._determine_dsmode( + [server_meta.get('cloudinit-dsmode')]) + if dsmode == sources.DSMODE_DISABLED: return False base64_fields = server_meta.get('base64_fields', '').split(',') @@ -120,17 +115,10 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] -class DataSourceCloudSigmaNet(DataSourceCloudSigma): - def __init__(self, sys_cfg, distro, paths): - DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)), - (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 52a9f543..20df5fcd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -22,6 +22,7 @@ import copy import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ DEFAULT_MODE = 'pass' DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("local", "net", "pass", "disabled") FS_TYPES = ('vfat', 'iso9660') LABEL_TYPES = ('config-2',) POSSIBLE_MOUNTS = ('sr', 'cd') @@ -47,12 +47,12 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths) self.source = None - self.dsmode = 'local' self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None self._network_config = None self.network_json = None + self.network_eni = None self.files = {} def __str__(self): @@ -98,38 +98,22 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): md = results.get('metadata', {}) md = util.mergemanydict([md, DEFAULT_METADATA]) - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - dsmode = get_ds_mode(cfgdrv_ver=results['version'], - ds_cfg=self.ds_cfg.get('dsmode'), - user=user_dsmode) + self.dsmode = self._determine_dsmode( + [results.get('dsmode'), self.ds_cfg.get('dsmode'), + sources.DSMODE_PASS if results['version'] == 1 else None]) - if dsmode == "disabled": - # most likely user specified + if self.dsmode == sources.DSMODE_DISABLED: return False - # TODO(smoser): fix this, its dirty. - # we want to do some things (writing files and network config) - # only on first boot, and even then, we want to do so in the - # local datasource (so they happen earlier) even if the configured - # dsmode is 'net' or 'pass'. To do this, we check the previous - # instance-id + # This is legacy and sneaky. If dsmode is 'pass' then write + # 'injected files' and apply legacy ENI network format. prev_iid = get_previous_iid(self.paths) cur_iid = md['instance-id'] - if prev_iid != cur_iid and self.dsmode == "local": + if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS: on_first_boot(results, distro=self.distro) - - # dsmode != self.dsmode here if: - # * dsmode = "pass", pass means it should only copy files and then - # pass to another datasource - # * dsmode = "net" and self.dsmode = "local" - # so that user boothooks would be applied with network, the - # local datasource just gets out of the way, and lets the net claim - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) return False self.source = found @@ -147,12 +131,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - try: - self.network_json = results.get('networkdata') - except ValueError as e: - LOG.warn("Invalid content in network-data: %s", e) - self.network_json = None - + # network_config is an /etc/network/interfaces formated file and is + # obsolete compared to networkdata (from network_data.json) but both + # might be present. + self.network_eni = results.get("network_config") + self.network_json = results.get('networkdata') return True def check_instance_id(self, sys_cfg): @@ -164,40 +147,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): if self._network_config is None: if self.network_json is not None: self._network_config = convert_network_data(self.network_json) + elif self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config -class DataSourceConfigDriveNet(DataSourceConfigDrive): - def __init__(self, sys_cfg, distro, paths): - DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - -def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None): - """Determine what mode should be used. - valid values are 'pass', 'disabled', 'local', 'net' - """ - # user passed data trumps everything - if user is not None: - return user - - if ds_cfg is not None: - return ds_cfg - - # at config-drive version 1, the default behavior was pass. That - # meant to not use use it as primary data source, but expect a ec2 metadata - # source. for version 2, we default to 'net', which means - # the DataSourceConfigDriveNet, would be used. - # - # this could change in the future. If there was definitive metadata - # that indicated presense of an openstack metadata service, then - # we could change to 'pass' by default also. The motivation for that - # would be 'cloud-init query' as the web service could be more dynamic - if cfgdrv_ver == 1: - return "pass" - return "net" - - def read_config_drive(source_dir): reader = openstack.ConfigDriveReader(source_dir) finders = [ @@ -231,9 +185,12 @@ def on_first_boot(data, distro=None): % (type(data))) net_conf = data.get("network_config", '') if net_conf and distro: - LOG.debug("Updating network interfaces from config drive") + LOG.warn("Updating network interfaces from config drive") distro.apply_network(net_conf) - files = data.get('files', {}) + write_injected_files(data.get('files')) + + +def write_injected_files(files): if files: LOG.debug("Writing %s injected files", len(files)) for (filename, content) in files.items(): @@ -296,7 +253,6 @@ def find_candidate_devs(probe_optical=True): # Used to match classes to dependencies datasources = [ (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), - (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 48c61a90..7e30118c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -24,6 +24,7 @@ import errno import os from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -35,7 +36,6 @@ class DataSourceNoCloud(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.dsmode = 'local' self.seed = None - self.cmdline_id = "ds=nocloud" self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'), os.path.join(paths.seed_dir, 'nocloud-net')] self.seed_dir = None @@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource): try: # Parse the kernel command line, getting data passed in md = {} - if parse_cmdline_data(self.cmdline_id, md): + if load_cmdline_data(md): found.append("cmdline") mydata = _merge_new_seed(mydata, {'meta-data': md}) except Exception: @@ -123,12 +123,6 @@ class DataSourceNoCloud(sources.DataSource): mydata = _merge_new_seed(mydata, seeded) - # For seed from a device, the default mode is 'net'. - # that is more likely to be what is desired. If they want - # dsmode of local, then they must specify that. - if 'dsmode' not in mydata['meta-data']: - mydata['meta-data']['dsmode'] = "net" - LOG.debug("Using data from %s", dev) found.append(dev) break @@ -144,7 +138,6 @@ class DataSourceNoCloud(sources.DataSource): if len(found) == 0: return False - seeded_network = None # The special argument "seedfrom" indicates we should # attempt to seed the userdata / metadata from its value # its primarily value is in allowing the user to type less @@ -160,10 +153,6 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Seed from %s not supported by %s", seedfrom, self) return False - if (mydata['meta-data'].get('network-interfaces') or - mydata.get('network-config')): - seeded_network = self.dsmode - # This could throw errors, but the user told us to do it # so if errors are raised, let them raise (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) @@ -179,35 +168,21 @@ class DataSourceNoCloud(sources.DataSource): mydata['meta-data'] = util.mergemanydict([mydata['meta-data'], defaults]) - netdata = {'format': None, 'data': None} - if mydata['meta-data'].get('network-interfaces'): - netdata['format'] = 'interfaces' - netdata['data'] = mydata['meta-data']['network-interfaces'] - elif mydata.get('network-config'): - netdata['format'] = 'network-config' - netdata['data'] = mydata['network-config'] - - # if this is the local datasource or 'seedfrom' was used - # and the source of the seed was self.dsmode. - # Then see if there is network config to apply. - # note this is obsolete network-interfaces style seeding. - if self.dsmode in ("local", seeded_network): - if mydata['meta-data'].get('network-interfaces'): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network( - mydata['meta-data']['network-interfaces']) - - if mydata['meta-data']['dsmode'] == self.dsmode: - self.seed = ",".join(found) - self.metadata = mydata['meta-data'] - self.userdata_raw = mydata['user-data'] - self.vendordata_raw = mydata['vendor-data'] - self._network_config = mydata['network-config'] - return True + self.dsmode = self._determine_dsmode( + [mydata['meta-data'].get('dsmode')]) - LOG.debug("%s: not claiming datasource, dsmode=%s", self, - mydata['meta-data']['dsmode']) - return False + if self.dsmode == sources.DSMODE_DISABLED: + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + self.dsmode) + return False + + self.seed = ",".join(found) + self.metadata = mydata['meta-data'] + self.userdata_raw = mydata['user-data'] + self.vendordata_raw = mydata['vendor-data'] + self._network_config = mydata['network-config'] + self._network_eni = mydata['meta-data'].get('network-interfaces') + return True def check_instance_id(self, sys_cfg): # quickly (local check only) if self.instance_id is still valid @@ -227,6 +202,9 @@ class DataSourceNoCloud(sources.DataSource): @property def network_config(self): + if self._network_config is None: + if self.network_eni is not None: + self._network_config = net.convert_eni_data(self.network_eni) return self._network_config @@ -254,8 +232,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None): return None +def load_cmdline_data(fill, cmdline=None): + pairs = [("ds=nocloud", sources.DSMODE_LOCAL), + ("ds=nocloud-net", sources.DSMODE_NETWORK)] + for idstr, dsmode in pairs: + if parse_cmdline_data(idstr, fill, cmdline): + # if dsmode was explicitly in the commanad line, then + # prefer it to the dsmode based on the command line id + if 'dsmode' not in fill: + fill['dsmode'] = dsmode + return True + return False + + # Returns true or false indicating if cmdline indicated -# that this module should be used +# that this module should be used. Updates dictionary 'fill' +# with data that was found. # Example cmdline: # root=LABEL=uec-rootfs ro ds=nocloud def parse_cmdline_data(ds_id, fill, cmdline=None): @@ -319,9 +311,7 @@ def _merge_new_seed(cur, seeded): class DataSourceNoCloudNet(DataSourceNoCloud): def __init__(self, sys_cfg, distro, paths): DataSourceNoCloud.__init__(self, sys_cfg, distro, paths) - self.cmdline_id = "ds=nocloud-net" self.supported_seed_starts = ("http://", "https://", "ftp://") - self.dsmode = "net" # Used to match classes to dependencies diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 681f3a96..15819a4f 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -37,16 +37,13 @@ from cloudinit import util LOG = logging.getLogger(__name__) DEFAULT_IID = "iid-dsopennebula" -DEFAULT_MODE = 'net' DEFAULT_PARSEUSER = 'nobody' CONTEXT_DISK_FILES = ["context.sh"] -VALID_DSMODES = ("local", "net", "disabled") class DataSourceOpenNebula(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'local' self.seed = None self.seed_dir = os.path.join(paths.seed_dir, 'opennebula') @@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource): md = util.mergemanydict([md, defaults]) # check for valid user specified dsmode - user_dsmode = results['metadata'].get('DSMODE', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("user specified invalid mode: %s", user_dsmode) - user_dsmode = None - - # decide dsmode - if user_dsmode: - dsmode = user_dsmode - elif self.ds_cfg.get('dsmode'): - dsmode = self.ds_cfg.get('dsmode') - else: - dsmode = DEFAULT_MODE - - if dsmode == "disabled": - # most likely user specified - return False - - # apply static network configuration only in 'local' dsmode - if ('network-interfaces' in results and self.dsmode == "local"): - LOG.debug("Updating network interfaces from %s", self) - self.distro.apply_network(results['network-interfaces']) + self.dsmode = self._determine_dsmode( + [results.get('DSMODE'), self.ds_cfg.get('dsmode')]) - if dsmode != self.dsmode: - LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode) + if self.dsmode == sources.DSMODE_DISABLED: return False self.seed = seed + self.network_eni = results.get("network_config") self.metadata = md self.userdata_raw = results.get('userdata') return True def get_hostname(self, fqdn=False, resolve_ip=None): if resolve_ip is None: - if self.dsmode == 'net': + if self.dsmode == sources.DSMODE_NET: resolve_ip = True else: resolve_ip = False return sources.DataSource.get_hostname(self, fqdn, resolve_ip) -class DataSourceOpenNebulaNet(DataSourceOpenNebula): - def __init__(self, sys_cfg, distro, paths): - DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths) - self.dsmode = 'net' - - class NonContextDiskDir(Exception): pass @@ -446,7 +418,6 @@ def read_context_disk_dir(source_dir, asuser=None): # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), - (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index dfd96035..c06d17f3 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack" DEFAULT_METADATA = { "instance-id": DEFAULT_IID, } -VALID_DSMODES = ("net", "disabled") class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def __init__(self, sys_cfg, distro, paths): super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths) - self.dsmode = 'net' self.metadata_address = None self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None @@ -125,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.metadata_address) return False - user_dsmode = results.get('dsmode', None) - if user_dsmode not in VALID_DSMODES + (None,): - LOG.warn("User specified invalid mode: %s", user_dsmode) - user_dsmode = None - if user_dsmode == 'disabled': + self.dsmode = self._determine_dsmode([results.get('dsmode')]) + if self.dsmode == sources.DSMODE_DISABLED: return False md = results.get('metadata', {}) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 43e4fd57..e0171e8c 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -34,6 +34,13 @@ from cloudinit import util from cloudinit.filters import launch_index from cloudinit.reporting import events +DSMODE_DISABLED = "disabled" +DSMODE_LOCAL = "net" +DSMODE_NETWORK = "local" +DSMODE_PASS = "pass" + +VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] + DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -57,6 +64,7 @@ class DataSource(object): self.userdata_raw = None self.vendordata = None self.vendordata_raw = None + self.dsmode = DSMODE_NETWORK # find the datasource config name. # remove 'DataSource' from classname on front, and remove 'Net' on end. @@ -223,10 +231,35 @@ class DataSource(object): # quickly (local check only) if self.instance_id is still return False + @staticmethod + def _determine_dsmode(candidates, default=None, valid=None): + # return the first candidate that is non None, warn if not valid + if default is None: + default = DSMODE_NETWORK + + if valid is None: + valid = VALID_DSMODES + + for candidate in candidates: + if candidate is None: + continue + if candidate in valid: + return candidate + else: + LOG.warn("invalid dsmode '%s', using default=%s", + candidate, default) + return default + + return default + @property def network_config(self): return None + @property + def first_instance_boot(self): + return + def normalize_pubkey_data(pubkey_data): keys = [] diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..53ebcb45 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -67,6 +67,7 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE self.ds_restored = False + self._previous_iid = None if reporter is None: reporter = events.ReportEventStack( @@ -213,6 +214,31 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) + def _restore_from_checked_cache(self, existing): + if existing not in ("check", "trust"): + raise ValueError("Unexpected value for existing: %s" % existing) + + ds = self._restore_from_cache() + if not ds: + return (None, "no cache found") + + run_iid_fn = self.paths.get_runpath('instance-id') + if os.path.exists(run_iid_fn): + run_iid = util.load_file(run_iid_fn).strip() + else: + run_iid = None + + if run_iid == ds.get_instance_id: + return (ds, "restored from cache with run check: %s" % ds) + elif existing == "trust": + return (ds, "restored from cache: %s" % ds) + else: + if (hasattr(ds, 'check_instance_id') and + ds.check_instance_id(self.cfg)): + return (ds, "restored from checked cache: %s" % ds) + else: + return (None, "cache invalid in datasource: %s" % ds) + def _get_data_source(self, existing): if self.datasource is not NULL_DATA_SOURCE: return self.datasource @@ -221,19 +247,9 @@ class Init(object): name="check-cache", description="attempting to read from cache [%s]" % existing, parent=self.reporter) as myrep: - ds = self._restore_from_cache() - if ds and existing == "trust": - myrep.description = "restored from cache: %s" % ds - elif ds and existing == "check": - if (hasattr(ds, 'check_instance_id') and - ds.check_instance_id(self.cfg)): - myrep.description = "restored from checked cache: %s" % ds - else: - myrep.description = "cache invalid in datasource: %s" % ds - ds = None - else: - myrep.description = "no cache found" + ds, desc = self._restore_from_checked_cache(existing) + myrep.description = desc self.ds_restored = bool(ds) LOG.debug(myrep.description) @@ -301,15 +317,15 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = None iid_fn = os.path.join(dp, 'instance-id') try: previous_iid = util.load_file(iid_fn).strip() except Exception: - pass + previous_iid = None if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) # Ensure needed components are regenerated @@ -318,6 +334,21 @@ class Init(object): self._reset() return iid + def previous_iid(self): + if self._previous_iid is not None: + return self._previous_iid + + dp = self.paths.get_cpath('data') + iid_fn = os.path.join(dp, 'instance-id') + try: + self._previous_iid = util.load_file(iid_fn).strip() + except Exception: + pass + return self._previous_iid + + def is_new_instance(self): + return self.datasource.get_instance_id() == self.previous_iid() + def fetch(self, existing="check"): return self._get_data_source(existing=existing) @@ -593,15 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self): + def apply_network_config(self, bringup): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return - LOG.info("Applying network configuration from %s: %s", src, netcfg) + LOG.info("Applying network configuration from %s bringup=%s: %s", + src, bringup, netcfg) try: - return self.distro.apply_network_config(netcfg) + return self.distro.apply_network_config(netcfg, bringup=bringup) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % diff --git a/tox.ini b/tox.ini index dafaaf6d..18d059df 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] envlist = py27,py3,flake8 -recreate = True +recreate = False +skip_install = True [testenv] commands = python -m nose {posargs:tests} -- cgit v1.2.3 From 4728b0de649bf231fa667373d5d8ebf451e19d0b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 20:21:04 -0400 Subject: packages/bddeb: fix to know about packages flake8 and hacking --- packages/bddeb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/bddeb b/packages/bddeb index c141b1ab..1b0f642c 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -40,6 +40,8 @@ STD_NAMED_PACKAGES = [ 'mock', 'nose', 'setuptools', + 'flake8', + 'hacking', ] NONSTD_NAMED_PACKAGES = { 'argparse': ('python-argparse', None), -- cgit v1.2.3 From ff56555eccd098e144332aad8faf5ebea5040244 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 20:36:04 -0400 Subject: fix ./tools/bddeb --- packages/bddeb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/bddeb b/packages/bddeb index c141b1ab..1b0f642c 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -40,6 +40,8 @@ STD_NAMED_PACKAGES = [ 'mock', 'nose', 'setuptools', + 'flake8', + 'hacking', ] NONSTD_NAMED_PACKAGES = { 'argparse': ('python-argparse', None), -- cgit v1.2.3 From 443df65a45e42c354b8eb7638528970adf036290 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 25 May 2016 20:36:40 -0400 Subject: fix bring_up --- bin/cloud-init | 4 ++-- cloudinit/stages.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 482b8402..1d421acf 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -260,7 +260,7 @@ def main_init(name, args): util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: - init.apply_network_config() + init.apply_network_config(bring_up=not args.local) if args.local: return (None, []) else: @@ -274,7 +274,7 @@ def main_init(name, args): # on new instance, apply network config. if not in local mode, # then we just bring up new networking as the OS has already # brought up the configured networking. - init.apply_network_config(bringup=not args.local) + init.apply_network_config(bring_up=not args.local) if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: return (init.datasource, []) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 53ebcb45..35eddc3b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -624,16 +624,16 @@ class Init(object): return (ncfg, loc) return (net.generate_fallback_config(), "fallback") - def apply_network_config(self, bringup): + def apply_network_config(self, bring_up): netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return LOG.info("Applying network configuration from %s bringup=%s: %s", - src, bringup, netcfg) + src, bring_up, netcfg) try: - return self.distro.apply_network_config(netcfg, bringup=bringup) + return self.distro.apply_network_config(netcfg, bring_up=bring_up) except NotImplementedError: LOG.warn("distro '%s' does not implement apply_network_config. " "networking may not be configured properly." % -- cgit v1.2.3 From 63501f44eff7ef2d6083900c47180faf444662fc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 09:02:17 -0400 Subject: kernel command line: override all local settings settings on the kernel command line (cc:) were documented to override all local settings, but a bug in implementation meant they would only override those that are in /etc/cloud/cloud.cfg, not any found in /etc/cloud/cloud.cfg.d. LP: #1582323 --- ChangeLog | 2 ++ cloudinit/stages.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6748e8fa..8db29e2e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -110,6 +110,8 @@ - Paths: fix instance path if datasource's id has a '/'. (LP: #1575938) [Robert Jennings] - Ec2: do not retry requests for user-data path on 404. + - settings on the kernel command line (cc:) override all local settings + rather than only those in /etc/cloud/cloud.cfg (LP: #1582323) 0.7.6: - open 0.7.6 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 62d066de..002e5832 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -794,16 +794,16 @@ class Modules(object): def fetch_base_config(): base_cfgs = [] default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) # Anything in your conf.d location?? # or the 'default' cloud.cfg location??? base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # Kernel/cmdline parameters override system config + kern_contents = util.read_cc_from_cmdline() + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + # And finally the default gets to play if default_cfg: base_cfgs.append(default_cfg) -- cgit v1.2.3 From f358f286dc4887fc40b5e2ad370dc35d46905121 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 10:50:32 -0400 Subject: clarify a comment --- bin/cloud-init | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 1d421acf..ffd1f2cc 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -271,9 +271,9 @@ def main_init(name, args): LOG.debug("%s will now be targeting instance id: %s", name, iid) if init.is_new_instance(): - # on new instance, apply network config. if not in local mode, - # then we just bring up new networking as the OS has already - # brought up the configured networking. + # on new instance, apply network config. + # in network mode 'bring_up' must be passed in as the OS + # has already brought up networking. init.apply_network_config(bring_up=not args.local) if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: -- cgit v1.2.3 From 399eb29662ee67f7744d3a482f63e8af377e75db Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:22:39 -0400 Subject: config drive: log where network config came from --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 20df5fcd..2d13a32f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -146,9 +146,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): def network_config(self): if self._network_config is None: if self.network_json is not None: + LOG.debug("network config provided via network_json") self._network_config = convert_network_data(self.network_json) elif self.network_eni is not None: self._network_config = net.convert_eni_data(self.network_eni) + LOG.debug("network config provided via converted eni data") + else: + LOG.debug("no network configuration available") return self._network_config -- cgit v1.2.3 From b4a298b10c26ee79ee6f21a164cf32ab767ca14f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:22:52 -0400 Subject: cloudinit/helpers.py: _get_path raise KeyError if input is bad. previously, if you did: paths.get_ipath("bogus") it would silenetly hand you back just the directory. now it will fail, which seems much more sane. --- cloudinit/helpers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index abfb0cbb..d4acbe7e 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -392,10 +392,9 @@ class Paths(object): return ipath def _get_path(self, base, name=None): - add_on = self.lookups.get(name) - if not add_on: + if name is None: return base - return os.path.join(base, add_on) + return os.path.join(base, self.lookups[name]) def get_runpath(self, name=None): return self._get_path(self.run_dir, name) -- cgit v1.2.3 From ac851b8ff106abb4314d2b71f2a7dc194b259899 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 11:24:28 -0400 Subject: fix usage of instance-id to instance_id in runpath, update cache on instancify the fix for instance_id is clear and necessary. making instancify write the cache is required for how we are having the local datasource be relevant. --- cloudinit/stages.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 35eddc3b..5141612f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -222,7 +222,7 @@ class Init(object): if not ds: return (None, "no cache found") - run_iid_fn = self.paths.get_runpath('instance-id') + run_iid_fn = self.paths.get_runpath('instance_id') if os.path.exists(run_iid_fn): run_iid = util.load_file(run_iid_fn).strip() else: @@ -325,9 +325,11 @@ class Init(object): if not previous_iid: previous_iid = iid util.write_file(iid_fn, "%s\n" % iid) - util.write_file(self.paths.get_runpath('instance-id'), "%s\n" % iid) + util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) + + self._write_to_cache() # Ensure needed components are regenerated # after change of instance which may cause # change of configuration @@ -363,8 +365,6 @@ class Init(object): reporter=self.reporter) def update(self): - if not self._write_to_cache(): - return self._store_userdata() self._store_vendordata() -- cgit v1.2.3 From e86794ba646863ac54dcb348c016d2ae0be2c180 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:50:40 -0400 Subject: compare instance id to get_instance_id() not the function itself. --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 90c9016c..cbf4df8b 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -228,7 +228,7 @@ class Init(object): else: run_iid = None - if run_iid == ds.get_instance_id: + if run_iid == ds.get_instance_id(): return (ds, "restored from cache with run check: %s" % ds) elif existing == "trust": return (ds, "restored from cache: %s" % ds) -- cgit v1.2.3 From 48b7526425055a3c636f11135305f1e77469562a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:50:59 -0400 Subject: fix typos in names --- cloudinit/sources/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index e0171e8c..2a6b8d90 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -35,8 +35,8 @@ from cloudinit.filters import launch_index from cloudinit.reporting import events DSMODE_DISABLED = "disabled" -DSMODE_LOCAL = "net" -DSMODE_NETWORK = "local" +DSMODE_LOCAL = "local" +DSMODE_NETWORK = "net" DSMODE_PASS = "pass" VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] -- cgit v1.2.3 From c21ef05925954e5781c793429316cd4eff64ee37 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 15:51:09 -0400 Subject: fixes. seemingly working first boot on config drive bigger things here: * fix the checking for stop_files. the check for no-net actually checked the size of the file and the implementation was just touching it. so it never would have been found. no-net is valid only in upstart anyway. do not stop early on presense of the obj_pkl but check it. this is required since we write the obj_pkl on exit when local mode finds a datasource but found in network mode. * use 'mode' rather than checking args.local. set mode to be sources.DSMODE_NETWORK or sources.DSMODE_LOCAL for easier / more consistent checking. * log exit paths. --- bin/cloud-init | 52 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index ffd1f2cc..29e9b521 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -211,32 +211,31 @@ def main_init(name, args): util.logexc(LOG, "Failed to initialize, likely bad things to come!") # Stage 4 path_helper = init.paths - if not args.local: + mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK + + if mode == sources.DSMODE_NETWORK: existing = "trust" sys.stderr.write("%s\n" % (netinfo.debug_info())) LOG.debug(("Checking to see if files that we need already" " exist from a previous run that would allow us" " to stop early.")) + # no-net is written by upstart cloud-init-nonet when network failed + # to come up stop_files = [ os.path.join(path_helper.get_cpath("data"), "no-net"), - path_helper.get_ipath_cur("obj_pkl"), ] existing_files = [] for fn in stop_files: - try: - c = util.load_file(fn) - if len(c): - existing_files.append((fn, len(c))) - except Exception: - pass + if os.path.isfile(fn): + existing_files.append(fn) + if existing_files: - LOG.debug("Exiting early due to the existence of %s files", - existing_files) + LOG.debug("[%s] Exiting. stop file %s existed", + mode, existing_files) return (None, []) else: LOG.debug("Execution continuing, no previous run detected that" " would allow us to stop early.") - else: existing = "check" if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False): @@ -249,35 +248,52 @@ def main_init(name, args): # Stage 5 try: init.fetch(existing=existing) + # if in network mode, and the datasource is local + # then work was done at that stage. + if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s in local mode", + mode, init.datasource) + return (None, []) except sources.DataSourceNotFoundException: # In the case of 'cloud-init init' without '--local' it is a bit # more likely that the user would consider it failure if nothing was # found. When using upstart it will also mentions job failure # in console log if exit code is != 0. - if args.local: + if mode == sources.DSMODE_LOCAL: LOG.debug("No local datasource found") else: util.logexc(LOG, ("No instance datasource found!" " Likely bad things to come!")) if not args.force: init.apply_network_config(bring_up=not args.local) - if args.local: + LOG.debug("[%s] Exiting without datasource in local mode", mode) + if mode == sources.DSMODE_LOCAL: return (None, []) else: return (None, ["No instance datasource found."]) + else: + LOG.debug("[%s] barreling on in force mode without datasource", + mode) # Stage 6 iid = init.instancify() - LOG.debug("%s will now be targeting instance id: %s", name, iid) + LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", + mode, name, iid, init.is_new_instance()) if init.is_new_instance(): # on new instance, apply network config. # in network mode 'bring_up' must be passed in as the OS # has already brought up networking. - init.apply_network_config(bring_up=not args.local) + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) - if args.local and init.datasource.dsmode != sources.DSMODE_LOCAL: - return (init.datasource, []) + if mode == sources.DSMODE_LOCAL: + if init.datasource.dsmode != mode: + LOG.debug("[%s] Exiting. datasource %s not in local mode.", + mode, init.datasource) + return (init.datasource, []) + else: + LOG.debug("[%s] %s is in local mode, will apply init modules now.", + mode, init.datasource) # update fully realizes user-data (pulling in #include if necessary) init.update() @@ -532,7 +548,7 @@ def status_wrapper(name, args, data_d=None, link_d=None): v1[mode]['errors'] = [str(e) for e in errors] except Exception as e: - util.logexc(LOG, "failed of stage %s", mode) + util.logexc(LOG, "failed stage %s", mode) print_exc("failed run of stage %s" % mode) v1[mode]['errors'] = [str(e)] -- cgit v1.2.3 From 1174e19116f2934f927d89fe00ed8d6e22983d75 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 16:02:53 -0400 Subject: fix logic in is_new_instance --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index cbf4df8b..e34662b1 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -349,7 +349,7 @@ class Init(object): return self._previous_iid def is_new_instance(self): - return self.datasource.get_instance_id() == self.previous_iid() + return self.datasource.get_instance_id() != self.previous_iid() def fetch(self, existing="check"): return self._get_data_source(existing=existing) -- cgit v1.2.3 From 22bd075cd77ccc1021e049333f498afd56de2c6a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 26 May 2016 16:10:22 -0500 Subject: Add smartos sdc:nics converter and network_config property for smartos configdrive --- cloudinit/sources/DataSourceSmartOS.py | 95 ++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 4 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 46cf117a..6355fc2a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -64,6 +64,7 @@ SMARTOS_ATTRIB_MAP = { 'availability_zone': ('sdc:datacenter_name', True), 'vendor-data': ('sdc:vendor-data', False), 'operator-script': ('sdc:operator-script', False), + 'network-data': ('sdc:nics', False), } DS_NAME = 'SmartOS' @@ -176,6 +177,8 @@ class DataSourceSmartOS(sources.DataSource): BUILTIN_DS_CONFIG]) self.metadata = {} + self.network_data = None + self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) @@ -195,7 +198,6 @@ class DataSourceSmartOS(sources.DataSource): serial_device=self.ds_cfg['serial_device'], serial_timeout=self.ds_cfg['serial_timeout']) - def _set_provisioned(self): '''Mark the instance provisioning state as successful. @@ -221,7 +223,7 @@ class DataSourceSmartOS(sources.DataSource): if not self.smartos_env: LOG.debug("Not running on smartos") return False - + if not self.md_client.exists(): LOG.debug("No metadata device '%r' found for SmartOS datasource", self.md_client) @@ -279,6 +281,11 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] + if not md['network-data']: + smartos_noun, strip = SMARTOS_ATTRIB_MAP.get('network-data') + self.network_data = self.md_client.get_json(smartos_noun, + strip=strip) + md['network-data'] = self.network_data self._set_provisioned() return True @@ -292,6 +299,14 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] + @property + def network_config(self): + if self._network_config is None: + if self.network_data is not None: + self._network_config = ( + convert_smartos_network_data(self.network_data)) + return self._network_config + class JoyentMetadataFetchException(Exception): pass @@ -386,8 +401,8 @@ class JoyentMetadataClient(object): result = result.strip() return result - def get_json(self, key, default=None): - result = self.get(key) + def get_json(self, key, default=None, strip=False): + result = self.get(key, default=default, strip=strip) if result is None: return default return json.loads(result) @@ -654,5 +669,77 @@ def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) +# Covert SMARTOS 'sdc:nics' data to network_config yaml +def convert_smartos_network_data(network_data=None): + """Return a dictionary of network_config by parsing provided + SMARTOS sdc:nics configuration data + + sdc:nics data is a dictionary of properties of a nic and the ip + configuration desired. Additional nic dictionaries are appended + to the list. + + Converting the format is straightforward though it does include + duplicate information as well as data which appears to be relevant + to the hostOS rather than the guest. + + For each entry in the nics list returned from query sdc:nics, we + create a type: physical entry, and extract the interface properties: + 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining + keys are related to ip configuration. For each ip in the 'ips' list + we create a subnet entry under 'subnets' pairing the ip to a one in + the 'gateways' list. + """ + + valid_keys = { + 'physical': [ + 'mac_address', + 'mtu', + 'name', + 'params', + 'subnets', + 'type', + ], + 'subnet': [ + 'address', + 'broadcast', + 'dns_nameservers', + 'dns_search', + 'gateway', + 'metric', + 'netmask', + 'pointopoint', + 'routes', + 'scope', + 'type', + ], + } + + config = [] + for nic in network_data: + cfg = {k: v for k, v in nic.items() + if k in valid_keys['physical']} + cfg.update({ + 'type': 'physical', + 'name': nic['interface'] + }) + if 'mac' in nic: + cfg.update({'mac_address': nic['mac']}) + + subnets = [] + for ip, gw in zip(nic['ips'], nic['gateways']): + subnet = {k: v for k, v in nic.items() + if k in valid_keys['subnet']} + subnet.update({ + 'type': 'static', + 'address': ip, + 'gateway': gw, + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + config.append(cfg) + + return {'version': 1, 'config': config} + + if __name__ == "__main__": jmc = JoyentMetadataClient(seed_file).get_metadata(noun) -- cgit v1.2.3 From f63f16c31be4f3b993a671e95b74550150f5715f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 21:53:38 -0400 Subject: hide the instance_id file in /run/cloud-init by using .instance_id i dont want to expose this as i'd rather have some json there or write to /run/cloud-init/status.json . would also like to indicate 'first_boot' somewhere. --- cloudinit/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index d4acbe7e..fb95babc 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -350,7 +350,7 @@ class Paths(object): "data": "data", "vendordata_raw": "vendor-data.txt", "vendordata": "vendor-data.txt.i", - "instance_id": "instance-id", + "instance_id": ".instance-id", } # Set when a datasource becomes active self.datasource = ds -- cgit v1.2.3 From 3ce4cebd417043e1d0a3bb387998f9b71bb76e03 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 26 May 2016 21:54:33 -0400 Subject: fix is_new_instance() to work better --- cloudinit/stages.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e34662b1..20c334b7 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -52,6 +52,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) NULL_DATA_SOURCE = None +NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID" class Init(object): @@ -68,6 +69,8 @@ class Init(object): self.datasource = NULL_DATA_SOURCE self.ds_restored = False self._previous_iid = None + # simply ensure this gets set + self.previous_iid() if reporter is None: reporter = events.ReportEventStack( @@ -345,11 +348,15 @@ class Init(object): try: self._previous_iid = util.load_file(iid_fn).strip() except Exception: + self._previous_iid = NO_PREVIOUS_INSTANCE_ID pass return self._previous_iid def is_new_instance(self): - return self.datasource.get_instance_id() != self.previous_iid() + previous = self.previous_iid() + ret = (previous == NO_PREVIOUS_INSTANCE_ID or + previous != self.datasource.get_instance_id()) + return ret def fetch(self, existing="check"): return self._get_data_source(existing=existing) -- cgit v1.2.3 From c1c1550d74d067d78cf4ba1cf64f38c2dae8c9e1 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 26 May 2016 21:17:29 -0500 Subject: Move sdc:nics to a JSON map. Add unittest for sdc:nics --- cloudinit/sources/DataSourceSmartOS.py | 19 +++++---- tests/unittests/test_datasource/test_smartos.py | 51 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 6355fc2a..35fa8c33 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -64,7 +64,11 @@ SMARTOS_ATTRIB_MAP = { 'availability_zone': ('sdc:datacenter_name', True), 'vendor-data': ('sdc:vendor-data', False), 'operator-script': ('sdc:operator-script', False), - 'network-data': ('sdc:nics', False), +} + +SMARTOS_ATTRIB_JSON = { + # Cloud-init Key : (SmartOS Key known JSON) + 'network-data': 'sdc:nics', } DS_NAME = 'SmartOS' @@ -233,6 +237,9 @@ class DataSourceSmartOS(sources.DataSource): smartos_noun, strip = attribute md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) + for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): + md[ci_noun] = self.md_client.get_json(smartos_noun) + # @datadictionary: This key may contain a program that is written # to a file in the filesystem of the guest on each boot and then # executed. It may be of any format that would be considered @@ -281,11 +288,7 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = util.mergemanydict([md, self.metadata]) self.userdata_raw = ud self.vendordata_raw = md['vendor-data'] - if not md['network-data']: - smartos_noun, strip = SMARTOS_ATTRIB_MAP.get('network-data') - self.network_data = self.md_client.get_json(smartos_noun, - strip=strip) - md['network-data'] = self.network_data + self.network_data = md['network-data'] self._set_provisioned() return True @@ -401,8 +404,8 @@ class JoyentMetadataClient(object): result = result.strip() return result - def get_json(self, key, default=None, strip=False): - result = self.get(key, default=default, strip=strip) + def get_json(self, key, default=None): + result = self.get(key, default=default) if result is None: return default return json.loads(result) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5c49966a..1ee64d60 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,6 +24,7 @@ from __future__ import print_function +import json import os import os.path import re @@ -47,6 +48,48 @@ try: except ImportError: import mock +SDC_NICS = json.loads(""" +[ + { + "nic_tag": "external", + "primary": true, + "mtu": 1500, + "model": "virtio", + "gateway": "8.12.42.1", + "netmask": "255.255.255.0", + "ip": "8.12.42.102", + "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe", + "gateways": [ + "8.12.42.1" + ], + "vlan_id": 324, + "mac": "90:b8:d0:f5:e4:f5", + "interface": "net0", + "ips": [ + "8.12.42.102/24" + ] + }, + { + "nic_tag": "sdc_overlay/16187209", + "gateway": "192.168.128.1", + "model": "virtio", + "mac": "90:b8:d0:a5:ff:cd", + "netmask": "255.255.252.0", + "ip": "192.168.128.93", + "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9", + "gateways": [ + "192.168.128.1" + ], + "vlan_id": 2, + "mtu": 8500, + "interface": "net1", + "ips": [ + "192.168.128.93/22" + ] + } +] +""") + MOCK_RETURNS = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', @@ -60,6 +103,7 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), + 'sdc:nics': SDC_NICS, } DMI_DATA_RETURN = 'smartdc' @@ -275,6 +319,13 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.assertEquals(MOCK_RETURNS['cloud-init:user-data'], dsrc.userdata_raw) + def test_sdc_nics(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['sdc:nics'], + dsrc.metadata['network-data']) + def test_sdc_scripts(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() -- cgit v1.2.3 From 5cd30a36eaa6ca1a239019a5409faa603f063f6c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 09:24:23 -0400 Subject: fix pyflakes, move datasources= to bottom --- cloudinit/sources/DataSourceSmartOS.py | 37 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 35fa8c33..4224f2ba 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -34,13 +34,11 @@ import base64 import binascii -import contextlib import json import os import random import re import socket -import stat import serial @@ -391,9 +389,6 @@ class JoyentMetadataClient(object): return None value = self._get_value_from_frame(request_id, response) - if value is None: - return default - return value def get(self, key, default=None, strip=False): @@ -442,11 +437,11 @@ class JoyentMetadataClient(object): class JoyentMetadataSocketClient(JoyentMetadataClient): def __init__(self, socketpath): - self.socketpath = metadata_socketfile + self.socketpath = socketpath def open_transport(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(path) + sock.connect(self.socketpath) self.fp = sock.makefile('rwb') def exists(self): @@ -577,7 +572,7 @@ def jmc_client_factory( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) elif smartos_type == 'lx-brand': - return JoyentMetadataSerialClient(socketpath=metadata_socketfile) + return JoyentMetadataSerialClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) @@ -661,17 +656,6 @@ def get_smartos_environ(uname_version=None, product_name=None, return None -# Used to match classes to dependencies -datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - # Covert SMARTOS 'sdc:nics' data to network_config yaml def convert_smartos_network_data(network_data=None): """Return a dictionary of network_config by parsing provided @@ -744,5 +728,18 @@ def convert_smartos_network_data(network_data=None): return {'version': 1, 'config': config} +# Used to match classes to dependencies +datasources = [ + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) + + if __name__ == "__main__": - jmc = JoyentMetadataClient(seed_file).get_metadata(noun) + import sys + jmc = jmc_client_factory() + jmc.get_metadata(sys.argv[1]) -- cgit v1.2.3 From 949cebd48c9100d4fd00b74232bcf048980e6e0d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 09:49:29 -0400 Subject: fix pyflakes and some pylint errors/warnings --- cloudinit/sources/DataSourceSmartOS.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 4224f2ba..cf4e00e5 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -69,6 +69,9 @@ SMARTOS_ATTRIB_JSON = { 'network-data': 'sdc:nics', } +SMARTOS_ENV_LX_BRAND = "lx-brand" +SMARTOS_ENV_KVM = "kvm" + DS_NAME = 'SmartOS' DS_CFG_PATH = ['datasource', DS_NAME] NO_BASE64_DECODE = [ @@ -183,6 +186,7 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) + self.smartos_env = None self._init() @@ -295,7 +299,9 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - return self.cfg + if self.smartos_env == SMARTOS_ENV_KVM: + return BUILTIN_CLOUD_CONFIG + return None def get_instance_id(self): return self.metadata['instance-id'] @@ -434,6 +440,9 @@ class JoyentMetadataClient(object): self.close_transport() return + def open_transport(self): + raise NotImplementedError + class JoyentMetadataSocketClient(JoyentMetadataClient): def __init__(self, socketpath): @@ -519,7 +528,7 @@ class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): # now add any b64- that has a true value for key in [k[3:] for k in keys if k.startswith("b64-")]: if util.is_true(self._get(key)): - b64_keys.append(key) + b64_keys.add(key) else: if key in b64_keys: b64_keys.remove(key) @@ -572,7 +581,7 @@ def jmc_client_factory( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) elif smartos_type == 'lx-brand': - return JoyentMetadataSerialClient(socketpath=metadata_sockfile) + return JoyentMetadataSocketClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) @@ -647,11 +656,15 @@ def get_smartos_environ(uname_version=None, product_name=None, if uname_version is None: uname_version = uname[3] if uname_version.lower() == 'brandz virtual linux': - return 'lx-brand' + return SMARTOS_ENV_LX_BRAND + + if product_name is None: + system_type = util.read_dmi_data("system-product-name") + else: + system_type = product_name - system_type = util.read_dmi_data("system-product-name") if system_type and 'smartdc' in system_type.lower(): - return 'kvm' + return SMARTOS_ENV_KVM return None @@ -742,4 +755,4 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() - jmc.get_metadata(sys.argv[1]) + jmc.get(sys.argv[1]) -- cgit v1.2.3 From e6e768769d67960cde12dee0b0c2b58deb2eb376 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:06:59 -0400 Subject: fix a bunch of the tests --- cloudinit/sources/DataSourceSmartOS.py | 12 +- tests/unittests/test_datasource/test_smartos.py | 312 ++++++++++++++++++++++-- 2 files changed, 298 insertions(+), 26 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index cf4e00e5..e64dea68 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -186,7 +186,7 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - self.smartos_env = None + self.smartos_type = None self._init() @@ -196,10 +196,11 @@ class DataSourceSmartOS(sources.DataSource): def _init(self): if self.smartos_environ == self._unset: - self.smartos_env = get_smartos_environ() + self.smartos_type = get_smartos_environ() if self.md_client == self._unset: self.md_client = jmc_client_factory( + smartos_type=self.smartos_type, metadata_sockfile=self.ds_cfg['metadata_sockfile'], serial_device=self.ds_cfg['serial_device'], serial_timeout=self.ds_cfg['serial_timeout']) @@ -226,7 +227,7 @@ class DataSourceSmartOS(sources.DataSource): md = {} ud = "" - if not self.smartos_env: + if not self.smartos_type: LOG.debug("Not running on smartos") return False @@ -299,7 +300,7 @@ class DataSourceSmartOS(sources.DataSource): return self.ds_cfg['disk_aliases'].get(name) def get_config_obj(self): - if self.smartos_env == SMARTOS_ENV_KVM: + if self.smartos_type == SMARTOS_ENV_KVM: return BUILTIN_CLOUD_CONFIG return None @@ -608,6 +609,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, bit and to the SmartOS default of assuming that bash. """ + print("content_f=%s" % content_f) if not content and os.path.exists(content_f): os.unlink(content_f) if link and os.path.islink(link): @@ -639,7 +641,7 @@ def write_boot_content(content, content_f, link=None, shebang=False, util.ensure_dir(os.path.dirname(link)) os.symlink(content_f, link) except IOError as e: - util.logexc(LOG, "failed establishing content link", e) + util.logexc(LOG, "failed establishing content link: %s", e) def get_smartos_environ(uname_version=None, product_name=None, diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index ea20777a..946286bd 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -40,13 +40,9 @@ import six from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e +from cloudinit import util -from .. import helpers - -try: - from unittest import mock -except ImportError: - import mock +from ..helpers import mock, TestCase, FilesystemMockingTestCase SDC_NICS = json.loads(""" [ @@ -103,7 +99,7 @@ MOCK_RETURNS = { 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), 'user-data': '\n'.join(['something', '']), 'user-script': '\n'.join(['/bin/true', '']), - 'sdc:nics': SDC_NICS, + 'sdc:nics': json.dumps(SDC_NICS), } DMI_DATA_RETURN = 'smartdc' @@ -115,12 +111,286 @@ def get_mock_client(mockdata): def __init__(self, serial): pass - def get_metadata(self, metadata_key): + def get(self, metadata_key): return mockdata.get(metadata_key) return MockMetadataClient -class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): +class PsuedoJoyentClient(object): + def __init__(self, data=None): + if data is None: + data = MOCK_RETURNS.copy() + self.data = data + return + + def get(self, key, default=None, strip=False): + if key in self.data: + r = self.data[key] + if strip: + r = r.strip() + else: + r = default + return r + + def get_json(self, key, default=None): + result = self.get(key, default=default) + if result is None: + return default + return json.loads(result) + + def exists(self): + return True + + +class TestSmartOSDataSource(FilesystemMockingTestCase): + def setUp(self): + super(TestSmartOSDataSource, self).setUp() + + dsmos = 'cloudinit.sources.DataSourceSmartOS' + patcher = mock.patch(dsmos + ".jmc_client_factory") + self.jmc_cfact = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch(dsmos + ".get_smartos_environ") + self.get_smartos_environ = patcher.start() + self.addCleanup(patcher.stop) + + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp) + self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) + + self.legacy_user_d = tempfile.mkdtemp() + self.orig_lud = DataSourceSmartOS.LEGACY_USER_D + DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d + + def tearDown(self): + DataSourceSmartOS.LEGACY_USER_D = self.orig_lud + super(TestSmartOSDataSource, self).tearDown() + + def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM, + sys_cfg=None, ds_cfg=None): + self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) + self.get_smartos_environ.return_value = mode + + if sys_cfg is None: + sys_cfg = {} + + if ds_cfg is not None: + sys_cfg['datasource'] = sys_cfg.get('datasource', {}) + sys_cfg['datasource']['SmartOS'] = ds_cfg + + return DataSourceSmartOS.DataSourceSmartOS( + sys_cfg, distro=None, paths=self.paths) + + def test_it_got_here(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + + def test_no_base64(self): + ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(ds_cfg=ds_cfg) + ret = dsrc.get_data() + self.assertTrue(ret) + + def test_uuid(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['sdc:uuid'], + dsrc.metadata['instance-id']) + + def test_root_keys(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_hostname_b64(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + + def test_hostname(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + + def test_userdata(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-data'], + dsrc.metadata['legacy-user-data']) + self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], + dsrc.userdata_raw) + + def test_sdc_nics(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(json.loads(MOCK_RETURNS['sdc:nics']), + dsrc.metadata['network-data']) + + def test_sdc_scripts(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEqual(user_script_perm, '700') + + def test_scripts_shebanged(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEqual(shebang, "#!/bin/bash") + user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] + self.assertEqual(user_script_perm, '700') + + def test_scripts_shebang_not_added(self): + """ + Test that the SmartOS requirement that plain text scripts + are executable. This test makes sure that plain texts scripts + with out file magic have it added appropriately by cloud-init. + """ + + my_returns = MOCK_RETURNS.copy() + my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', + 'print("hi")', '']) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(my_returns['user-script'], + dsrc.metadata['user-script']) + + legacy_script_f = "%s/user-script" % self.legacy_user_d + self.assertTrue(os.path.exists(legacy_script_f)) + self.assertTrue(os.path.islink(legacy_script_f)) + shebang = None + with open(legacy_script_f, 'r') as f: + shebang = f.readlines()[0].strip() + self.assertEqual(shebang, "#!/usr/bin/perl") + + def test_userdata_removed(self): + """ + User-data in the SmartOS world is supposed to be written to a file + each and every boot. This tests to make sure that in the event the + legacy user-data is removed, the existing user-data is backed-up + and there is no /var/db/user-data left. + """ + + user_data_f = "%s/mdata-user-data" % self.legacy_user_d + with open(user_data_f, 'w') as f: + f.write("PREVIOUS") + + my_returns = MOCK_RETURNS.copy() + del my_returns['user-data'] + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertFalse(dsrc.metadata.get('legacy-user-data')) + + found_new = False + for root, _dirs, files in os.walk(self.legacy_user_d): + for name in files: + name_f = os.path.join(root, name) + permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] + if re.match(r'.*\/mdata-user-data$', name_f): + found_new = True + print(name_f) + self.assertEqual(permissions, '400') + + self.assertFalse(found_new) + + def test_vendor_data_not_default(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], + dsrc.metadata['vendor-data']) + + def test_default_vendor_data(self): + my_returns = MOCK_RETURNS.copy() + def_op_script = my_returns['sdc:vendor-data'] + del my_returns['sdc:vendor-data'] + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) + + # we expect default vendor-data is a boothook + self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) + + def test_disable_iptables_flag(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) + + def test_motd_sys_info(self): + dsrc = self._get_ds(mockdata=MOCK_RETURNS) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) + + def test_default_ephemeral(self): + # Test to make sure that the builtin config has the ephemeral + # configuration. + dsrc = self._get_ds() + cfg = dsrc.get_config_obj() + + ret = dsrc.get_data() + self.assertTrue(ret) + + assert 'disk_setup' in cfg + assert 'fs_setup' in cfg + self.assertIsInstance(cfg['disk_setup'], dict) + self.assertIsInstance(cfg['fs_setup'], list) + + def test_override_disk_aliases(self): + # Test to make sure that the built-in DS is overriden + builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG + + mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} + + # expect that these values are in builtin, or this is pointless + for k in mydscfg: + self.assertIn(k, builtin) + + dsrc = self._get_ds(ds_cfg=mydscfg) + ret = dsrc.get_data() + self.assertTrue(ret) + + self.assertEqual(mydscfg['disk_aliases']['FOO'], + dsrc.ds_cfg['disk_aliases']['FOO']) + + self.assertEqual(dsrc.device_name_to_device('FOO'), + mydscfg['disk_aliases']['FOO']) + + +class OldTestSmartOSDataSource(FilesystemMockingTestCase): def setUp(self): super(TestSmartOSDataSource, self).setUp() @@ -141,7 +411,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): super(TestSmartOSDataSource, self).setUp() def tearDown(self): - helpers.FilesystemMockingTestCase.tearDown(self) + FilesystemMockingTestCase.tearDown(self) if self._log_handler and self._log: self._log.removeHandler(self._log_handler) apply_patches([i for i in reversed(self.unapply)]) @@ -166,7 +436,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): if dmi_data is None: dmi_data = DMI_DATA_RETURN - def _dmi_data(): + def _dmi_data(item): return dmi_data def _os_uname(): @@ -188,12 +458,12 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) self.apply_patches([ (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(mod, 'dmi_data', _dmi_data)]) + self.apply_patches([(util, 'read_dmi_data', _dmi_data)]) self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(mod, 'device_exists', lambda d: True)]) + self.apply_patches([(os.path, 'exists', lambda d: True)]) dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, paths=self.paths) - self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) + #self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) return dsrc def test_seed(self): @@ -492,7 +762,7 @@ def apply_patches(patches): return ret -class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): +class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): super(TestJoyentMetadataClient, self).setUp() @@ -532,7 +802,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataClient(self.serial) + return DataSourceSmartOS.JoyentMetadataSerialClient(self.serial) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -546,7 +816,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_writes_a_single_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(1, self.serial.write.call_count) written_line = self.serial.write.call_args[0][0] print(type(written_line)) @@ -556,7 +826,7 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def _get_written_line(self, key='some_key'): client = self._get_client() - client.get_metadata(key) + client.get(key) return self.serial.write.call_args[0][0] def test_get_metadata_writes_bytes(self): @@ -600,12 +870,12 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): def test_get_metadata_reads_a_line(self): client = self._get_client() - client.get_metadata('some_key') + client.get('some_key') self.assertEqual(self.metasource_data_len, self.serial.read.call_count) def test_get_metadata_returns_valid_value(self): client = self._get_client() - value = client.get_metadata('some_key') + value = client.get('some_key') self.assertEqual(self.metadata_value, value) def test_get_metadata_throws_exception_for_incorrect_length(self): @@ -633,4 +903,4 @@ class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): self.response_parts['length'] = 17 client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] - self.assertIsNone(client.get_metadata('some_key')) + self.assertIsNone(client.get('some_key')) -- cgit v1.2.3 From 8a148ed934156c63a76a28c9d3a33278e52d71d1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:27:50 -0400 Subject: fix the remaining tests --- cloudinit/sources/DataSourceSmartOS.py | 4 +- tests/unittests/test_datasource/test_smartos.py | 391 +----------------------- 2 files changed, 7 insertions(+), 388 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e64dea68..e9ff1235 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -332,11 +332,11 @@ class JoyentMetadataClient(object): r' (?P(?P[0-9a-f]+) (?PSUCCESS|NOTFOUND)' r'( (?P.+))?)') - def __init__(self, smartos_type=None): + def __init__(self, smartos_type=None, fp=None): if smartos_type is None: smartos_type = get_smartos_environ() self.smartos_type = smartos_type - self.fp = None + self.fp = fp def _checksum(self, body): return '{0:08x}'.format( diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 946286bd..56d85f99 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -105,17 +105,6 @@ MOCK_RETURNS = { DMI_DATA_RETURN = 'smartdc' -def get_mock_client(mockdata): - class MockMetadataClient(object): - - def __init__(self, serial): - pass - - def get(self, metadata_key): - return mockdata.get(metadata_key) - return MockMetadataClient - - class PsuedoJoyentClient(object): def __init__(self, data=None): if data is None: @@ -390,377 +379,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) -class OldTestSmartOSDataSource(FilesystemMockingTestCase): - def setUp(self): - super(TestSmartOSDataSource, self).setUp() - - self.tmp = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.tmp) - self.legacy_user_d = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, self.legacy_user_d) - - # If you should want to watch the logs... - self._log = None - self._log_file = None - self._log_handler = None - - # patch cloud_dir, so our 'seed_dir' is guaranteed empty - self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) - - self.unapply = [] - super(TestSmartOSDataSource, self).setUp() - - def tearDown(self): - FilesystemMockingTestCase.tearDown(self) - if self._log_handler and self._log: - self._log.removeHandler(self._log_handler) - apply_patches([i for i in reversed(self.unapply)]) - super(TestSmartOSDataSource, self).tearDown() - - def _patchIn(self, root): - self.restore() - self.patchOS(root) - self.patchUtils(root) - - def apply_patches(self, patches): - ret = apply_patches(patches) - self.unapply += ret - - def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None, - is_lxbrand=False): - mod = DataSourceSmartOS - - if mockdata is None: - mockdata = MOCK_RETURNS - - if dmi_data is None: - dmi_data = DMI_DATA_RETURN - - def _dmi_data(item): - return dmi_data - - def _os_uname(): - if not is_lxbrand: - # LP: #1243287. tests assume this runs, but running test on - # arm would cause them all to fail. - return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64') - else: - return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX', - 'X86_64') - - if sys_cfg is None: - sys_cfg = {} - - if ds_cfg is not None: - sys_cfg['datasource'] = sys_cfg.get('datasource', {}) - sys_cfg['datasource']['SmartOS'] = ds_cfg - - self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) - self.apply_patches([ - (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) - self.apply_patches([(util, 'read_dmi_data', _dmi_data)]) - self.apply_patches([(os, 'uname', _os_uname)]) - self.apply_patches([(os.path, 'exists', lambda d: True)]) - dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, - paths=self.paths) - #self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) - return dsrc - - def test_seed(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual('kvm', dsrc.smartos_type) - self.assertEqual('/dev/ttyS1', dsrc.seed) - - def test_seed_lxbrand(self): - # default seed should be /dev/ttyS1 - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual('lx-brand', dsrc.smartos_type) - self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed) - - def test_issmartdc(self): - dsrc = self._get_ds() - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_issmartdc_lxbrand(self): - dsrc = self._get_ds(is_lxbrand=True) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertTrue(dsrc.is_smartdc) - - def test_no_base64(self): - ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} - dsrc = self._get_ds(ds_cfg=ds_cfg) - ret = dsrc.get_data() - self.assertTrue(ret) - - def test_uuid(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['sdc:uuid'], - dsrc.metadata['instance-id']) - - def test_root_keys(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_hostname_b64(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_hostname(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - - def test_base64_all(self): - # metadata provided base64_all of true - my_returns = MOCK_RETURNS.copy() - my_returns['base64_all'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_b64_userdata(self): - my_returns = MOCK_RETURNS.copy() - my_returns['b64-cloud-init:user-data'] = "true" - my_returns['b64-hostname'] = "true" - for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - self.assertEqual(MOCK_RETURNS['root_authorized_keys'], - dsrc.metadata['public-keys']) - - def test_b64_keys(self): - my_returns = MOCK_RETURNS.copy() - my_returns['base64_keys'] = 'hostname,ignored' - for k in ('hostname',): - my_returns[k] = b64e(my_returns[k]) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['hostname'], - dsrc.metadata['local-hostname']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - - def test_userdata(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-data'], - dsrc.metadata['legacy-user-data']) - self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], - dsrc.userdata_raw) - - def test_sdc_nics(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEquals(MOCK_RETURNS['sdc:nics'], - dsrc.metadata['network-data']) - - def test_sdc_scripts(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, '700') - - def test_scripts_shebanged(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - shebang = None - with open(legacy_script_f, 'r') as f: - shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/bin/bash") - user_script_perm = oct(os.stat(legacy_script_f)[stat.ST_MODE])[-3:] - self.assertEqual(user_script_perm, '700') - - def test_scripts_shebang_not_added(self): - """ - Test that the SmartOS requirement that plain text scripts - are executable. This test makes sure that plain texts scripts - with out file magic have it added appropriately by cloud-init. - """ - - my_returns = MOCK_RETURNS.copy() - my_returns['user-script'] = '\n'.join(['#!/usr/bin/perl', - 'print("hi")', '']) - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(my_returns['user-script'], - dsrc.metadata['user-script']) - - legacy_script_f = "%s/user-script" % self.legacy_user_d - self.assertTrue(os.path.exists(legacy_script_f)) - self.assertTrue(os.path.islink(legacy_script_f)) - shebang = None - with open(legacy_script_f, 'r') as f: - shebang = f.readlines()[0].strip() - self.assertEqual(shebang, "#!/usr/bin/perl") - - def test_userdata_removed(self): - """ - User-data in the SmartOS world is supposed to be written to a file - each and every boot. This tests to make sure that in the event the - legacy user-data is removed, the existing user-data is backed-up - and there is no /var/db/user-data left. - """ - - user_data_f = "%s/mdata-user-data" % self.legacy_user_d - with open(user_data_f, 'w') as f: - f.write("PREVIOUS") - - my_returns = MOCK_RETURNS.copy() - del my_returns['user-data'] - - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertFalse(dsrc.metadata.get('legacy-user-data')) - - found_new = False - for root, _dirs, files in os.walk(self.legacy_user_d): - for name in files: - name_f = os.path.join(root, name) - permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:] - if re.match(r'.*\/mdata-user-data$', name_f): - found_new = True - print(name_f) - self.assertEqual(permissions, '400') - - self.assertFalse(found_new) - - def test_vendor_data_not_default(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['sdc:vendor-data'], - dsrc.metadata['vendor-data']) - - def test_default_vendor_data(self): - my_returns = MOCK_RETURNS.copy() - def_op_script = my_returns['sdc:vendor-data'] - del my_returns['sdc:vendor-data'] - dsrc = self._get_ds(mockdata=my_returns) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertNotEqual(def_op_script, dsrc.metadata['vendor-data']) - - # we expect default vendor-data is a boothook - self.assertTrue(dsrc.vendordata_raw.startswith("#cloud-boothook")) - - def test_disable_iptables_flag(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], - dsrc.metadata['iptables_disable']) - - def test_motd_sys_info(self): - dsrc = self._get_ds(mockdata=MOCK_RETURNS) - ret = dsrc.get_data() - self.assertTrue(ret) - self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], - dsrc.metadata['motd_sys_info']) - - def test_default_ephemeral(self): - # Test to make sure that the builtin config has the ephemeral - # configuration. - dsrc = self._get_ds() - cfg = dsrc.get_config_obj() - - ret = dsrc.get_data() - self.assertTrue(ret) - - assert 'disk_setup' in cfg - assert 'fs_setup' in cfg - self.assertIsInstance(cfg['disk_setup'], dict) - self.assertIsInstance(cfg['fs_setup'], list) - - def test_override_disk_aliases(self): - # Test to make sure that the built-in DS is overriden - builtin = DataSourceSmartOS.BUILTIN_DS_CONFIG - - mydscfg = {'disk_aliases': {'FOO': '/dev/bar'}} - - # expect that these values are in builtin, or this is pointless - for k in mydscfg: - self.assertIn(k, builtin) - - dsrc = self._get_ds(ds_cfg=mydscfg) - ret = dsrc.get_data() - self.assertTrue(ret) - - self.assertEqual(mydscfg['disk_aliases']['FOO'], - dsrc.ds_cfg['disk_aliases']['FOO']) - - self.assertEqual(dsrc.device_name_to_device('FOO'), - mydscfg['disk_aliases']['FOO']) - - -def apply_patches(patches): - ret = [] - for (ref, name, replace) in patches: - if replace is None: - continue - orig = getattr(ref, name) - setattr(ref, name, replace) - ret.append((ref, name, orig)) - return ret - class TestJoyentMetadataClient(FilesystemMockingTestCase): @@ -802,7 +420,8 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): mock.Mock(return_value=self.request_id))) def _get_client(self): - return DataSourceSmartOS.JoyentMetadataSerialClient(self.serial) + return DataSourceSmartOS.JoyentMetadataClient( + fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM) def assertEndsWith(self, haystack, prefix): self.assertTrue(haystack.endswith(prefix), @@ -882,20 +501,20 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): self.response_parts['length'] = 0 client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_incorrect_crc(self): self.response_parts['crc'] = 'deadbeef' client = self._get_client() self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_throws_exception_for_request_id_mismatch(self): self.response_parts['request_id'] = 'deadbeef' client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, - client.get_metadata, 'some_key') + client.get, 'some_key') def test_get_metadata_returns_None_if_value_not_found(self): self.response_parts['payload'] = '' -- cgit v1.2.3 From 91734552a2d338938ed0e3aa4885f77b99409ead Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:32:43 -0400 Subject: fix pyflakes and flake8 --- cloudinit/sources/DataSourceSmartOS.py | 3 +-- tests/unittests/test_datasource/test_smartos.py | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e9ff1235..f5820696 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -722,8 +722,7 @@ def convert_smartos_network_data(network_data=None): if k in valid_keys['physical']} cfg.update({ 'type': 'physical', - 'name': nic['interface'] - }) + 'name': nic['interface']}) if 'mac' in nic: cfg.update({'mac_address': nic['mac']}) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 56d85f99..ae45513d 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -40,9 +40,8 @@ import six from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e -from cloudinit import util -from ..helpers import mock, TestCase, FilesystemMockingTestCase +from ..helpers import mock, FilesystemMockingTestCase SDC_NICS = json.loads(""" [ @@ -111,7 +110,7 @@ class PsuedoJoyentClient(object): data = MOCK_RETURNS.copy() self.data = data return - + def get(self, key, default=None, strip=False): if key in self.data: r = self.data[key] @@ -169,10 +168,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): return DataSourceSmartOS.DataSourceSmartOS( sys_cfg, distro=None, paths=self.paths) - - def test_it_got_here(self): - dsrc = self._get_ds() - ret = dsrc.get_data() def test_no_base64(self): ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} @@ -379,7 +374,6 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): mydscfg['disk_aliases']['FOO']) - class TestJoyentMetadataClient(FilesystemMockingTestCase): def setUp(self): -- cgit v1.2.3 From 71e4da45263e6cb3eb5d5938908656ed04c3db9f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:33:20 -0400 Subject: remove debug print --- cloudinit/sources/DataSourceSmartOS.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f5820696..9c249ddf 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -609,7 +609,6 @@ def write_boot_content(content, content_f, link=None, shebang=False, bit and to the SmartOS default of assuming that bash. """ - print("content_f=%s" % content_f) if not content and os.path.exists(content_f): os.unlink(content_f) if link and os.path.islink(link): -- cgit v1.2.3 From a9533cd924e8eae89234a19d8359a87c23a30e12 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:52:13 -0400 Subject: add nicer main --- cloudinit/sources/DataSourceSmartOS.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9c249ddf..3d7297c9 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -755,4 +755,23 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() - jmc.get(sys.argv[1]) + if len(sys.argv) == 1: + keys = (list(SMARTOS_ATTRIB_JSON.keys()) + + list(SMARTOS_ATTRIB_MAP.keys())) + else: + keys = sys.argv[1:] + + data = {} + for key in keys: + if key in SMARTOS_ATTRIB_JSON: + keyname = SMARTOS_ATTRIB_JSON[key] + data[key] = jmc.get_json(keyname) + else: + if key in SMARTOS_ATTRIB_MAP: + keyname, strip = SMARTOS_ATTRIB_MAP[key] + else: + keyname, strip = (key, False) + val = jmc.get(keyname, strip=strip) + data[key] = jmc.get(keyname, strip=strip) + + print(json.dumps(data, indent=1)) -- cgit v1.2.3 From 9a43053c564210aa088f96ab7877a66a1c3b48fa Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 14:55:52 -0400 Subject: Smartos datasource is local. --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 3d7297c9..24331d37 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -743,7 +743,7 @@ def convert_smartos_network_data(network_data=None): # Used to match classes to dependencies datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), ] -- cgit v1.2.3 From 7ab03cab899c5bd355c24a4b894c74d63c6dd8b6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:03:08 -0400 Subject: smartos is local, but it is named DataSourceSmartOS not DataSourceConfigDrive --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 24331d37..2821402a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -743,7 +743,7 @@ def convert_smartos_network_data(network_data=None): # Used to match classes to dependencies datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), ] -- cgit v1.2.3 From c0788ee76bf62fa0be570932273ff47212bc9d6c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:07:54 -0400 Subject: add a unit test for conversion --- tests/unittests/test_datasource/test_smartos.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index ae45513d..10e3113e 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -41,7 +41,7 @@ from cloudinit import helpers as c_helpers from cloudinit.sources import DataSourceSmartOS from cloudinit.util import b64e -from ..helpers import mock, FilesystemMockingTestCase +from ..helpers import mock, FilesystemMockingTestCase, TestCase SDC_NICS = json.loads(""" [ @@ -517,3 +517,23 @@ class TestJoyentMetadataClient(FilesystemMockingTestCase): client = self._get_client() client._checksum = lambda _: self.response_parts['crc'] self.assertIsNone(client.get('some_key')) + + +class TestNetworkConversion(TestCase): + + def test_convert_simple(self): + expected = { + 'version': 1, + 'config': [ + {'name': 'net0', 'type': 'physical', + 'subnets': [{'type': 'static', 'gateway': '8.12.42.1', + 'netmask': '255.255.255.0', + 'address': '8.12.42.102/24'}], + 'mtu': 1500, 'mac_address': '90:b8:d0:f5:e4:f5'}, + {'name': 'net1', 'type': 'physical', + 'subnets': [{'type': 'static', 'gateway': '192.168.128.1', + 'netmask': '255.255.252.0', + 'address': '192.168.128.93/22'}], + 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]} + found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS) + self.assertEqual(expected, found) -- cgit v1.2.3 From f5be39ff385b5c1e15f536e3f05415c4b95ff1f3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:16:33 -0400 Subject: assertEquals --- tests/unittests/test_datasource/test_smartos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 10e3113e..96d7346f 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -536,4 +536,4 @@ class TestNetworkConversion(TestCase): 'address': '192.168.128.93/22'}], 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]} found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS) - self.assertEqual(expected, found) + self.assertEquals(expected, found) -- cgit v1.2.3 From 75109ffd12ad4dcbbccf1b0603506efcae413433 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:31:52 -0400 Subject: return dict not None on get_config_obj --- cloudinit/sources/DataSourceSmartOS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 2821402a..7c8928b3 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -302,7 +302,7 @@ class DataSourceSmartOS(sources.DataSource): def get_config_obj(self): if self.smartos_type == SMARTOS_ENV_KVM: return BUILTIN_CLOUD_CONFIG - return None + return {} def get_instance_id(self): return self.metadata['instance-id'] -- cgit v1.2.3 From 3f5b6386e80e0b31e6adcb7db1859a115a3184df Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 15:57:45 -0400 Subject: assertEqual --- tests/unittests/test_datasource/test_smartos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 96d7346f..9eb06f5f 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -216,7 +216,7 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(json.loads(MOCK_RETURNS['sdc:nics']), + self.assertEqual(json.loads(MOCK_RETURNS['sdc:nics']), dsrc.metadata['network-data']) def test_sdc_scripts(self): @@ -536,4 +536,4 @@ class TestNetworkConversion(TestCase): 'address': '192.168.128.93/22'}], 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]} found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS) - self.assertEquals(expected, found) + self.assertEqual(expected, found) -- cgit v1.2.3 From 653676831e70192d4a0322ee453d6a4c3e6541da Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 16:57:18 -0400 Subject: fix test cases by avoiding rendering paths to early --- cloudinit/stages.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 20c334b7..f164d6f6 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -69,8 +69,6 @@ class Init(object): self.datasource = NULL_DATA_SOURCE self.ds_restored = False self._previous_iid = None - # simply ensure this gets set - self.previous_iid() if reporter is None: reporter = events.ReportEventStack( @@ -321,12 +319,8 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() iid_fn = os.path.join(dp, 'instance-id') - try: - previous_iid = util.load_file(iid_fn).strip() - except Exception: - previous_iid = None - if not previous_iid: - previous_iid = iid + + previous_iid = self.previous_iid() util.write_file(iid_fn, "%s\n" % iid) util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), @@ -349,7 +343,8 @@ class Init(object): self._previous_iid = util.load_file(iid_fn).strip() except Exception: self._previous_iid = NO_PREVIOUS_INSTANCE_ID - pass + + LOG.debug("previous iid found to be %s", self._previous_iid) return self._previous_iid def is_new_instance(self): -- cgit v1.2.3 From b16533714e02fc71d463ca73abee5c9f6237f115 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 27 May 2016 17:03:49 -0400 Subject: remove blocking udev functionality This didn't really work. See bug for more info. LP: #1577844 --- setup.py | 1 - systemd/cloud-init-generator | 3 -- udev/79-cloud-init-net-wait.rules | 10 ------ udev/cloud-init-wait | 70 --------------------------------------- 4 files changed, 84 deletions(-) delete mode 100644 udev/79-cloud-init-net-wait.rules delete mode 100755 udev/cloud-init-wait diff --git a/setup.py b/setup.py index db46eae9..57d946ca 100755 --- a/setup.py +++ b/setup.py @@ -184,7 +184,6 @@ else: (USR + '/share/doc/cloud-init/examples/seed', [f for f in glob('doc/examples/seed/*') if is_f(f)]), (LIB + '/udev/rules.d', [f for f in glob('udev/*.rules')]), - (LIB + '/udev', ['udev/cloud-init-wait']), ] # Use a subclass for install that handles # adding on the right init system configuration files diff --git a/systemd/cloud-init-generator b/systemd/cloud-init-generator index ae286d58..2d319695 100755 --- a/systemd/cloud-init-generator +++ b/systemd/cloud-init-generator @@ -107,9 +107,6 @@ main() { "ln $CLOUD_SYSTEM_TARGET $link_path" fi fi - # this touches /run/cloud-init/enabled, which is read by - # udev/cloud-init-wait. If not present, it will exit quickly. - touch "$LOG_D/$ENABLE" elif [ "$result" = "$DISABLE" ]; then if [ -f "$link_path" ]; then if rm -f "$link_path"; then diff --git a/udev/79-cloud-init-net-wait.rules b/udev/79-cloud-init-net-wait.rules deleted file mode 100644 index 8344222a..00000000 --- a/udev/79-cloud-init-net-wait.rules +++ /dev/null @@ -1,10 +0,0 @@ -# cloud-init cold/hot-plug blocking mechanism -# this file blocks further processing of network events -# until cloud-init local has had a chance to read and apply network -SUBSYSTEM!="net", GOTO="cloudinit_naming_end" -ACTION!="add", GOTO="cloudinit_naming_end" - -IMPORT{program}="/lib/udev/cloud-init-wait" - -LABEL="cloudinit_naming_end" -# vi: ts=4 expandtab syntax=udevrules diff --git a/udev/cloud-init-wait b/udev/cloud-init-wait deleted file mode 100755 index b434005d..00000000 --- a/udev/cloud-init-wait +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/sh - -CI_NET_READY="/run/cloud-init/network-config-ready" -LOG="/run/cloud-init/${0##*/}.log" -LOG_INIT=0 -MAX_WAIT=60 -DEBUG=0 - -block_until_ready() { - local fname="$1" max="$2" - [ -f "$fname" ] && return 0 - # udevadm settle below will exit at the first of 3 conditions - # 1.) timeout 2.) file exists 3.) all in-flight udev events are processed - # since this is being run from a udev event, the 3 wont happen. - # thus, this is essentially a inotify wait or timeout on a file in /run - # that is created by cloud-init-local. - udevadm settle "--timeout=$max" "--exit-if-exists=$fname" -} - -log() { - [ -n "${LOG}" ] || return - [ "${DEBUG:-0}" = "0" ] && return - - if [ $LOG_INIT = 0 ]; then - if [ -d "${LOG%/*}" ] || mkdir -p "${LOG%/*}"; then - LOG_INIT=1 - else - echo "${0##*/}: WARN: log init to ${LOG%/*}" 1>&2 - return - fi - elif [ "$LOG_INIT" = "-1" ]; then - return - fi - local info="$$ $INTERFACE" - if [ "$DEBUG" -gt 1 ]; then - local up idle - read up idle < /proc/uptime - info="$$ $INTERFACE $up" - fi - echo "[$info]" "$@" >> "$LOG" -} - -main() { - local name="" readyfile="$CI_NET_READY" - local info="INTERFACE=${INTERFACE} ID_NET_NAME=${ID_NET_NAME}" - info="$info ID_NET_NAME_PATH=${ID_NET_NAME_PATH}" - info="$info MAC_ADDRESS=${MAC_ADDRESS}" - log "$info" - - ## Check to see if cloud-init.target is set. If cloud-init is - ## disabled we do not want to do anything. - if [ ! -f "/run/cloud-init/enabled" ]; then - log "cloud-init disabled" - return 0 - fi - - if [ "${INTERFACE#lo}" != "$INTERFACE" ]; then - return 0 - fi - - block_until_ready "$readyfile" "$MAX_WAIT" || - { log "failed waiting for ready on $INTERFACE"; return 1; } - - log "net config ready" -} - -main "$@" -exit - -# vi: ts=4 expandtab -- cgit v1.2.3 From aa236033b159b691f5ec31885750a8167c63b2a1 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:27:40 +0200 Subject: drop errorlist from convert_to_new_format --- cloudinit/config/cc_apt_configure.py | 12 +++--------- tests/unittests/test_handler/test_handler_apt_source.py | 6 ++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index ffbf7513..2dd48844 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -198,24 +198,18 @@ def add_key(ent): add_key_raw(ent['key']) -def convert_to_new_format(srclist, errorlist): +def convert_to_new_format(srclist): """ convert_to_new_format convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): - fnfallbackused = None for srcent in srclist: if 'filename' not in srcent: # file collides for multiple !filename cases for compatibility # yet we need them all processed, so not same dictionary key srcent['filename'] = "cloud_config_sources.list" key = util.rand_dict_key(srcdict, "cloud_config_sources.list") - if fnfallbackused is not None: - errorlist.append(["multiple apt_source entries without", - "filename will conflict: %s vs %s" % - (srcent, fnfallbackused)]) - fnfallbackused = srcent else: # all with filename use that as key (matching new format) key = srcent['filename'] @@ -223,7 +217,7 @@ def convert_to_new_format(srclist, errorlist): elif isinstance(srclist, dict): srcdict = srclist else: - errorlist.append(["srclist", "unknown apt_sources format"]) + raise ValueError("unknown apt_sources format") return srcdict @@ -242,7 +236,7 @@ def add_sources(srclist, template_params=None, aa_repo_match=None): return False errorlist = [] - srcdict = convert_to_new_format(srclist, errorlist) + srcdict = convert_to_new_format(srclist) for filename in srcdict: ent = srcdict[filename] diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 753e86a8..4536c5b2 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -529,7 +529,6 @@ class TestAptSourceConfig(TestCase): 'filename': self.aptlistfile2} cfg3 = {'source': 'deb $MIRROR $RELEASE universe', 'filename': self.aptlistfile3} - errorlist = [] checkcfg = {self.aptlistfile: {'filename': self.aptlistfile, 'source': 'deb $MIRROR $RELEASE ' 'multiverse'}, @@ -539,11 +538,10 @@ class TestAptSourceConfig(TestCase): 'source': 'deb $MIRROR $RELEASE ' 'universe'}} - newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3], - errorlist) + newcfg = cc_apt_configure.convert_to_new_format([cfg1, cfg2, cfg3]) self.assertEqual(newcfg, checkcfg) - newcfg2 = cc_apt_configure.convert_to_new_format(newcfg, errorlist) + newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) self.assertEqual(newcfg2, checkcfg) -- cgit v1.2.3 From c7b9b2b04490fda948631eecdba1bb678f0e4db9 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:48:00 +0200 Subject: add test for wrong apt_source format --- tests/unittests/test_handler/test_handler_apt_source.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index 4536c5b2..fe2ffae5 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -544,5 +544,8 @@ class TestAptSourceConfig(TestCase): newcfg2 = cc_apt_configure.convert_to_new_format(newcfg) self.assertEqual(newcfg2, checkcfg) + with self.assertRaises(ValueError): + cc_apt_configure.convert_to_new_format(5) + # vi: ts=4 expandtab -- cgit v1.2.3 From 964ec3ae45c27cf55e0c1349138294ff11debab8 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 12:53:29 +0200 Subject: improve wording in the examples --- doc/examples/cloud-config.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index df59ff57..62b297bc 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -220,7 +220,7 @@ apt_sources: -----END PGP PUBLIC KEY BLOCK----- # Custom gpg key: - # * As the keyid also a key can be specified withut a related source + # * As with keyid, a key may also be specified without a related source. # * all other facts mentioned above still apply # since this doesn't generate a source.list file the filename key is ignored ignored4: -- cgit v1.2.3 From 3c85315373306729443ef79fd8e54af46a7bc849 Mon Sep 17 00:00:00 2001 From: Christian Ehrhardt Date: Mon, 30 May 2016 13:07:22 +0200 Subject: fix EXPORT_GPG_KEYID for existing keys This was broken for keys already existing in the local keyring. There instead of the keycontent it reported the header like: pub 1024R/03683F77 2009-10-27 uid Launchpad PPA for Scott Moser --- cloudinit/config/cc_apt_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 2dd48844..d603f417 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -40,7 +40,7 @@ EXPORT_GPG_KEYID = """ k=${1} ks=${2}; exec 2>/dev/null [ -n "$k" ] || exit 1; - armour=$(gpg --list-keys --armour "${k}") + armour=$(gpg --export --armour "${k}") if [ -z "${armour}" ]; then gpg --keyserver ${ks} --recv "${k}" >/dev/null && armour=$(gpg --export --armour "${k}") && -- cgit v1.2.3 From 0ce31b75247458b735b1b52dd5985519190d48fb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 May 2016 13:14:17 -0400 Subject: use constants for kvm and lx-brand --- cloudinit/sources/DataSourceSmartOS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 7c8928b3..d6a9bf28 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -577,11 +577,11 @@ def jmc_client_factory( if smartos_type is None: smartos_type = get_smartos_environ(uname_version) - if smartos_type == 'kvm': + if smartos_type == SMARTOS_ENV_KVM: return JoyentMetadataLegacySerialClient( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) - elif smartos_type == 'lx-brand': + elif smartos_type == SMARTOS_ENV_LX_BRAND: return JoyentMetadataSocketClient(socketpath=metadata_sockfile) raise ValueError("Unknown value for smartos_type: %s" % smartos_type) -- cgit v1.2.3 From 1b8a09389654a29af7e618b803bffaed0185e9e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 31 May 2016 17:17:39 -0400 Subject: add renaming code for renaming interfaces currently does not work in lxc https://github.com/lxc/lxd/issues/2063 --- bin/cloud-init | 6 +-- cloudinit/distros/__init__.py | 4 ++ cloudinit/net/__init__.py | 86 +++++++++++++++++++++++++++++++++++++++++++ cloudinit/stages.py | 10 +++++ 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/bin/cloud-init b/bin/cloud-init index 29e9b521..21c3a684 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -280,11 +280,7 @@ def main_init(name, args): LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s", mode, name, iid, init.is_new_instance()) - if init.is_new_instance(): - # on new instance, apply network config. - # in network mode 'bring_up' must be passed in as the OS - # has already brought up networking. - init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) + init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL)) if mode == sources.DSMODE_LOCAL: if init.datasource.dsmode != mode: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 3bfbc484..5c29c804 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -31,6 +31,7 @@ import stat from cloudinit import importer from cloudinit import log as logging +from cloudinit import net from cloudinit import ssh_util from cloudinit import type_utils from cloudinit import util @@ -145,6 +146,9 @@ class Distro(object): return self._bring_up_interfaces(dev_names) return False + def apply_network_config_names(self, netconfig): + net.apply_network_config_names(netconfig) + @abc.abstractmethod def apply_locale(self, locale, out_fn=None): raise NotImplementedError() diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 40d330b5..ec1b3835 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -813,4 +813,90 @@ def _ifaces_to_net_config_data(ifaces): 'config': [devs[d] for d in sorted(devs)]} +def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): + """read the network config and rename devices accordingly. + if strict_present is false, then do not raise exception if no devices + match. if strict_busy is false, then do not raise exception if the + device cannot be renamed because it is currently configured.""" + renames = [] + for ent in netcfg.get('config', {}): + if ent.get('type') != 'physical': + continue + mac = ent.get('mac_address') + name = ent.get('name') + if not mac: + continue + renames.append([mac, name]) + + return rename_interfaces(renames) + + +def rename_interfaces(renames, strict_present=True, strict_busy=True): + cur_bymac = {get_interface_mac(n): n for n in get_devicelist()} + expected = {mac: name for mac, name in renames} + cur_byname = {v: k for k, v in cur_bymac.items()} + + tmpname_fmt = "cirename%d" + tmpi = -1 + + moves = [] + changes = [] + errors = [] + for mac, new_name in expected.items(): + cur_name = cur_bymac.get(mac) + if cur_name == new_name: + # nothing to do + continue + + if not cur_name: + if strict_present: + errors.append( + "[nic not present] Cannot rename mac=%s to %s" + ", not available." % (mac, new_name)) + elif is_up(cur_name): + if strict_busy: + errors.append("[busy] Error renaming mac=%s from %s to %s." % + (mac, cur_name, new_name)) + elif new_name in cur_byname: + if is_up(new_name): + if strict_busy: + errors.append( + "[busy-target] Error renaming mac=%s from %s to %s." % + (mac, cur_name, new_name)) + else: + tmp_name = None + while tmp_name is None or tmp_name in cur_byname: + tmpi += 1 + tmp_name = tmpname_fmt % tmpi + moves.append((mac, cur_name, tmp_name)) + changes.append((mac, tmp_name, new_name)) + else: + changes.append((mac, cur_name, new_name)) + + def rename_dev(cur, new): + cmd = ["ip", "link", "set", cur, "name", new] + util.subp(cmd) + + for mac, cur, new in moves + changes: + try: + rename_dev(cur, new) + except util.ProcessExecutionError as e: + errors.append( + "[unknown] Error renaming mac=%s from %s to %s. (%s)" % + (mac, cur, new, e)) + + if len(errors): + raise Exception('\n'.join(errors)) + + +def get_interface_mac(ifname): + """Returns the string value of an interface's MAC Address""" + return read_sys_net(ifname, "address", enoent=False) + + +def get_ifname_mac_pairs(): + """Build a list of tuples (ifname, mac)""" + return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()] + + # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/stages.py b/cloudinit/stages.py index f164d6f6..211d2286 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -632,6 +632,16 @@ class Init(object): LOG.info("network config is disabled by %s", src) return + try: + LOG.debug("applying net config names for %s" % netcfg) + self.distro.apply_network_config_names(netcfg) + except Exception as e: + LOG.warn("Failed to rename devices: %s", e) + + if not self.is_new_instance(): + LOG.debug("not a new instance. network config is not applied.") + return + LOG.info("Applying network configuration from %s bringup=%s: %s", src, bring_up, netcfg) try: -- cgit v1.2.3 From f67e8f104b199c9402cf047637b939516526e0ac Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 17:14:50 -0400 Subject: support renaming and fix logic in rename_interfaces The one issue i'm aware of currently is that tap devices (ip tuntap add mode tap user root mytap1) do not work correctly with 'is_up' which means the check does not bring them down and the rename fails. The LOG.debug message should be cleaned up too, as it currently references the function rather function.__name__ for nicer message. --- cloudinit/net/__init__.py | 133 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 98 insertions(+), 35 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ec1b3835..f5ae7705 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -831,19 +831,65 @@ def apply_network_config_names(netcfg, strict_present=True, strict_busy=True): return rename_interfaces(renames) -def rename_interfaces(renames, strict_present=True, strict_busy=True): - cur_bymac = {get_interface_mac(n): n for n in get_devicelist()} - expected = {mac: name for mac, name in renames} - cur_byname = {v: k for k, v in cur_bymac.items()} +def _get_current_rename_info(check_downable=True): + """Collect information necessary for rename_interfaces.""" + names = get_devicelist() + bymac = {} + for n in names: + bymac[get_interface_mac(n)] = { + 'name': n, 'up': is_up(n), 'downable': None} + if check_downable: + nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]") + ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent', + 'scope', 'global'], capture=True) + ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True) + + nics_with_addresses = set() + for bytes_out in (ipv6, ipv4): + nics_with_addresses.update(nmatch.findall(bytes_out)) + + for d in bymac.values(): + d['downable'] = (d['up'] is False or + d['name'] not in nics_with_addresses) + + return bymac + + +def rename_interfaces(renames, strict_present=True, strict_busy=True, + current_info=None): + if current_info is None: + current_info = _get_current_rename_info() + + cur_bymac = {} + for mac, data in current_info.items(): + cur = data.copy() + cur['mac'] = mac + cur_bymac[mac] = cur + + def update_byname(bymac): + return {data['name']: data for data in bymac.values()} + + def rename(cur, new): + util.subp(["ip", "link", "set", cur, "name", new], capture=True) + + def down(name): + util.subp(["ip", "link", "set", name, "down"], capture=True) + + def up(name): + util.subp(["ip", "link", "set", name, "up"], capture=True) + + ops = [] + errors = [] + ups = [] + cur_byname = update_byname(cur_bymac) tmpname_fmt = "cirename%d" tmpi = -1 - moves = [] - changes = [] - errors = [] - for mac, new_name in expected.items(): - cur_name = cur_bymac.get(mac) + for mac, new_name in renames: + cur = cur_bymac.get(mac, {}) + cur_name = cur.get('name') + cur_ops = [] if cur_name == new_name: # nothing to do continue @@ -853,37 +899,54 @@ def rename_interfaces(renames, strict_present=True, strict_busy=True): errors.append( "[nic not present] Cannot rename mac=%s to %s" ", not available." % (mac, new_name)) - elif is_up(cur_name): - if strict_busy: - errors.append("[busy] Error renaming mac=%s from %s to %s." % - (mac, cur_name, new_name)) - elif new_name in cur_byname: - if is_up(new_name): + continue + + if cur['up']: + msg = "[busy] Error renaming mac=%s from %s to %s" + if not cur['downable']: if strict_busy: - errors.append( - "[busy-target] Error renaming mac=%s from %s to %s." % - (mac, cur_name, new_name)) - else: - tmp_name = None - while tmp_name is None or tmp_name in cur_byname: - tmpi += 1 - tmp_name = tmpname_fmt % tmpi - moves.append((mac, cur_name, tmp_name)) - changes.append((mac, tmp_name, new_name)) - else: - changes.append((mac, cur_name, new_name)) + errors.append(msg % (mac, cur_name, new_name)) + continue + cur['up'] = False + cur_ops.append((down, mac, new_name, (cur_name,))) + ups.append((up, mac, new_name, (new_name,))) + + if new_name in cur_byname: + target = cur_byname[new_name] + if target['up']: + msg = "[busy-target] Error renaming mac=%s from %s to %s." + if not target['downable']: + if strict_busy: + errors.append(msg % (mac, cur_name, new_name)) + continue + else: + cur_ops.append((down, mac, new_name, (new_name,))) + + tmp_name = None + while tmp_name is None or tmp_name in cur_byname: + tmpi += 1 + tmp_name = tmpname_fmt % tmpi + + cur_ops.append((rename, mac, new_name, (new_name, tmp_name))) + target['name'] = tmp_name + cur_byname = update_byname(cur_bymac) + if target['up']: + ups.append((up, mac, new_name, (tmp_name,))) + + cur_ops.append((rename, mac, new_name, (cur['name'], new_name))) + cur['name'] = new_name + cur_byname = update_byname(cur_bymac) + ops += cur_ops - def rename_dev(cur, new): - cmd = ["ip", "link", "set", cur, "name", new] - util.subp(cmd) + LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) - for mac, cur, new in moves + changes: + for op, mac, new_name, params in ops + ups: try: - rename_dev(cur, new) - except util.ProcessExecutionError as e: + op(*params) + except Exception as e: errors.append( - "[unknown] Error renaming mac=%s from %s to %s. (%s)" % - (mac, cur, new, e)) + "[unknown] Error performing %s%s for %s, %s: %s" % + (op.__name__, params, mac, new_name, e)) if len(errors): raise Exception('\n'.join(errors)) -- cgit v1.2.3 From 0fc5c222d2ff335b9a1de3e910bb8822b6d73031 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 17:17:11 -0400 Subject: cloudinit/stages.py: if no datasource found, do not attempt is_new_instance if local does not find a datasource, then we try to apply networking. but that would then hit the NULL_DATA_SOURCE which does not work with is_new_instance. avoid that. --- cloudinit/stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 211d2286..5756e74d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -638,7 +638,8 @@ class Init(object): except Exception as e: LOG.warn("Failed to rename devices: %s", e) - if not self.is_new_instance(): + if (self.datasource is not NULL_DATA_SOURCE and + not self.is_new_instance()): LOG.debug("not a new instance. network config is not applied.") return -- cgit v1.2.3 From 7f46de87ee543a82c9a95137478676edaba2acc1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 20:23:00 -0400 Subject: clean up log message a bit. --- cloudinit/net/__init__.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f5ae7705..6d9ea575 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -908,8 +908,8 @@ def rename_interfaces(renames, strict_present=True, strict_busy=True, errors.append(msg % (mac, cur_name, new_name)) continue cur['up'] = False - cur_ops.append((down, mac, new_name, (cur_name,))) - ups.append((up, mac, new_name, (new_name,))) + cur_ops.append(("down", mac, new_name, (cur_name,))) + ups.append(("up", mac, new_name, (new_name,))) if new_name in cur_byname: target = cur_byname[new_name] @@ -920,33 +920,41 @@ def rename_interfaces(renames, strict_present=True, strict_busy=True, errors.append(msg % (mac, cur_name, new_name)) continue else: - cur_ops.append((down, mac, new_name, (new_name,))) + cur_ops.append(("down", mac, new_name, (new_name,))) tmp_name = None while tmp_name is None or tmp_name in cur_byname: tmpi += 1 tmp_name = tmpname_fmt % tmpi - cur_ops.append((rename, mac, new_name, (new_name, tmp_name))) + cur_ops.append(("rename", mac, new_name, (new_name, tmp_name))) target['name'] = tmp_name cur_byname = update_byname(cur_bymac) if target['up']: - ups.append((up, mac, new_name, (tmp_name,))) + ups.append(("up", mac, new_name, (tmp_name,))) - cur_ops.append((rename, mac, new_name, (cur['name'], new_name))) + cur_ops.append(("rename", mac, new_name, (cur['name'], new_name))) cur['name'] = new_name cur_byname = update_byname(cur_bymac) ops += cur_ops - LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) + opmap = {'rename': rename, 'down': down, 'up': up} - for op, mac, new_name, params in ops + ups: - try: - op(*params) - except Exception as e: - errors.append( - "[unknown] Error performing %s%s for %s, %s: %s" % - (op.__name__, params, mac, new_name, e)) + if len(ops) + len(ups) == 0: + if len(errors): + LOG.debug("unable to do any work for renaming of %s", renames) + else: + LOG.debug("no work necessary for renaming of %s", renames) + else: + LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups) + + for op, mac, new_name, params in ops + ups: + try: + opmap.get(op)(*params) + except Exception as e: + errors.append( + "[unknown] Error performing %s%s for %s, %s: %s" % + (op, params, mac, new_name, e)) if len(errors): raise Exception('\n'.join(errors)) -- cgit v1.2.3 From 654bd419145f079386b47fe3ec59afdd68fc5080 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 1 Jun 2016 20:51:03 -0400 Subject: revert unintended change to tox.ini --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 18d059df..dafaaf6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] envlist = py27,py3,flake8 -recreate = False -skip_install = True +recreate = True [testenv] commands = python -m nose {posargs:tests} -- cgit v1.2.3 From 7c27cb1fcae01d34b32895630d57b2c9bc624caf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 09:47:29 -0400 Subject: fix log message in emit_upstart --- cloudinit/config/cc_emit_upstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index 06c53272..98828b9e 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -56,7 +56,7 @@ def handle(name, _cfg, cloud, log, args): event_names = ['cloud-config'] if not is_upstart_system(): - log.debug("not upstart system, '%s' disabled") + log.debug("not upstart system, '%s' disabled", name) return cfgpath = cloud.paths.get_ipath_cur("cloud_config") -- cgit v1.2.3 From 5b208a557c2878d0cd1b9b681984a9927de5d6cb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 10:52:53 -0400 Subject: fix tox --- tests/unittests/test_datasource/test_smartos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 9eb06f5f..c809117b 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -217,7 +217,7 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): ret = dsrc.get_data() self.assertTrue(ret) self.assertEqual(json.loads(MOCK_RETURNS['sdc:nics']), - dsrc.metadata['network-data']) + dsrc.metadata['network-data']) def test_sdc_scripts(self): dsrc = self._get_ds(mockdata=MOCK_RETURNS) -- cgit v1.2.3 From 75907ec96181599dc0282aac989ce02a88c3b39d Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Thu, 2 Jun 2016 16:20:18 +0100 Subject: Integrate MP feedback --- doc/merging.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/merging.rst b/doc/merging.rst index 2967ca9d..afe1a6dd 100644 --- a/doc/merging.rst +++ b/doc/merging.rst @@ -190,5 +190,5 @@ used the previous merging are also, similarly, now extensible (metadata merging, for example). Note, however, that merge algorithms are not used *across* types of -configuration. As was previously the case, merged user-data will still -overwrite conf.d configuration. +configuration. As was the case before merging was implemented, +user-data will overwrite conf.d configuration without merging. -- cgit v1.2.3 From b071e4bbbbe1b5a6ced02796696b05d2e1b16778 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 13:18:23 -0400 Subject: openstack: support decoding when reading files, use that for network_config The network config file is /etc/network/interfaces formated. We will decode that here so that the user can expect that it is a string. The issue was that it was bytes but convert_eni_data was expecting a string. --- cloudinit/sources/helpers/openstack.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 156aba6c..3ccb11d3 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -190,14 +190,14 @@ class BaseReader(object): versions_available) return selected_version - def _read_content_path(self, item): + def _read_content_path(self, item, decode=False): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") valid_pieces = [p for p in path_pieces if len(p)] if not valid_pieces: raise BrokenMetadata("Item %s has no valid content path" % (item)) path = self._path_join(self.base_path, "openstack", *path_pieces) - return self._path_read(path) + return self._path_read(path, decode=decode) def read_v2(self): """Reads a version 2 formatted location. @@ -298,7 +298,8 @@ class BaseReader(object): net_item = metadata.get("network_config", None) if net_item: try: - results['network_config'] = self._read_content_path(net_item) + content = self._read_content_path(net_item, decode=True) + results['network_config'] = content except IOError as e: raise BrokenMetadata("Failed to read network" " configuration: %s" % (e)) @@ -333,8 +334,8 @@ class ConfigDriveReader(BaseReader): components = [base] + list(add_ons) return os.path.join(*components) - def _path_read(self, path): - return util.load_file(path, decode=False) + def _path_read(self, path, decode=False): + return util.load_file(path, decode=decode) def _fetch_available_versions(self): if self._versions is None: @@ -446,7 +447,7 @@ class MetadataReader(BaseReader): self._versions = found return self._versions - def _path_read(self, path): + def _path_read(self, path, decode=False): def should_retry_cb(_request_args, cause): try: @@ -463,7 +464,10 @@ class MetadataReader(BaseReader): ssl_details=self.ssl_details, timeout=self.timeout, exception_cb=should_retry_cb) - return response.contents + if decode: + return response.contents.decode() + else: + return response.contents def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) -- cgit v1.2.3 From 80648a623fe6c7ae397629da30c04e52d79759f2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 13:19:50 -0400 Subject: eni parsing: support 'ether' in hwaddress, netmask and broadcast this adds ability to support ENI that has: hwadress ether 36:4c:e1:3b:14:31 or hwaddress 36:4c:e1:3b:14:31 the former is written by openstack (at least on dreamhost). Also, in the conversion of eni to network config support broadcast and netmask. --- cloudinit/net/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 6d9ea575..05152ead 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -201,7 +201,11 @@ def parse_deb_config_data(ifaces, contents, src_dir, src_path): ifaces[iface]['method'] = method currif = iface elif option == "hwaddress": - ifaces[currif]['hwaddress'] = split[1] + if split[1] == "ether": + val = split[2] + else: + val = split[1] + ifaces[currif]['hwaddress'] = val elif option in NET_CONFIG_OPTIONS: ifaces[currif][option] = split[1] elif option in NET_CONFIG_COMMANDS: @@ -800,8 +804,9 @@ def _ifaces_to_net_config_data(ifaces): if data.get('method') == 'static': subnet['address'] = data['address'] - if 'gateway' in data: - subnet['gateway'] = data['gateway'] + for copy_key in ('netmask', 'gateway', 'broadcast'): + if copy_key in data: + subnet[copy_key] = data[copy_key] if 'dns' in data: for n in ('nameservers', 'search'): -- cgit v1.2.3 From 1abff1453a24ed14375cb6294364295a0c2c7ee3 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 14:08:44 -0400 Subject: smartos: do not raise error when not on smartos if get_smartos_environ() returned a None, then the datasoure would raise a ValueError when get_data was called. Fix that. --- cloudinit/sources/DataSourceSmartOS.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index d6a9bf28..55dc94bb 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -171,7 +171,7 @@ LEGACY_USER_D = "/var/db" class DataSourceSmartOS(sources.DataSource): _unset = "_unset" - smartos_environ = _unset + smartos_type = _unset md_client = _unset def __init__(self, sys_cfg, distro, paths): @@ -195,8 +195,10 @@ class DataSourceSmartOS(sources.DataSource): return "%s [client=%s]" % (root, self.md_client) def _init(self): - if self.smartos_environ == self._unset: + if self.smartos_type == self._unset: self.smartos_type = get_smartos_environ() + if self.smartos_type is None: + self.md_client = None if self.md_client == self._unset: self.md_client = jmc_client_factory( @@ -577,7 +579,9 @@ def jmc_client_factory( if smartos_type is None: smartos_type = get_smartos_environ(uname_version) - if smartos_type == SMARTOS_ENV_KVM: + if smartos_type is None: + return None + elif smartos_type == SMARTOS_ENV_KVM: return JoyentMetadataLegacySerialClient( device=serial_device, timeout=serial_timeout, smartos_type=smartos_type) @@ -755,6 +759,9 @@ def get_datasource_list(depends): if __name__ == "__main__": import sys jmc = jmc_client_factory() + if jmc is None: + print("Do not appear to be on smartos.") + sys.exit(1) if len(sys.argv) == 1: keys = (list(SMARTOS_ATTRIB_JSON.keys()) + list(SMARTOS_ATTRIB_MAP.keys())) -- cgit v1.2.3 From bd31ab1e78f59c88b4aba031ffdaca506b3b04ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 14:36:51 -0400 Subject: fix untested previous change to smartos --- cloudinit/sources/DataSourceSmartOS.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 55dc94bb..0e03b04f 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -186,7 +186,6 @@ class DataSourceSmartOS(sources.DataSource): self._network_config = None self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - self.smartos_type = None self._init() -- cgit v1.2.3 From d709524941ba2b4e06940a9eb0861f0819d5560f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 15:18:27 -0400 Subject: re-add the 'Net' classes for datasources When the .pkl file is loaded, the module that it is loaded from must have the same symbol. Ie, if booted once and got DataSourceConfigDriveNet then upgraded and rebooted, then next boot would show Can't get attribute 'DataSourceConfigDriveNet' --- cloudinit/sources/DataSourceCloudSigma.py | 3 +++ cloudinit/sources/DataSourceConfigDrive.py | 25 ++++++++++++++----------- cloudinit/sources/DataSourceOpenNebula.py | 3 +++ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py index 07e8ae11..d1f806d6 100644 --- a/cloudinit/sources/DataSourceCloudSigma.py +++ b/cloudinit/sources/DataSourceCloudSigma.py @@ -115,6 +115,9 @@ class DataSourceCloudSigma(sources.DataSource): return self.metadata['uuid'] +# Legacy: Must be present in case we load an old pkl object +DataSourceCloudSigmaNet = DataSourceCloudSigma + # Used to match classes to dependencies. Since this datasource uses the serial # port network is not really required, so it's okay to load without it, too. datasources = [ diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 2d13a32f..61d967d9 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -254,17 +254,6 @@ def find_candidate_devs(probe_optical=True): return devices -# Used to match classes to dependencies -datasources = [ - (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - # Convert OpenStack ConfigDrive NetworkData json to network_config yaml def convert_network_data(network_json=None): """Return a dictionary of network_config by parsing provided @@ -382,3 +371,17 @@ def convert_network_data(network_json=None): config.append(cfg) return {'version': 1, 'config': config} + + +# Legacy: Must be present in case we load an old pkl object +DataSourceConfigDriveNet = DataSourceConfigDrive + +# Used to match classes to dependencies +datasources = [ + (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 15819a4f..8f85b115 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -415,6 +415,9 @@ def read_context_disk_dir(source_dir, asuser=None): return results +# Legacy: Must be present in case we load an old pkl object +DataSourceOpenNebulaNet = DataSourceOpenNebula + # Used to match classes to dependencies datasources = [ (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )), -- cgit v1.2.3 From 6bd7fbc35ac8726a8a0f422cd802d290c236bf3b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 2 Jun 2016 23:03:38 -0400 Subject: ConfigDrive: do not use 'id' on a link for the device name 'id' on a link in the openstack spec should be "Generic, generated ID". current implementation was to use the host's name for the host side nic. Which provided names like 'tap-adfasdffd'. We do not want to name devices like that as its quite unexpected and non user friendly. So here we use the system name for any nic that is present, but then require that the nics found also be present at the time of rendering. The end result is that if the system boots with net.ifnames=0 then it will get 'eth0' like names. and if it boots without net.ifnames then it will get enp0s1 like names. --- cloudinit/net/__init__.py | 15 +++++--- cloudinit/sources/DataSourceConfigDrive.py | 29 ++++++++++++--- .../unittests/test_datasource/test_configdrive.py | 41 ++++++++++++++++++++-- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 05152ead..f47053b2 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -970,9 +970,16 @@ def get_interface_mac(ifname): return read_sys_net(ifname, "address", enoent=False) -def get_ifname_mac_pairs(): - """Build a list of tuples (ifname, mac)""" - return [(ifname, get_interface_mac(ifname)) for ifname in get_devicelist()] - +def get_interfaces_by_mac(devs=None): + """Build a dictionary of tuples {mac: name}""" + if devs is None: + devs = get_devicelist() + ret = {} + for name in devs: + mac = get_interface_mac(name) + # some devices may not have a mac (tun0) + if mac: + ret[mac] = name + return ret # vi: ts=4 expandtab syntax=python diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 61d967d9..3cc9155d 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -255,7 +255,7 @@ def find_candidate_devs(probe_optical=True): # Convert OpenStack ConfigDrive NetworkData json to network_config yaml -def convert_network_data(network_json=None): +def convert_network_data(network_json=None, known_macs=None): """Return a dictionary of network_config by parsing provided OpenStack ConfigDrive NetworkData json format @@ -319,9 +319,15 @@ def convert_network_data(network_json=None): subnets = [] cfg = {k: v for k, v in link.items() if k in valid_keys['physical']} - cfg.update({'name': link['id']}) - for network in [net for net in networks - if net['link'] == link['id']]: + # 'name' is not in openstack spec yet, but we will support it if it is + # present. The 'id' in the spec is currently implemented as the host + # nic's name, meaning something like 'tap-adfasdffd'. We do not want + # to name guest devices with such ugly names. + if 'name' in link: + cfg['name'] = link['name'] + + for network in [n for n in networks + if n['link'] == link['id']]: subnet = {k: v for k, v in network.items() if k in valid_keys['subnet']} if 'dhcp' in network['type']: @@ -365,6 +371,21 @@ def convert_network_data(network_json=None): config.append(cfg) + need_names = [d for d in config + if d.get('type') == 'physical' and 'name' not in d] + + if need_names: + if known_macs is None: + known_macs = net.get_interfaces_by_mac() + + for d in need_names: + mac = d.get('mac_address') + if not mac: + raise ValueError("No mac_address or name entry for %s" % d) + if mac not in known_macs: + raise ValueError("Unable to find a system nic for %s" % d) + d['name'] = known_macs[mac] + for service in services: cfg = service cfg.update({'type': 'nameserver'}) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 195b8207..1364b39d 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -73,7 +73,7 @@ NETWORK_DATA = { 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', 'ethernet_mac_address': 'fa:16:3e:05:30:fe', - 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04', 'name': 'nic0'} ], 'networks': [ {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', @@ -88,6 +88,10 @@ NETWORK_DATA = { ] } +KNOWN_MACS = { + 'fa:16:3e:69:b0:58': 'enp0s1', + 'fa:16:3e:d4:57:ad': 'enp0s2'} + CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), 'ec2/2009-04-04/user-data': USER_DATA, @@ -365,10 +369,40 @@ class TestConfigDriveDataSource(TestCase): """Verify that network_data is converted and present on ds object.""" populate_dir(self.tmp, CFG_DRIVE_FILES_V2) myds = cfg_ds_from_dir(self.tmp) - network_config = ds.convert_network_data(NETWORK_DATA) + network_config = ds.convert_network_data(NETWORK_DATA, + known_macs=KNOWN_MACS) self.assertEqual(myds.network_config, network_config) +class TestConvertNetworkData(TestCase): + def _getnames_in_config(self, ncfg): + return set([n['name'] for n in ncfg['config'] + if n['type'] == 'physical']) + + def test_conversion_fills_names(self): + ncfg = ds.convert_network_data(NETWORK_DATA, known_macs=KNOWN_MACS) + expected = set(['nic0', 'enp0s1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + def test_convert_reads_system_prefers_name(self, get_interfaces_by_mac): + macs = KNOWN_MACS.copy() + macs.update({'fa:16:3e:05:30:fe': 'foonic1', + 'fa:16:3e:69:b0:58': 'ens1'}) + get_interfaces_by_mac.return_value = macs + + ncfg = ds.convert_network_data(NETWORK_DATA) + expected = set(['nic0', 'ens1', 'enp0s2']) + found = self._getnames_in_config(ncfg) + self.assertEqual(found, expected) + + def test_convert_raises_value_error_on_missing_name(self): + macs = {'aa:aa:aa:aa:aa:00': 'ens1'} + self.assertRaises(ValueError, ds.convert_network_data, + NETWORK_DATA, known_macs=macs) + + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) cfg_ds = ds.DataSourceConfigDrive(settings.CFG_BUILTIN, None, @@ -387,7 +421,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') cfg_ds.network_json = results.get('networkdata') - cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) + cfg_ds._network_config = ds.convert_network_data( + cfg_ds.network_json, known_macs=KNOWN_MACS) def populate_dir(seed_dir, files): -- cgit v1.2.3 From e2c2d70cde19211b18e5ec333e1cb0382d93f14d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 00:42:22 -0400 Subject: lxd: fix log messsage --- cloudinit/config/cc_lxd.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index b1de8f84..70d4e7c3 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -52,7 +52,8 @@ def handle(name, cfg, cloud, log, args): # Get config lxd_cfg = cfg.get('lxd') if not lxd_cfg: - log.debug("Skipping module named %s, not present or disabled by cfg") + log.debug("Skipping module named %s, not present or disabled by cfg", + name) return if not isinstance(lxd_cfg, dict): log.warn("lxd config must be a dictionary. found a '%s'", -- cgit v1.2.3 From 8a8d7eed2ed5696d58a825ec7301d8424c23ce5e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 14:23:34 -0400 Subject: avoid rendering 'lo' twice by not writing it in network config. --- cloudinit/net/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index f47053b2..c72b6ff8 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -788,6 +788,10 @@ def _ifaces_to_net_config_data(ifaces): for name, data in ifaces.items(): # devname is 'eth0' for name='eth0:1' devname = name.partition(":")[0] + if devname == "lo": + # currently provding 'lo' in network config results in duplicate + # entries. in rendered interfaces file. so skip it. + continue if devname not in devs: devs[devname] = {'type': 'physical', 'name': devname, 'subnets': []} -- cgit v1.2.3 From f495947a701d5629b6dbfd2ff9e01dad7bd5166b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 14:58:51 -0400 Subject: fix issue with routes on subnets not getting rendered --- cloudinit/net/__init__.py | 2 ++ .../unittests/test_datasource/test_configdrive.py | 41 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index c72b6ff8..49e9d5c2 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -574,6 +574,8 @@ def render_interfaces(network_state): content += iface_start_entry(iface, index) content += iface_add_subnet(iface, subnet) content += iface_add_attrs(iface) + for route in subnet.get('routes', []): + content += render_route(route, indent=" ") else: # ifenslave docs say to auto the slave devices if 'bond-master' in iface: diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 1364b39d..83d75f9c 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -15,6 +15,7 @@ except ImportError: from contextlib2 import ExitStack from cloudinit import helpers +from cloudinit import net from cloudinit import settings from cloudinit.sources import DataSourceConfigDrive as ds from cloudinit.sources.helpers import openstack @@ -88,9 +89,33 @@ NETWORK_DATA = { ] } +NETWORK_DATA_2 = { + "services": [ + {"type": "dns", "address": "1.1.1.191"}, + {"type": "dns", "address": "1.1.1.4"}], + "networks": [ + {"network_id": "d94bbe94-7abc-48d4-9c82-4628ea26164a", "type": "ipv4", + "netmask": "255.255.255.248", "link": "eth0", + "routes": [{"netmask": "0.0.0.0", "network": "0.0.0.0", + "gateway": "2.2.2.9"}], + "ip_address": "2.2.2.10", "id": "network0-ipv4"}, + {"network_id": "ca447c83-6409-499b-aaef-6ad1ae995348", "type": "ipv4", + "netmask": "255.255.255.224", "link": "eth1", + "routes": [], "ip_address": "3.3.3.24", "id": "network1-ipv4"}], + "links": [ + {"ethernet_mac_address": "fa:16:3e:dd:50:9a", "mtu": 1500, + "type": "vif", "id": "eth0", "vif_id": "vif-foo1"}, + {"ethernet_mac_address": "fa:16:3e:a8:14:69", "mtu": 1500, + "type": "vif", "id": "eth1", "vif_id": "vif-foo2"}] +} + + KNOWN_MACS = { 'fa:16:3e:69:b0:58': 'enp0s1', - 'fa:16:3e:d4:57:ad': 'enp0s2'} + 'fa:16:3e:d4:57:ad': 'enp0s2', + 'fa:16:3e:dd:50:9a': 'foo1', + 'fa:16:3e:a8:14:69': 'foo2', +} CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -402,6 +427,20 @@ class TestConvertNetworkData(TestCase): self.assertRaises(ValueError, ds.convert_network_data, NETWORK_DATA, known_macs=macs) + def test_conversion_with_route(self): + ncfg = ds.convert_network_data(NETWORK_DATA_2, known_macs=KNOWN_MACS) + # not the best test, but see that we get a route in the + # network config and that it gets rendered to an ENI file + routes = [] + for n in ncfg['config']: + for s in n.get('subnets', []): + routes.extend(s.get('routes', [])) + self.assertIn( + {'network': '0.0.0.0', 'netmask': '0.0.0.0', 'gateway': '2.2.2.9'}, + routes) + eni = net.render_interfaces(net.parse_net_config_data(ncfg)) + self.assertIn("route add default gw 2.2.2.9", eni) + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) -- cgit v1.2.3 From 42a7d2b6d44be5fd6e41734902e08897b709015d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:06:55 -0400 Subject: config drive conversion: recognize 'bridge' as a physical type, fix mtu the network json in openstack provides a type of 'bridge' when the underlying (host) type is a bridge. Silly, but we need to consider that a physical device as it will be for us. also, the 'mtu' will appear on the link, not on the route --- cloudinit/sources/DataSourceConfigDrive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 3cc9155d..c87f57fd 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -293,6 +293,7 @@ def convert_network_data(network_json=None, known_macs=None): 'mac_address', 'subnets', 'params', + 'mtu', ], 'subnet': [ 'type', @@ -302,7 +303,6 @@ def convert_network_data(network_json=None, known_macs=None): 'metric', 'gateway', 'pointopoint', - 'mtu', 'scope', 'dns_nameservers', 'dns_search', @@ -342,7 +342,7 @@ def convert_network_data(network_json=None, known_macs=None): }) subnets.append(subnet) cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']: cfg.update({ 'type': 'physical', 'mac_address': link['ethernet_mac_address']}) -- cgit v1.2.3 From 80931f7008971c9a7705c054fabc29fec7a133e2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 3 Jun 2016 15:27:32 -0400 Subject: fix tox -e flake8 --- cloudinit/config/cc_apt_configure.py | 4 +- .../test_handler_apt_configure_sources_list.py | 17 +++--- .../test_handler/test_handler_apt_source.py | 66 +++++++++++----------- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index d603f417..7a9777c0 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -199,8 +199,8 @@ def add_key(ent): def convert_to_new_format(srclist): - """ convert_to_new_format - convert the old list based format to the new dict based one + """convert_to_new_format + convert the old list based format to the new dict based one """ srcdict = {} if isinstance(srclist, list): diff --git a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py index 5cf386f8..5d0417a2 100644 --- a/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py +++ b/tests/unittests/test_handler/test_handler_apt_configure_sources_list.py @@ -3,7 +3,6 @@ Test templating of sources list """ import logging import os -import re import shutil import tempfile @@ -62,14 +61,14 @@ deb-src http://archive.ubuntu.com/ubuntu/ fakerelease main restricted def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): - """ TestAptSourceConfigSourceList + """TestAptSourceConfigSourceList Main Class to test sources list rendering """ def setUp(self): @@ -89,7 +88,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): return cloud.Cloud(myds, paths, {}, mydist, None) def apt_source_list(self, distro, mirror, mirrorcheck=None): - """ apt_source_list + """apt_source_list Test rendering of a source.list from template for a given distro """ if mirrorcheck is None: @@ -115,19 +114,19 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): {'codename': '', 'primary': mirrorcheck, 'mirror': mirrorcheck}) def test_apt_source_list_debian(self): - """ test_apt_source_list_debian + """test_apt_source_list_debian Test rendering of a source.list from template for debian """ self.apt_source_list('debian', 'http://httpredir.debian.org/debian') def test_apt_source_list_ubuntu(self): - """ test_apt_source_list_ubuntu + """test_apt_source_list_ubuntu Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_debian_mirrorfail(self): - """ test_apt_source_list_debian_mirrorfail + """test_apt_source_list_debian_mirrorfail Test rendering of a source.list from template for debian """ self.apt_source_list('debian', ['http://does.not.exist', @@ -135,7 +134,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://httpredir.debian.org/debian') def test_apt_srcl_ubuntu_mirrorfail(self): - """ test_apt_source_list_ubuntu_mirrorfail + """test_apt_source_list_ubuntu_mirrorfail Test rendering of a source.list from template for ubuntu """ self.apt_source_list('ubuntu', ['http://does.not.exist', @@ -143,7 +142,7 @@ class TestAptSourceConfigSourceList(t_help.FilesystemMockingTestCase): 'http://archive.ubuntu.com/ubuntu/') def test_apt_srcl_custom(self): - """ test_apt_srcl_custom + """test_apt_srcl_custom Test rendering from a custom source.list template """ cfg = util.load_yaml(YAML_TEXT_CUSTOM_SL) diff --git a/tests/unittests/test_handler/test_handler_apt_source.py b/tests/unittests/test_handler/test_handler_apt_source.py index fe2ffae5..4dbe69f0 100644 --- a/tests/unittests/test_handler/test_handler_apt_source.py +++ b/tests/unittests/test_handler/test_handler_apt_source.py @@ -33,14 +33,14 @@ S0ORP6HXET3+jC8BMG4tBWCTK/XEZw== def load_tfile_or_url(*args, **kwargs): - """ load_tfile_or_url + """load_tfile_or_url load file and return content after decoding """ return util.decode_binary(util.read_file_or_url(*args, **kwargs).contents) class TestAptSourceConfig(TestCase): - """ TestAptSourceConfig + """TestAptSourceConfig Main Class to test apt_source configs """ def setUp(self): @@ -57,7 +57,7 @@ class TestAptSourceConfig(TestCase): @staticmethod def _get_default_params(): - """ get_default_params + """get_default_params Get the most basic default mrror and release info to be used in tests """ params = {} @@ -66,7 +66,7 @@ class TestAptSourceConfig(TestCase): return params def myjoin(self, *args, **kwargs): - """ myjoin - redir into writable tmpdir""" + """myjoin - redir into writable tmpdir""" if (args[0] == "/etc/apt/sources.list.d/" and args[1] == "cloud_config_sources.list" and len(args) == 2): @@ -75,7 +75,7 @@ class TestAptSourceConfig(TestCase): return self.join(*args, **kwargs) def apt_src_basic(self, filename, cfg): - """ apt_src_basic + """apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params """ params = self._get_default_params() @@ -92,7 +92,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic(self): - """ test_apt_src_basic + """test_apt_src_basic Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. """ @@ -103,7 +103,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, [cfg]) def test_apt_src_basic_dict(self): - """ test_apt_src_basic_dict + """test_apt_src_basic_dict Test Fix deb source string, has to overwrite mirror conf in params. Test with a filename provided in config. Provided in a dictionary with filename being the key (new format) @@ -115,7 +115,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.aptlistfile, cfg) def apt_src_basic_tri(self, cfg): - """ apt_src_basic_tri + """apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. generic part to check three files with different content @@ -137,7 +137,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_basic_tri(self): - """ test_apt_src_basic_tri + """test_apt_src_basic_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. """ @@ -156,7 +156,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri([cfg1, cfg2, cfg3]) def test_apt_src_basic_dict_tri(self): - """ test_apt_src_basic_dict_tri + """test_apt_src_basic_dict_tri Test Fix three deb source string, has to overwrite mirror conf in params. Test with filenames provided in config. Provided in a dictionary with filename being the key (new format) @@ -176,7 +176,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic_tri(cfg) def test_apt_src_basic_nofn(self): - """ test_apt_src_basic_nofn + """test_apt_src_basic_nofn Test Fix deb source string, has to overwrite mirror conf in params. Test without a filename provided in config and test for known fallback. """ @@ -187,7 +187,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_basic(self.fallbackfn, [cfg]) def apt_src_replacement(self, filename, cfg): - """ apt_src_replace + """apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs """ params = self._get_default_params() @@ -202,7 +202,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace(self): - """ test_apt_src_replace + """test_apt_src_replace Test Autoreplacement of MIRROR and RELEASE in source specs with Filename being set """ @@ -211,7 +211,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.aptlistfile, [cfg]) def apt_src_replace_tri(self, cfg): - """ apt_src_replace_tri + """apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with generic part """ @@ -231,7 +231,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_replace_tri(self): - """ test_apt_src_replace_tri + """test_apt_src_replace_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set """ @@ -244,7 +244,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replace_tri([cfg1, cfg2, cfg3]) def test_apt_src_replace_dict_tri(self): - """ test_apt_src_replace_dict_tri + """test_apt_src_replace_dict_tri Test three autoreplacements of MIRROR and RELEASE in source specs with Filename being set Provided in a dictionary with filename being the key (new format) @@ -252,13 +252,13 @@ class TestAptSourceConfig(TestCase): filenames to be overwritten inside the directory entry. """ cfg = {self.aptlistfile: {'source': 'deb $MIRROR $RELEASE multiverse'}, - 'notused': {'source': 'deb $MIRROR $RELEASE main', - 'filename': self.aptlistfile2}, + 'notused': {'source': 'deb $MIRROR $RELEASE main', + 'filename': self.aptlistfile2}, self.aptlistfile3: {'source': 'deb $MIRROR $RELEASE universe'}} self.apt_src_replace_tri(cfg) def test_apt_src_replace_nofn(self): - """ test_apt_src_replace_nofn + """test_apt_src_replace_nofn Test Autoreplacement of MIRROR and RELEASE in source specs with No filename being set """ @@ -267,7 +267,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_replacement(self.fallbackfn, [cfg]) def apt_src_keyid(self, filename, cfg, keynum): - """ apt_src_keyid + """apt_src_keyid Test specification of a source + keyid """ params = self._get_default_params() @@ -293,7 +293,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid(self): - """ test_apt_src_keyid + """test_apt_src_keyid Test specification of a source + keyid with filename being set """ cfg = {'source': ('deb ' @@ -305,7 +305,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.aptlistfile, [cfg], 1) def test_apt_src_keyid_tri(self): - """ test_apt_src_keyid_tri + """test_apt_src_keyid_tri Test specification of a source + keyid with filename being set Setting three of such, check for content and keys """ @@ -345,7 +345,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_keyid_nofn(self): - """ test_apt_src_keyid_nofn + """test_apt_src_keyid_nofn Test specification of a source + keyid without filename being set """ cfg = {'source': ('deb ' @@ -357,7 +357,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_keyid(self.fallbackfn, [cfg], 1) def apt_src_key(self, filename, cfg): - """ apt_src_key + """apt_src_key Test specification of a source + key """ params = self._get_default_params() @@ -378,7 +378,7 @@ class TestAptSourceConfig(TestCase): contents, flags=re.IGNORECASE)) def test_apt_src_key(self): - """ test_apt_src_key + """test_apt_src_key Test specification of a source + key with filename being set """ cfg = {'source': ('deb ' @@ -390,7 +390,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.aptlistfile, cfg) def test_apt_src_key_nofn(self): - """ test_apt_src_key_nofn + """test_apt_src_key_nofn Test specification of a source + key without filename being set """ cfg = {'source': ('deb ' @@ -402,7 +402,7 @@ class TestAptSourceConfig(TestCase): self.apt_src_key(self.fallbackfn, cfg) def test_apt_src_keyonly(self): - """ test_apt_src_keyonly + """test_apt_src_keyonly Test specification key without source """ params = self._get_default_params() @@ -419,7 +419,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyidonly(self): - """ test_apt_src_keyidonly + """test_apt_src_keyidonly Test specification of a keyid without source """ params = self._get_default_params() @@ -436,7 +436,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_keyid_real(self): - """ test_apt_src_keyid_real + """test_apt_src_keyid_real Test specification of a keyid without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -454,7 +454,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_longkeyid_real(self): - """ test_apt_src_longkeyid_real + """test_apt_src_longkeyid_real Test specification of a long key fingerprint without source incl up to addition of the key (nothing but add_key_raw mocked) """ @@ -472,7 +472,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa(self): - """ test_apt_src_ppa + """test_apt_src_ppa Test specification of a ppa """ params = self._get_default_params() @@ -491,7 +491,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile)) def test_apt_src_ppa_tri(self): - """ test_apt_src_ppa_tri + """test_apt_src_ppa_tri Test specification of a ppa """ params = self._get_default_params() @@ -519,7 +519,7 @@ class TestAptSourceConfig(TestCase): self.assertFalse(os.path.isfile(self.aptlistfile3)) def test_convert_to_new_format(self): - """ test_convert_to_new_format + """test_convert_to_new_format Test the conversion of old to new format And the noop conversion of new to new format as well """ -- cgit v1.2.3