diff options
| -rw-r--r-- | ChangeLog | 10 | ||||
| -rw-r--r-- | cloudinit/config/cc_emit_upstart.py | 3 | ||||
| -rw-r--r-- | cloudinit/config/cc_snappy.py | 259 | ||||
| -rw-r--r-- | cloudinit/distros/__init__.py | 15 | ||||
| -rw-r--r-- | cloudinit/handlers/__init__.py | 11 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceGCE.py | 92 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceNoCloud.py | 5 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceSmartOS.py | 89 | ||||
| -rw-r--r-- | cloudinit/user_data.py | 9 | ||||
| -rw-r--r-- | cloudinit/util.py | 8 | ||||
| -rw-r--r-- | doc/rtd/topics/datasources.rst | 2 | ||||
| -rw-r--r-- | doc/sources/cloudstack/README.rst | 29 | ||||
| -rw-r--r-- | systemd/cloud-config.service | 5 | ||||
| -rw-r--r-- | systemd/cloud-config.target | 3 | ||||
| -rw-r--r-- | systemd/cloud-final.service | 5 | ||||
| -rw-r--r-- | systemd/cloud-init.service | 4 | ||||
| -rw-r--r-- | tests/unittests/test_data.py | 32 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_gce.py | 49 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_smartos.py | 230 | ||||
| -rw-r--r-- | tests/unittests/test_handler/test_handler_snappy.py | 311 |
20 files changed, 970 insertions, 201 deletions
@@ -27,6 +27,16 @@ - CloudStack: support fetching password from virtual router [Daniel Watkins] (LP: #1422388) - readurl, read_file_or_url returns bytes, user must convert as necessary + - SmartOS: use v2 metadata service (LP: #1436417) [Daniel Watkins] + - NoCloud: fix local datasource claiming found without explicit dsmode + - Snappy: add support for installing snappy packages and configuring. + - systemd: use network-online instead of network.target (LP: #1440180) + [Steve Langasek] + - Add functionality to fixate the uid of a newly added user. + - Don't overwrite the hostname if the user has changed it after we set it. + - GCE datasource does not handle instance ssh keys (LP: 1403617) + - sysvinit: make cloud-init-local run before network (LP: #1275098) + [Surojit Pathak] 0.7.6: - open 0.7.6 - Enable vendordata on CloudSigma datasource (LP: #1303986) diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py index e1b9a4c2..86ae97ab 100644 --- a/cloudinit/config/cc_emit_upstart.py +++ b/cloudinit/config/cc_emit_upstart.py @@ -32,8 +32,7 @@ LOG = logging.getLogger(__name__) def is_upstart_system(): if not os.path.isfile("/sbin/initctl"): - LOG.debug(("Skipping module named %s," - " no /sbin/initctl located"), name) + LOG.debug("no /sbin/initctl located") return False myenv = os.environ.copy() diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 133336d4..bfe76558 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -1,5 +1,45 @@ # vi: ts=4 expandtab # +""" +snappy modules allows configuration of snappy. +Example config: + #cloud-config + snappy: + system_snappy: auto + ssh_enabled: False + packages: [etcd, pkg2.smoser] + config: + pkgname: + key2: value2 + pkg2: + key1: value1 + packages_dir: '/writable/user-data/cloud-init/snaps' + + - ssh_enabled: + This defaults to 'False'. Set to a non-false value to enable ssh service + - snap installation and config + The above would install 'etcd', and then install 'pkg2.smoser' with a + '<config-file>' argument where 'config-file' has 'config-blob' inside it. + If 'pkgname' is installed already, then 'snappy config pkgname <file>' + will be called where 'file' has 'pkgname-config-blob' as its content. + + Entries in 'config' can be namespaced or non-namespaced for a package. + In either case, the config provided to snappy command is non-namespaced. + The package name is provided as it appears. + + If 'packages_dir' has files in it that end in '.snap', then they are + installed. Given 3 files: + <packages_dir>/foo.snap + <packages_dir>/foo.config + <packages_dir>/bar.snap + cloud-init will invoke: + snappy install <packages_dir>/foo.snap <packages_dir>/foo.config + snappy install <packages_dir>/bar.snap + + Note, that if provided a 'config' entry for 'ubuntu-core', then + cloud-init will invoke: snappy config ubuntu-core <config> + Allowing you to configure ubuntu-core in this way. +""" from cloudinit import log as logging from cloudinit import templater @@ -7,63 +47,168 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE import glob +import six +import tempfile import os LOG = logging.getLogger(__name__) frequency = PER_INSTANCE -SNAPPY_ENV_PATH = "/writable/system-data/etc/snappy.env" +SNAPPY_CMD = "snappy" +NAMESPACE_DELIM = '.' BUILTIN_CFG = { 'packages': [], - 'packages_dir': '/writable/user-data/cloud-init/click_packages', + 'packages_dir': '/writable/user-data/cloud-init/snaps', 'ssh_enabled': False, - 'system_snappy': "auto" + 'system_snappy': "auto", + 'config': {}, } -""" -snappy: - system_snappy: auto - ssh_enabled: True - packages: - - etcd - - {'name': 'pkg1', 'config': "wark"} -""" +def parse_filename(fname): + fname = os.path.basename(fname) + fname_noext = fname.rpartition(".")[0] + name = fname_noext.partition("_")[0] + shortname = name.partition(".")[0] + return(name, shortname, fname_noext) + + +def get_fs_package_ops(fspath): + if not fspath: + return [] + ops = [] + for snapfile in sorted(glob.glob(os.path.sep.join([fspath, '*.snap']))): + (name, shortname, fname_noext) = parse_filename(snapfile) + cfg = None + for cand in (fname_noext, name, shortname): + fpcand = os.path.sep.join([fspath, cand]) + ".config" + if os.path.isfile(fpcand): + cfg = fpcand + break + ops.append(makeop('install', name, config=None, + path=snapfile, cfgfile=cfg)) + return ops + + +def makeop(op, name, config=None, path=None, cfgfile=None): + return({'op': op, 'name': name, 'config': config, 'path': path, + 'cfgfile': cfgfile}) + + +def get_package_config(configs, name): + # load the package's config from the configs dict. + # prefer full-name entry (config-example.canonical) + # over short name entry (config-example) + if name in configs: + return configs[name] + return configs.get(name.partition(NAMESPACE_DELIM)[0]) + + +def get_package_ops(packages, configs, installed=None, fspath=None): + # get the install an config operations that should be done + if installed is None: + installed = read_installed_packages() + short_installed = [p.partition(NAMESPACE_DELIM)[0] for p in installed] + + if not packages: + packages = [] + if not configs: + configs = {} + + ops = [] + ops += get_fs_package_ops(fspath) + + for name in packages: + ops.append(makeop('install', name, get_package_config(configs, name))) + + to_install = [f['name'] for f in ops] + short_to_install = [f['name'].partition(NAMESPACE_DELIM)[0] for f in ops] -def install_package(pkg_name, config=None): - cmd = ["snappy", "install"] - if config: - if os.path.isfile(config): - cmd.append("--config-file=" + config) + for name in configs: + if name in to_install: + continue + shortname = name.partition(NAMESPACE_DELIM)[0] + if shortname in short_to_install: + continue + if name in installed or shortname in short_installed: + ops.append(makeop('config', name, + config=get_package_config(configs, name))) + + # prefer config entries to filepath entries + for op in ops: + if op['op'] != 'install' or not op['cfgfile']: + continue + name = op['name'] + fromcfg = get_package_config(configs, op['name']) + if fromcfg: + LOG.debug("preferring configs[%(name)s] over '%(cfgfile)s'", op) + op['cfgfile'] = None + op['config'] = fromcfg + + return ops + + +def render_snap_op(op, name, path=None, cfgfile=None, config=None): + if op not in ('install', 'config'): + raise ValueError("cannot render op '%s'" % op) + + shortname = name.partition(NAMESPACE_DELIM)[0] + try: + cfg_tmpf = None + if config is not None: + # input to 'snappy config packagename' must have nested data. odd. + # config: + # packagename: + # config + # Note, however, we do not touch config files on disk. + nested_cfg = {'config': {shortname: config}} + (fd, cfg_tmpf) = tempfile.mkstemp() + os.write(fd, util.yaml_dumps(nested_cfg).encode()) + os.close(fd) + cfgfile = cfg_tmpf + + cmd = [SNAPPY_CMD, op] + if op == 'install': + if path: + cmd.append("--allow-unauthenticated") + cmd.append(path) + else: + cmd.append(name) + if cfgfile: + cmd.append(cfgfile) + elif op == 'config': + cmd += [name, cfgfile] + + util.subp(cmd) + + finally: + if cfg_tmpf: + os.unlink(cfg_tmpf) + + +def read_installed_packages(): + ret = [] + for (name, date, version, dev) in read_pkg_data(): + if dev: + ret.append(NAMESPACE_DELIM.join([name, dev])) else: - cmd.append("--config=" + config) - cmd.append(pkg_name) - util.subp(cmd) - - -def install_packages(package_dir, packages): - local_pkgs = glob.glob(os.path.sep.join([package_dir, '*.click'])) - LOG.debug("installing local packages %s" % local_pkgs) - if local_pkgs: - for pkg in local_pkgs: - cfg = pkg.replace(".click", ".config") - if not os.path.isfile(cfg): - cfg = None - install_package(pkg, config=cfg) - - LOG.debug("installing click packages") - if packages: - for pkg in packages: - if not pkg: - continue - if isinstance(pkg, str): - name = pkg - config = None - elif pkg: - name = pkg.get('name', pkg) - config = pkg.get('config') - install_package(pkg_name=name, config=config) + ret.append(name) + return ret + + +def read_pkg_data(): + out, err = util.subp([SNAPPY_CMD, "list"]) + pkg_data = [] + for line in out.splitlines()[1:]: + toks = line.split(sep=None, maxsplit=3) + if len(toks) == 3: + (name, date, version) = toks + dev = None + else: + (name, date, version, dev) = toks + pkg_data.append((name, date, version, dev,)) + return pkg_data def disable_enable_ssh(enabled): @@ -92,6 +237,15 @@ def system_is_snappy(): return False +def set_snappy_command(): + global SNAPPY_CMD + if util.which("snappy-go"): + SNAPPY_CMD = "snappy-go" + else: + SNAPPY_CMD = "snappy" + LOG.debug("snappy command is '%s'", SNAPPY_CMD) + + def handle(name, cfg, cloud, log, args): cfgin = cfg.get('snappy') if not cfgin: @@ -107,7 +261,22 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: 'auto' mode, and system not snappy", name) return - install_packages(mycfg['packages_dir'], - mycfg['packages']) + set_snappy_command() + + pkg_ops = get_package_ops(packages=mycfg['packages'], + configs=mycfg['config'], + fspath=mycfg['packages_dir']) + + fails = [] + for pkg_op in pkg_ops: + try: + render_snap_op(**pkg_op) + except Exception as e: + fails.append((pkg_op, e,)) + LOG.warn("'%s' failed for '%s': %s", + pkg_op['op'], pkg_op['name'], e) disable_enable_ssh(mycfg.get('ssh_enabled', False)) + + if fails: + raise Exception("failed to install/configure snaps") diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index ab874b45..05721922 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -208,6 +208,15 @@ class Distro(object): and sys_hostname != hostname): update_files.append(sys_fn) + # If something else has changed the hostname after we set it + # initially, we should not overwrite those changes (we should + # only be setting the hostname once per instance) + if (sys_hostname and prev_hostname and + sys_hostname != prev_hostname): + LOG.info("%s differs from %s, assuming user maintained hostname.", + prev_hostname_fn, sys_fn) + return + # Remove duplicates (incase the previous config filename) # is the same as the system config filename, don't bother # doing it twice @@ -222,11 +231,6 @@ class Distro(object): util.logexc(LOG, "Failed to write hostname %s to %s", hostname, fn) - if (sys_hostname and prev_hostname and - sys_hostname != prev_hostname): - LOG.debug("%s differs from %s, assuming user maintained hostname.", - prev_hostname_fn, sys_fn) - # If the system hostname file name was provided set the # non-fqdn as the transient hostname. if sys_fn in update_files: @@ -318,6 +322,7 @@ class Distro(object): "gecos": '--comment', "homedir": '--home', "primary_group": '--gid', + "uid": '--uid', "groups": '--groups', "passwd": '--password', "shell": '--shell', diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d62fcd19..53d5604a 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -170,12 +170,12 @@ def _extract_first_or_bytes(blob, size): start = blob.split("\n", 1)[0] else: # We want to avoid decoding the whole blob (it might be huge) - # By taking 4*size bytes we have a guarantee to decode size utf8 chars - start = blob[:4*size].decode(errors='ignore').split("\n", 1)[0] + # By taking 4*size bytes we guarantee to decode size utf8 chars + start = blob[:4 * size].decode(errors='ignore').split("\n", 1)[0] if len(start) >= size: start = start[:size] except UnicodeDecodeError: - # Bytes array doesn't contain a text object -- return chunk of raw bytes + # Bytes array doesn't contain text so return chunk of raw bytes start = blob[0:size] return start @@ -263,7 +263,10 @@ def fixup_handler(mod, def_freq=PER_INSTANCE): def type_from_starts_with(payload, default=None): - payload_lc = payload.lower() + try: + payload_lc = util.decode_binary(payload).lower() + except UnicodeDecodeError: + return default payload_lc = payload_lc.lstrip() for text in INCLUSION_SRCH: if payload_lc.startswith(text): diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py index 608c07f1..f4ed915d 100644 --- a/cloudinit/sources/DataSourceGCE.py +++ b/cloudinit/sources/DataSourceGCE.py @@ -30,6 +30,31 @@ BUILTIN_DS_CONFIG = { REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') +class GoogleMetadataFetcher(object): + headers = {'X-Google-Metadata-Request': True} + + def __init__(self, metadata_address): + self.metadata_address = metadata_address + + def get_value(self, path, is_text): + value = None + try: + resp = url_helper.readurl(url=self.metadata_address + path, + headers=self.headers) + except url_helper.UrlError as exc: + msg = "url %s raised exception %s" + LOG.debug(msg, path, exc) + else: + if resp.code == 200: + if is_text: + value = util.decode_binary(resp.contents) + else: + value = resp.contents + else: + LOG.debug("url %s returned code %s", path, resp.code) + return value + + class DataSourceGCE(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -50,17 +75,15 @@ class DataSourceGCE(sources.DataSource): return public_key def get_data(self): - # GCE metadata server requires a custom header since v1 - headers = {'X-Google-Metadata-Request': True} - # url_map: (our-key, path, required, is_text) url_map = [ - ('instance-id', 'instance/id', True, True), - ('availability-zone', 'instance/zone', True, True), - ('local-hostname', 'instance/hostname', True, True), - ('public-keys', 'project/attributes/sshKeys', False, True), - ('user-data', 'instance/attributes/user-data', False, False), - ('user-data-encoding', 'instance/attributes/user-data-encoding', + ('instance-id', ('instance/id',), True, True), + ('availability-zone', ('instance/zone',), True, True), + ('local-hostname', ('instance/hostname',), True, True), + ('public-keys', ('project/attributes/sshKeys', + 'instance/attributes/sshKeys'), False, True), + ('user-data', ('instance/attributes/user-data',), False, False), + ('user-data-encoding', ('instance/attributes/user-data-encoding',), False, True), ] @@ -69,40 +92,25 @@ class DataSourceGCE(sources.DataSource): LOG.debug("%s is not resolvable", self.metadata_address) return False + metadata_fetcher = GoogleMetadataFetcher(self.metadata_address) # iterate over url_map keys to get metadata items - found = False - for (mkey, path, required, is_text) in url_map: - try: - resp = url_helper.readurl(url=self.metadata_address + path, - headers=headers) - if resp.code == 200: - found = True - if is_text: - self.metadata[mkey] = util.decode_binary(resp.contents) - else: - self.metadata[mkey] = resp.contents + running_on_gce = False + for (mkey, paths, required, is_text) in url_map: + value = None + for path in paths: + new_value = metadata_fetcher.get_value(path, is_text) + if new_value is not None: + value = new_value + if value: + running_on_gce = True + if required and value is None: + msg = "required key %s returned nothing. not GCE" + if not running_on_gce: + LOG.debug(msg, mkey) else: - if required: - msg = "required url %s returned code %s. not GCE" - if not found: - LOG.debug(msg, path, resp.code) - else: - LOG.warn(msg, path, resp.code) - return False - else: - self.metadata[mkey] = None - except url_helper.UrlError as e: - if required: - msg = "required url %s raised exception %s. not GCE" - if not found: - LOG.debug(msg, path, e) - else: - LOG.warn(msg, path, e) - return False - msg = "Failed to get %s metadata item: %s." - LOG.debug(msg, path, e) - - self.metadata[mkey] = None + LOG.warn(msg, mkey) + return False + self.metadata[mkey] = value if self.metadata['public-keys']: lines = self.metadata['public-keys'].splitlines() @@ -116,7 +124,7 @@ class DataSourceGCE(sources.DataSource): else: LOG.warn('unknown user-data-encoding: %s, ignoring', encoding) - return found + return running_on_gce @property def launch_index(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index c26a645c..6a861af3 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -124,7 +124,7 @@ class DataSourceNoCloud(sources.DataSource): # that is more likely to be what is desired. If they want # dsmode of local, then they must specify that. if 'dsmode' not in mydata['meta-data']: - mydata['dsmode'] = "net" + mydata['meta-data']['dsmode'] = "net" LOG.debug("Using data from %s", dev) found.append(dev) @@ -193,7 +193,8 @@ class DataSourceNoCloud(sources.DataSource): self.vendordata = mydata['vendor-data'] return True - LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode']) + LOG.debug("%s: not claiming datasource, dsmode=%s", self, + mydata['meta-data']['dsmode']) return False diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 9d48beab..c9b497df 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -29,9 +29,12 @@ # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html # Comments with "@datadictionary" are snippets of the definition -import base64 import binascii +import contextlib import os +import random +import re + import serial from cloudinit import log as logging @@ -301,6 +304,65 @@ def get_serial(seed_device, seed_timeout): return ser +class JoyentMetadataFetchException(Exception): + pass + + +class JoyentMetadataClient(object): + """ + A client implementing v2 of the Joyent Metadata Protocol Specification. + + The full specification can be found at + http://eng.joyent.com/mdata/protocol.html + """ + line_regex = re.compile( + r'V2 (?P<length>\d+) (?P<checksum>[0-9a-f]+)' + r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)' + r'( (?P<payload>.+))?)') + + def __init__(self, serial): + self.serial = serial + + def _checksum(self, body): + return '{0:08x}'.format( + binascii.crc32(body.encode('utf-8')) & 0xffffffff) + + def _get_value_from_frame(self, expected_request_id, frame): + frame_data = self.line_regex.match(frame).groupdict() + if int(frame_data['length']) != len(frame_data['body']): + raise JoyentMetadataFetchException( + 'Incorrect frame length given ({0} != {1}).'.format( + frame_data['length'], len(frame_data['body']))) + expected_checksum = self._checksum(frame_data['body']) + if frame_data['checksum'] != expected_checksum: + raise JoyentMetadataFetchException( + 'Invalid checksum (expected: {0}; got {1}).'.format( + expected_checksum, frame_data['checksum'])) + if frame_data['request_id'] != expected_request_id: + raise JoyentMetadataFetchException( + 'Request ID mismatch (expected: {0}; got {1}).'.format( + expected_request_id, frame_data['request_id'])) + if not frame_data.get('payload', None): + LOG.debug('No value found.') + return None + value = util.b64d(frame_data['payload']) + LOG.debug('Value "%s" found.', value) + return value + + def get_metadata(self, metadata_key): + LOG.debug('Fetching metadata key "%s"...', metadata_key) + request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) + message_body = '{0} GET {1}'.format(request_id, + util.b64e(metadata_key)) + msg = 'V2 {0} {1} {2}\n'.format( + len(message_body), self._checksum(message_body), message_body) + LOG.debug('Writing "%s" to serial port.', msg) + self.serial.write(msg.encode('ascii')) + response = self.serial.readline().decode('ascii') + LOG.debug('Read "%s" from serial port.', response) + return self._get_value_from_frame(request_id, response) + + def query_data(noun, seed_device, seed_timeout, strip=False, default=None, b64=None): """Makes a request to via the serial console via "GET <NOUN>" @@ -314,33 +376,20 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None, encoded, so this method relies on being told if the data is base64 or not. """ - if not noun: return False - ser = get_serial(seed_device, seed_timeout) - ser.write("GET %s\n" % noun.rstrip()) - status = str(ser.readline()).rstrip() - response = [] - eom_found = False + with contextlib.closing(get_serial(seed_device, seed_timeout)) as ser: + client = JoyentMetadataClient(ser) + response = client.get_metadata(noun) - if 'SUCCESS' not in status: - ser.close() + if response is None: return default - while not eom_found: - m = ser.readline() - if m.rstrip() == ".": - eom_found = True - else: - response.append(m) - - ser.close() - if b64 is None: b64 = query_data('b64-%s' % noun, seed_device=seed_device, - seed_timeout=seed_timeout, b64=False, - default=False, strip=True) + seed_timeout=seed_timeout, b64=False, + default=False, strip=True) b64 = util.is_true(b64) resp = None diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index eb3c7336..f7c5787c 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -49,6 +49,7 @@ INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url'] ARCHIVE_TYPES = ["text/cloud-config-archive"] UNDEF_TYPE = "text/plain" ARCHIVE_UNDEF_TYPE = "text/cloud-config" +ARCHIVE_UNDEF_BINARY_TYPE = "application/octet-stream" # This seems to hit most of the gzip possible content types. DECOMP_TYPES = [ @@ -265,11 +266,15 @@ class UserDataProcessor(object): content = ent.get('content', '') mtype = ent.get('type') if not mtype: - mtype = handlers.type_from_starts_with(content, - ARCHIVE_UNDEF_TYPE) + default = ARCHIVE_UNDEF_TYPE + if isinstance(content, six.binary_type): + default = ARCHIVE_UNDEF_BINARY_TYPE + mtype = handlers.type_from_starts_with(content, default) maintype, subtype = mtype.split('/', 1) if maintype == "text": + if isinstance(content, six.binary_type): + content = content.decode() msg = MIMEText(content, _subtype=subtype) else: msg = MIMEBase(maintype, subtype) diff --git a/cloudinit/util.py b/cloudinit/util.py index 971c1c2d..cae57770 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -121,8 +121,12 @@ def fully_decoded_payload(part): if (six.PY3 and part.get_content_maintype() == 'text' and isinstance(cte_payload, bytes)): - charset = part.get_charset() or 'utf-8' - return cte_payload.decode(charset, errors='surrogateescape') + charset = part.get_charset() + if charset and charset.input_codec: + encoding = charset.input_codec + else: + encoding = 'utf-8' + return cte_payload.decode(encoding, errors='surrogateescape') return cte_payload diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst index cc0d0ede..a2024bdc 100644 --- a/doc/rtd/topics/datasources.rst +++ b/doc/rtd/topics/datasources.rst @@ -166,7 +166,7 @@ For now see: http://maas.ubuntu.com/ CloudStack --------------------------- -*TODO* +.. include:: ../../sources/cloudstack/README.rst --------------------------- OVF diff --git a/doc/sources/cloudstack/README.rst b/doc/sources/cloudstack/README.rst new file mode 100644 index 00000000..eba1cd7e --- /dev/null +++ b/doc/sources/cloudstack/README.rst @@ -0,0 +1,29 @@ +`Apache CloudStack`_ expose user-data, meta-data, user password and account +sshkey thru the Virtual-Router. For more details on meta-data and user-data, +refer the `CloudStack Administrator Guide`_. + +URLs to access user-data and meta-data from the Virtual Machine. Here 10.1.1.1 +is the Virtual Router IP: + +.. code:: bash + + http://10.1.1.1/latest/user-data + http://10.1.1.1/latest/meta-data + http://10.1.1.1/latest/meta-data/{metadata type} + +Configuration +~~~~~~~~~~~~~ + +Apache CloudStack datasource can be configured as follows: + +.. code:: yaml + + datasource: + CloudStack: {} + None: {} + datasource_list: + - CloudStack + + +.. _Apache CloudStack: http://cloudstack.apache.org/ +.. _CloudStack Administrator Guide: http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/latest/virtual_machines.html#user-data-and-meta-data
\ No newline at end of file diff --git a/systemd/cloud-config.service b/systemd/cloud-config.service index 41a86147..f9f1996e 100644 --- a/systemd/cloud-config.service +++ b/systemd/cloud-config.service @@ -1,8 +1,7 @@ [Unit] Description=Apply the settings specified in cloud-config -After=network.target syslog.target cloud-config.target -Requires=cloud-config.target -Wants=network.target +After=network-online.target cloud-config.target syslog.target +Wants=network-online.target cloud-config.target [Service] Type=oneshot diff --git a/systemd/cloud-config.target b/systemd/cloud-config.target index 28f5bcf1..ae9b7d02 100644 --- a/systemd/cloud-config.target +++ b/systemd/cloud-config.target @@ -7,4 +7,5 @@ [Unit] Description=Cloud-config availability -Requires=cloud-init-local.service cloud-init.service +Wants=cloud-init-local.service cloud-init.service +After=cloud-init-local.service cloud-init.service diff --git a/systemd/cloud-final.service b/systemd/cloud-final.service index ef0f52b9..c023ad94 100644 --- a/systemd/cloud-final.service +++ b/systemd/cloud-final.service @@ -1,8 +1,7 @@ [Unit] Description=Execute cloud user/final scripts -After=network.target syslog.target cloud-config.service rc-local.service -Requires=cloud-config.target -Wants=network.target +After=network-online.target cloud-config.service syslog.target rc-local.service +Wants=network-online.target cloud-config.service [Service] Type=oneshot diff --git a/systemd/cloud-init.service b/systemd/cloud-init.service index 398b90ea..48920283 100644 --- a/systemd/cloud-init.service +++ b/systemd/cloud-init.service @@ -1,8 +1,8 @@ [Unit] Description=Initial cloud-init job (metadata service crawler) -After=local-fs.target network.target cloud-init-local.service +After=local-fs.target network-online.target cloud-init-local.service Before=sshd.service sshd-keygen.service systemd-user-sessions.service -Requires=network.target +Requires=network-online.target Wants=local-fs.target cloud-init-local.service sshd.service sshd-keygen.service [Service] diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 4f24e2dd..1b15dafa 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -494,10 +494,10 @@ c: 4 ]) def test_mime_application_octet_stream(self): - """Mime message of type application/octet-stream is ignored but shows warning.""" + """Mime type application/octet-stream is ignored but shows warning.""" ci = stages.Init() message = MIMEBase("application", "octet-stream") - message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc\xbf') + message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc') encoders.encode_base64(message) ci.datasource = FakeDataSource(message.as_string().encode()) @@ -511,6 +511,34 @@ c: 4 mockobj.assert_called_once_with( ci.paths.get_ipath("cloud_config"), "", 0o600) + + def test_cloud_config_archive(self): + non_decodable = b'\x11\xc9\xb4gTH\xee\x12' + data = [{'content': '#cloud-config\npassword: gocubs\n'}, + {'content': '#cloud-config\nlocale: chicago\n'}, + {'content': non_decodable}] + message = b'#cloud-config-archive\n' + util.yaml_dumps(data).encode() + + ci = stages.Init() + ci.datasource = FakeDataSource(message) + + fs = {} + + def fsstore(filename, content, mode=0o0644, omode="wb"): + fs[filename] = content + + # consuming the user-data provided should write 'cloud_config' file + # which will have our yaml in it. + with mock.patch('cloudinit.util.write_file') as mockobj: + mockobj.side_effect = fsstore + ci.fetch() + ci.consume_data() + + cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")]) + self.assertEqual(cfg.get('password'), 'gocubs') + self.assertEqual(cfg.get('locale'), 'chicago') + + class TestUDProcess(helpers.ResourceUsingTestCase): def test_bytes_in_userdata(self): diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index 4280abc4..1fb100f7 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -113,10 +113,6 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): self.assertEqual(GCE_META.get('instance/attributes/user-data'), self.ds.get_userdata_raw()) - # we expect a list of public ssh keys with user names stripped - self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], - self.ds.get_public_ssh_keys()) - # test partial metadata (missing user-data in particular) @httpretty.activate def test_metadata_partial(self): @@ -141,3 +137,48 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): decoded = b64decode( GCE_META_ENCODING.get('instance/attributes/user-data')) self.assertEqual(decoded, self.ds.get_userdata_raw()) + + @httpretty.activate + def test_missing_required_keys_return_false(self): + for required_key in ['instance/id', 'instance/zone', + 'instance/hostname']: + meta = GCE_META_PARTIAL.copy() + del meta[required_key] + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback(meta)) + self.assertEqual(False, self.ds.get_data()) + httpretty.reset() + + @httpretty.activate + def test_project_level_ssh_keys_are_used(self): + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback()) + self.ds.get_data() + + # we expect a list of public ssh keys with user names stripped + self.assertEqual(['ssh-rsa AA2..+aRD0fyVw== root@server'], + self.ds.get_public_ssh_keys()) + + @httpretty.activate + def test_instance_level_ssh_keys_are_used(self): + key_content = 'ssh-rsa JustAUser root@server' + meta = GCE_META.copy() + meta['instance/attributes/sshKeys'] = 'user:{0}'.format(key_content) + + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback(meta)) + self.ds.get_data() + + self.assertIn(key_content, self.ds.get_public_ssh_keys()) + + @httpretty.activate + def test_instance_level_keys_replace_project_level_keys(self): + key_content = 'ssh-rsa JustAUser root@server' + meta = GCE_META.copy() + meta['instance/attributes/sshKeys'] = 'user:{0}'.format(key_content) + + httpretty.register_uri(httpretty.GET, MD_URL_RE, + body=_new_request_callback(meta)) + self.ds.get_data() + + self.assertEqual([key_content], self.ds.get_public_ssh_keys()) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 8b62b1b1..28b41eaf 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -24,18 +24,30 @@ from __future__ import print_function -from cloudinit import helpers as c_helpers -from cloudinit.sources import DataSourceSmartOS -from cloudinit.util import b64e -from .. import helpers import os import os.path import re import shutil -import tempfile import stat +import tempfile import uuid +from binascii import crc32 +import serial +import six + +import six + +from cloudinit import helpers as c_helpers +from cloudinit.sources import DataSourceSmartOS +from cloudinit.util import b64e + +from .. import helpers + +try: + from unittest import mock +except ImportError: + import mock MOCK_RETURNS = { 'hostname': 'test-host', @@ -54,60 +66,15 @@ MOCK_RETURNS = { DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc') -class MockSerial(object): - """Fake a serial terminal for testing the code that - interfaces with the serial""" - - port = None +def get_mock_client(mockdata): + class MockMetadataClient(object): - def __init__(self, mockdata): - self.last = None - self.last = None - self.new = True - self.count = 0 - self.mocked_out = [] - self.mockdata = mockdata + def __init__(self, serial): + pass - def open(self): - return True - - def close(self): - return True - - def isOpen(self): - return True - - def write(self, line): - line = line.replace('GET ', '') - self.last = line.rstrip() - - def readline(self): - if self.new: - self.new = False - if self.last in self.mockdata: - return 'SUCCESS\n' - else: - return 'NOTFOUND %s\n' % self.last - - if self.last in self.mockdata: - if not self.mocked_out: - self.mocked_out = [x for x in self._format_out()] - - if len(self.mocked_out) > self.count: - self.count += 1 - return self.mocked_out[self.count - 1] - - def _format_out(self): - if self.last in self.mockdata: - _mret = self.mockdata[self.last] - try: - for l in _mret.splitlines(): - yield "%s\n" % l.rstrip() - except: - yield "%s\n" % _mret.rstrip() - - yield '.' - yield '\n' + def get_metadata(self, metadata_key): + return mockdata.get(metadata_key) + return MockMetadataClient class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): @@ -155,9 +122,6 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): if dmi_data is None: dmi_data = DMI_DATA_RETURN - def _get_serial(*_): - return MockSerial(mockdata) - def _dmi_data(): return dmi_data @@ -174,7 +138,9 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): sys_cfg['datasource']['SmartOS'] = ds_cfg self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) - self.apply_patches([(mod, 'get_serial', _get_serial)]) + self.apply_patches([(mod, 'get_serial', mock.MagicMock())]) + self.apply_patches([ + (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) self.apply_patches([(os, 'uname', _os_uname)]) self.apply_patches([(mod, 'device_exists', lambda d: True)]) @@ -443,6 +409,18 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): self.assertEqual(dsrc.device_name_to_device('FOO'), mydscfg['disk_aliases']['FOO']) + @mock.patch('cloudinit.sources.DataSourceSmartOS.JoyentMetadataClient') + @mock.patch('cloudinit.sources.DataSourceSmartOS.get_serial') + def test_serial_console_closed_on_error(self, get_serial, metadata_client): + class OurException(Exception): + pass + metadata_client.side_effect = OurException + try: + DataSourceSmartOS.query_data('noun', 'device', 0) + except OurException: + pass + self.assertEqual(1, get_serial.return_value.close.call_count) + def apply_patches(patches): ret = [] @@ -453,3 +431,133 @@ def apply_patches(patches): setattr(ref, name, replace) ret.append((ref, name, orig)) return ret + + +class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): + + def setUp(self): + super(TestJoyentMetadataClient, self).setUp() + self.serial = mock.MagicMock(spec=serial.Serial) + self.request_id = 0xabcdef12 + self.metadata_value = 'value' + self.response_parts = { + 'command': 'SUCCESS', + 'crc': 'b5a9ff00', + 'length': 17 + len(b64e(self.metadata_value)), + 'payload': b64e(self.metadata_value), + 'request_id': '{0:08x}'.format(self.request_id), + } + + def make_response(): + payload = '' + if self.response_parts['payload']: + payload = ' {0}'.format(self.response_parts['payload']) + del self.response_parts['payload'] + return ( + 'V2 {length} {crc} {request_id} {command}{payload}\n'.format( + payload=payload, **self.response_parts).encode('ascii')) + self.serial.readline.side_effect = make_response + self.patched_funcs.enter_context( + mock.patch('cloudinit.sources.DataSourceSmartOS.random.randint', + mock.Mock(return_value=self.request_id))) + + def _get_client(self): + return DataSourceSmartOS.JoyentMetadataClient(self.serial) + + def assertEndsWith(self, haystack, prefix): + self.assertTrue(haystack.endswith(prefix), + "{0} does not end with '{1}'".format( + repr(haystack), prefix)) + + def assertStartsWith(self, haystack, prefix): + self.assertTrue(haystack.startswith(prefix), + "{0} does not start with '{1}'".format( + repr(haystack), prefix)) + + def test_get_metadata_writes_a_single_line(self): + client = self._get_client() + client.get_metadata('some_key') + self.assertEqual(1, self.serial.write.call_count) + written_line = self.serial.write.call_args[0][0] + self.assertEndsWith(written_line, b'\n') + self.assertEqual(1, written_line.count(b'\n')) + + def _get_written_line(self, key='some_key'): + client = self._get_client() + client.get_metadata(key) + return self.serial.write.call_args[0][0] + + def test_get_metadata_writes_bytes(self): + self.assertIsInstance(self._get_written_line(), six.binary_type) + + def test_get_metadata_line_starts_with_v2(self): + self.assertStartsWith(self._get_written_line(), b'V2') + + def test_get_metadata_uses_get_command(self): + parts = self._get_written_line().decode('ascii').strip().split(' ') + self.assertEqual('GET', parts[4]) + + def test_get_metadata_base64_encodes_argument(self): + key = 'my_key' + parts = self._get_written_line(key).decode('ascii').strip().split(' ') + self.assertEqual(b64e(key), parts[5]) + + def test_get_metadata_calculates_length_correctly(self): + parts = self._get_written_line().decode('ascii').strip().split(' ') + expected_length = len(' '.join(parts[3:])) + self.assertEqual(expected_length, int(parts[1])) + + def test_get_metadata_uses_appropriate_request_id(self): + parts = self._get_written_line().decode('ascii').strip().split(' ') + request_id = parts[3] + self.assertEqual(8, len(request_id)) + self.assertEqual(request_id, request_id.lower()) + + def test_get_metadata_uses_random_number_for_request_id(self): + line = self._get_written_line() + request_id = line.decode('ascii').strip().split(' ')[3] + self.assertEqual('{0:08x}'.format(self.request_id), request_id) + + def test_get_metadata_checksums_correctly(self): + parts = self._get_written_line().decode('ascii').strip().split(' ') + expected_checksum = '{0:08x}'.format( + crc32(' '.join(parts[3:]).encode('utf-8')) & 0xffffffff) + checksum = parts[2] + self.assertEqual(expected_checksum, checksum) + + def test_get_metadata_reads_a_line(self): + client = self._get_client() + client.get_metadata('some_key') + self.assertEqual(1, self.serial.readline.call_count) + + def test_get_metadata_returns_valid_value(self): + client = self._get_client() + value = client.get_metadata('some_key') + self.assertEqual(self.metadata_value, value) + + def test_get_metadata_throws_exception_for_incorrect_length(self): + self.response_parts['length'] = 0 + client = self._get_client() + self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, + client.get_metadata, 'some_key') + + def test_get_metadata_throws_exception_for_incorrect_crc(self): + self.response_parts['crc'] = 'deadbeef' + client = self._get_client() + self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, + client.get_metadata, 'some_key') + + def test_get_metadata_throws_exception_for_request_id_mismatch(self): + self.response_parts['request_id'] = 'deadbeef' + client = self._get_client() + client._checksum = lambda _: self.response_parts['crc'] + self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, + client.get_metadata, 'some_key') + + def test_get_metadata_returns_None_if_value_not_found(self): + self.response_parts['payload'] = '' + self.response_parts['command'] = 'NOTFOUND' + self.response_parts['length'] = 17 + client = self._get_client() + client._checksum = lambda _: self.response_parts['crc'] + self.assertIsNone(client.get_metadata('some_key')) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py new file mode 100644 index 00000000..f3109bac --- /dev/null +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -0,0 +1,311 @@ +from cloudinit.config.cc_snappy import ( + makeop, get_package_ops, render_snap_op) +from cloudinit import util +from .. import helpers as t_help + +import os +import shutil +import tempfile +import yaml + +ALLOWED = (dict, list, int, str) + + +class TestInstallPackages(t_help.TestCase): + def setUp(self): + super(TestInstallPackages, self).setUp() + self.unapply = [] + + # by default 'which' has nothing in its path + self.apply_patches([(util, 'subp', self._subp)]) + self.subp_called = [] + self.snapcmds = [] + self.tmp = tempfile.mkdtemp(prefix="TestInstallPackages") + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + shutil.rmtree(self.tmp) + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def populate_tmp(self, files): + return t_help.populate_dir(self.tmp, files) + + def _subp(self, *args, **kwargs): + # supports subp calling with cmd as args or kwargs + if 'args' not in kwargs: + kwargs['args'] = args[0] + self.subp_called.append(kwargs) + snap_cmds = [] + args = kwargs['args'] + # here we basically parse the snappy command invoked + # and append to snapcmds a list of (mode, pkg, config) + if args[0:2] == ['snappy', 'config']: + if args[3] == "-": + config = kwargs.get('data', '') + else: + with open(args[3], "rb") as fp: + config = yaml.safe_load(fp.read()) + self.snapcmds.append(['config', args[2], config]) + elif args[0:2] == ['snappy', 'install']: + config = None + pkg = None + for arg in args[2:]: + if arg.startswith("-"): + continue + if not pkg: + pkg = arg + elif not config: + cfgfile = arg + if cfgfile == "-": + config = kwargs.get('data', '') + elif cfgfile: + with open(cfgfile, "rb") as fp: + config = yaml.safe_load(fp.read()) + self.snapcmds.append(['install', pkg, config]) + + def test_package_ops_1(self): + ret = get_package_ops( + packages=['pkg1', 'pkg2', 'pkg3'], + configs={'pkg2': b'mycfg2'}, installed=[]) + self.assertEqual( + ret, [makeop('install', 'pkg1', None, None), + makeop('install', 'pkg2', b'mycfg2', None), + makeop('install', 'pkg3', None, None)]) + + def test_package_ops_config_only(self): + ret = get_package_ops( + packages=None, + configs={'pkg2': b'mycfg2'}, installed=['pkg1', 'pkg2']) + self.assertEqual( + ret, [makeop('config', 'pkg2', b'mycfg2')]) + + def test_package_ops_install_and_config(self): + ret = get_package_ops( + packages=['pkg3', 'pkg2'], + configs={'pkg2': b'mycfg2', 'xinstalled': b'xcfg'}, + installed=['xinstalled']) + self.assertEqual( + ret, [makeop('install', 'pkg3'), + makeop('install', 'pkg2', b'mycfg2'), + makeop('config', 'xinstalled', b'xcfg')]) + + def test_package_ops_install_long_config_short(self): + # a package can be installed by full name, but have config by short + cfg = {'k1': 'k2'} + ret = get_package_ops( + packages=['config-example.canonical'], + configs={'config-example': cfg}, installed=[]) + self.assertEqual( + ret, [makeop('install', 'config-example.canonical', cfg)]) + + def test_package_ops_with_file(self): + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg", + "snapf2.snap": b"foo2", "foo.bar": "ignored"}) + ret = get_package_ops( + packages=['pkg1'], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config"), + makeop_tmpd(self.tmp, 'install', 'snapf2', path="snapf2.snap"), + makeop('install', 'pkg1')]) + + def test_package_ops_common_filename(self): + # fish package name from filename + # package names likely look like: pkgname.namespace_version_arch.snap + fname = "xkcd-webserver.canonical_0.3.4_all.snap" + name = "xkcd-webserver.canonical" + shortname = "xkcd-webserver" + + # find filenames + self.populate_tmp( + {"pkg-ws.smoser_0.3.4_all.snap": "pkg-ws-snapdata", + "pkg-ws.config": "pkg-ws-config", + "pkg1.smoser_1.2.3_all.snap": "pkg1.snapdata", + "pkg1.smoser.config": "pkg1.smoser.config-data", + "pkg1.config": "pkg1.config-data", + "pkg2.smoser_0.0_amd64.snap": "pkg2-snapdata", + "pkg2.smoser_0.0_amd64.config": "pkg2.config", + }) + + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'pkg-ws.smoser', + path="pkg-ws.smoser_0.3.4_all.snap", + cfgfile="pkg-ws.config"), + makeop_tmpd(self.tmp, 'install', 'pkg1.smoser', + path="pkg1.smoser_1.2.3_all.snap", + cfgfile="pkg1.smoser.config"), + makeop_tmpd(self.tmp, 'install', 'pkg2.smoser', + path="pkg2.smoser_0.0_amd64.snap", + cfgfile="pkg2.smoser_0.0_amd64.config"), + ]) + + def test_package_ops_config_overrides_file(self): + # config data overrides local file .config + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": b"snapf1cfg"}) + ret = get_package_ops( + packages=[], configs={'snapf1': 'snapf1cfg-config'}, + installed=[], fspath=self.tmp) + self.assertEqual( + ret, [makeop_tmpd(self.tmp, 'install', 'snapf1', + path="snapf1.snap", config="snapf1cfg-config")]) + + def test_package_ops_namespacing(self): + cfgs = { + 'config-example': {'k1': 'v1'}, + 'pkg1': {'p1': 'p2'}, + 'ubuntu-core': {'c1': 'c2'}, + 'notinstalled.smoser': {'s1': 's2'}, + } + cfg = {'config-example-k1': 'config-example-k2'} + ret = get_package_ops( + packages=['config-example.canonical'], configs=cfgs, + installed=['config-example.smoser', 'pkg1.canonical', + 'ubuntu-core']) + + expected_configs = [ + makeop('config', 'pkg1', config=cfgs['pkg1']), + makeop('config', 'ubuntu-core', config=cfgs['ubuntu-core'])] + expected_installs = [ + makeop('install', 'config-example.canonical', + config=cfgs['config-example'])] + + installs = [i for i in ret if i['op'] == 'install'] + configs = [c for c in ret if c['op'] == 'config'] + + self.assertEqual(installs, expected_installs) + # configs are not ordered + self.assertEqual(len(configs), len(expected_configs)) + self.assertTrue(all(found in expected_configs for found in configs)) + + def test_render_op_localsnap(self): + self.populate_tmp({"snapf1.snap": b"foo1"}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', op['path'], None]]) + + def test_render_op_localsnap_localconfig(self): + self.populate_tmp( + {"snapf1.snap": b"foo1", 'snapf1.config': b'snapf1cfg'}) + op = makeop_tmpd(self.tmp, 'install', 'snapf1', + path='snapf1.snap', cfgfile='snapf1.config') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', op['path'], 'snapf1cfg']]) + + def test_render_op_snap(self): + op = makeop('install', 'snapf1') + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', 'snapf1', None]]) + + def test_render_op_snap_config(self): + mycfg = {'key1': 'value1'} + name = "snapf1" + op = makeop('install', name, config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['install', name, {'config': {name: mycfg}}]]) + + def test_render_op_config_bytes(self): + name = "snapf1" + mycfg = b'myconfig' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) + + def test_render_op_config_string(self): + name = 'snapf1' + mycfg = 'myconfig: foo\nhisconfig: bar\n' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + self.assertEqual( + self.snapcmds, [['config', 'snapf1', {'config': {name: mycfg}}]]) + + def test_render_op_config_dict(self): + # config entry for package can be a dict, not a string blob + mycfg = {'foo': 'bar'} + name = 'snapf1' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + # snapcmds is a list of 3-entry lists. data_found will be the + # blob of data in the file in 'snappy install --config=<file>' + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_op_config_list(self): + # config entry for package can be a list, not a string blob + mycfg = ['foo', 'bar', 'wark', {'f1': 'b1'}] + name = "snapf1" + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_op_config_int(self): + # config entry for package can be a list, not a string blob + mycfg = 1 + name = 'snapf1' + op = makeop('config', name, config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_long_configs_short(self): + # install a namespaced package should have un-namespaced config + mycfg = {'k1': 'k2'} + name = 'snapf1' + op = makeop('install', name + ".smoser", config=mycfg) + render_snap_op(**op) + data_found = self.snapcmds[0][2] + self.assertEqual(mycfg, data_found['config'][name]) + + def test_render_does_not_pad_cfgfile(self): + # package_ops with cfgfile should not modify --file= content. + mydata = "foo1: bar1\nk: [l1, l2, l3]\n" + self.populate_tmp( + {"snapf1.snap": b"foo1", "snapf1.config": mydata.encode()}) + ret = get_package_ops( + packages=[], configs={}, installed=[], fspath=self.tmp) + self.assertEqual( + ret, + [makeop_tmpd(self.tmp, 'install', 'snapf1', path="snapf1.snap", + cfgfile="snapf1.config")]) + + # now the op was ok, but test that render didn't mess it up. + render_snap_op(**ret[0]) + data_found = self.snapcmds[0][2] + # the data found gets loaded in the snapcmd interpretation + # so this comparison is a bit lossy, but input to snappy config + # is expected to be yaml loadable, so it should be OK. + self.assertEqual(yaml.safe_load(mydata), data_found) + + +def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): + if cfgfile: + cfgfile = os.path.sep.join([tmpd, cfgfile]) + if path: + path = os.path.sep.join([tmpd, path]) + return(makeop(op=op, name=name, config=config, path=path, cfgfile=cfgfile)) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret |